WSL/SLF GitLab Repository

Commit 5cf5f9ad authored by Mathias Bavay's avatar Mathias Bavay
Browse files

A new version of the fill method for arrays has been implemented: it only...

A new version of the fill method for arrays has been implemented: it only takes a sub-array and the lower left coordinate where to insert it. The file reading optimization that has been implemented in GeotTop and SNIO has also been implemented in SMETIO.
parent f3d55dbe
......@@ -117,6 +117,8 @@ template<class T> class Array2D {
void fill(const Array2D<T>& i_array2D, const unsigned int& i_nx, const unsigned int& i_ny,
const unsigned int& i_ncols, const unsigned int& i_nrows);
void fill(const Array2D<T>& i_array2D, const unsigned int& i_nx, const unsigned int& i_ny);
void resize(const unsigned int& nx, const unsigned int& ny);
void resize(const unsigned int& nx, const unsigned int& ny, const T& init);
void size(unsigned int& nx, unsigned int& ny) const;
......@@ -253,6 +255,13 @@ template<class T> void Array2D<T>::subset(const Array2D<T>& i_array2D, const uns
}
}
template<class T> void Array2D<T>::fill(const Array2D<T>& i_array2D, const unsigned int& i_nx, const unsigned int& i_ny)
{
unsigned int i_ncols, i_nrows;
i_array2D.size(i_ncols, i_nrows);
fill(i_array2D, i_nx, i_ny, i_ncols, i_nrows);
}
template<class T> void Array2D<T>::fill(const Array2D<T>& i_array2D, const unsigned int& i_nx, const unsigned int& i_ny,
const unsigned int& i_ncols, const unsigned int& i_nrows)
{
......
......@@ -152,6 +152,8 @@ template<class T> class Array3D {
const unsigned int& i_nx, const unsigned int& i_ny, const unsigned int& i_nz,
const unsigned int& i_ncols, const unsigned int& i_nrows, const unsigned int& i_ndepth);
void fill(const Array3D<T>& i_array3D, const unsigned int& i_nx, const unsigned int& i_ny, const unsigned int& i_nz);
void resize(const unsigned int& anx, const unsigned int& any, const unsigned int& anz);
void resize(const unsigned int& anx, const unsigned int& any, const unsigned int& anz, const T& init);
void size(unsigned int& anx, unsigned int& any, unsigned int& anz) const;
......@@ -298,6 +300,13 @@ template<class T> void Array3D<T>::subset(const Array3D<T>& i_array3D,
}
}
template<class T> void Array3D<T>::fill(const Array3D<T>& i_array3D, const unsigned int& i_nx, const unsigned int& i_ny, const unsigned int& i_nz)
{
unsigned int i_ncols, i_nrows, i_ndepth;
i_array3D.size(i_ncols, i_nrows, i_ndepth);
fill(i_array3D, i_nx, i_ny, i_nz, i_ncols, i_nrows, i_ndepth);
}
template<class T> void Array3D<T>::fill(const Array3D<T>& i_array3D,
const unsigned int& i_nx, const unsigned int& i_ny, const unsigned int& i_nz,
const unsigned int& i_ncols, const unsigned int& i_nrows, const unsigned int& i_ndepth)
......
......@@ -279,6 +279,9 @@ void SMETIO::parseInputOutputSection()
nr_stations = counter - 1;
}
if (vec_streampos.size() == 0) //the vec_streampos save file pointers for certain dates
vec_streampos = vector< map<Date, std::streampos> >(nr_stations);
//Parse output section: extract info on whether to write ASCII or BINARY format, gzipped or not
outpath = "";
outputIsAscii = true;
......@@ -379,7 +382,7 @@ void SMETIO::readMeteoData(const Date& dateStart, const Date& dateEnd,
//3. Read DATA
if (isAscii){
readDataAscii(eoln, filename, timezone, sd, vecDataSequence, vecUnitsOffset,
vecUnitsMultiplier, dateStart, dateEnd, vecMeteo[ii]);
vecUnitsMultiplier, dateStart, dateEnd, vecMeteo[ii], ii);
} else {
streampos currpos = fin.tellg();
fin.close();
......@@ -469,7 +472,7 @@ void SMETIO::readDataBinary(const char&, const std::string&, const double& timez
void SMETIO::readDataAscii(const char& eoln, const std::string& filename, const double& timezone,
const StationData& sd, const std::vector<std::string>& vecDataSequence,
const std::vector<double>& vecUnitsOffset, std::vector<double>& vecUnitsMultiplier,
const Date& dateStart, const Date& dateEnd, std::vector<MeteoData>& vecMeteo)
const Date& dateStart, const Date& dateEnd, std::vector<MeteoData>& vecMeteo, const size_t& stat_idx)
{
string line = "";
vector<string> tmpvec;
......@@ -478,9 +481,14 @@ void SMETIO::readDataAscii(const char& eoln, const std::string& filename, const
tmpvec.reserve(nrOfColumns);
vecMeteo.reserve(buffer_reserve);
//The following 4 lines are an optimization to jump to the correct position in the file
streampos current_fpointer = -1; //the filepointer for the current valid date
map<Date,streampos>::const_iterator it = vec_streampos.at(stat_idx).find(dateStart);
if (it != vec_streampos.at(stat_idx).end())
fin.seekg(it->second); //jump to position in the file
while (!fin.eof()){
//HACK nodata mapping is NOT done!!!!!!
//something like lat = IOUtils::standardizeNodata(lat, plugin_nodata); should be done
streampos tmp_fpointer = fin.tellg();
getline(fin, line, eoln);
IOUtils::stripComments(line);
IOUtils::trim(line);
......@@ -501,8 +509,11 @@ void SMETIO::readDataAscii(const char& eoln, const std::string& filename, const
throw InvalidFormatException("In "+filename+": Timestamp "+tmpvec[ii]+" invalid in data line", AT);
if (md.date < dateStart)
continue;
if (md.date > dateEnd)
if (md.date > dateEnd) {
//save stream position and the corresponding end date
if (current_fpointer != ((ifstream::pos_type)-1)) vec_streampos.at(stat_idx)[dateEnd] = current_fpointer;
return;
}
} else if (vecDataSequence[ii] == "latitude"){
if (!IOUtils::convertString(lat, tmpvec[ii]))
......@@ -537,6 +548,7 @@ void SMETIO::readDataAscii(const char& eoln, const std::string& filename, const
if (md.date >= dateStart){
md.meta = tmpsd;
vecMeteo.push_back(md);
current_fpointer = tmp_fpointer; //save this file pointer, it's a valid one for sure
}
}
}
......@@ -689,16 +701,15 @@ void SMETIO::writeMeteoData(const std::vector< std::vector<MeteoData> >& vecMete
//1. check consitency of station data position -> write location in header or data section
StationData sd;
sd.position.setProj(coordout, coordoutparam);
bool isConsistent = checkConsistency(vecMeteo.at(ii), sd);
const bool isConsistent = checkConsistency(vecMeteo.at(ii), sd);
if (sd.stationID == ""){
stringstream ss;
ss << "Station" << ii+1;
sd.stationID = ss.str();
}
string filename = outpath + "/" + sd.stationID + ".smet";
const string filename = outpath + "/" + sd.stationID + ".smet";
if (!IOUtils::validFileName(filename)) //Check whether filename is valid
throw InvalidFileNameException(filename, AT);
......
......@@ -52,7 +52,7 @@ class SMETIO : public IOInterface {
std::vector< std::vector<MeteoData> >& vecMeteo,
const unsigned int& stationindex=IOUtils::npos);
virtual void writeMeteoData(const std::vector< std::vector<MeteoData> >& vecMeteo,
virtual void writeMeteoData(const std::vector< std::vector<MeteoData> >& vecMeteo,
const std::string& name="");
virtual void readAssimilationData(const Date&, Grid2DObject& da_out);
......@@ -64,16 +64,16 @@ class SMETIO : public IOInterface {
static const unsigned int buffer_reserve; //for optimizing vectors (ie: avoid unecessary resizing)
static std::map<std::string, MeteoData::Parameters> mapParameterByName; ///<Associate name and meteo parameter
static const bool __init; ///<helper variable to enable the init of static collection data
static bool initStaticData();///<initialize the static map meteoparamname
static bool initStaticData();///<initialize the static map meteoparamname
static double& getParameter(const std::string& columnName, MeteoData& md);
static void checkColumnNames(const std::vector<std::string>& vecColumns, const bool& locationInHeader);
void cleanup() throw();
void parseInputOutputSection();
bool checkConsistency(const std::vector<MeteoData>& vecMeteo, StationData& sd);
void checkForUsedParameters(const std::vector<MeteoData>& vecMeteo, double& timezone,
void checkForUsedParameters(const std::vector<MeteoData>& vecMeteo, double& timezone,
std::vector<bool>& vecParamInUse);
void writeHeaderSection(const bool& writeLocationInHeader, const StationData& sd,
void writeHeaderSection(const bool& writeLocationInHeader, const StationData& sd,
const double& timezone, const std::vector<bool>& vecParamInUse);
void writeDataAscii(const bool& writeLocationInHeader, const std::vector<MeteoData>& vecMeteo,
const std::vector<bool>& vecParamInUse);
......@@ -86,7 +86,7 @@ class SMETIO : public IOInterface {
void readDataAscii(const char& eoln, const std::string& filename, const double& timezone,
const StationData& sd, const std::vector<std::string>& vecDataSequence,
const std::vector<double>& vecUnitsOffset, std::vector<double>& vecUnitsMultiplier,
const Date& dateStart, const Date& dateEnd, std::vector<MeteoData>& vecMeteo);
const Date& dateStart, const Date& dateEnd, std::vector<MeteoData>& vecMeteo, const size_t& stat_idx);
void readDataBinary(const char& eoln, const std::string& filename, const double& timezone,
const StationData& sd, const std::vector<std::string>& vecDataSequence,
const std::vector<double>& vecUnitsOffset, std::vector<double>& vecUnitsMultiplier,
......@@ -106,6 +106,7 @@ class SMETIO : public IOInterface {
std::ifstream fin; //Input file streams
std::ofstream fout; //Output file streams
std::string coordin, coordinparam, coordout, coordoutparam; //default projection parameters
std::vector< std::map <Date, std::streampos> > vec_streampos; //in order to save file pointers
};
} //namespace
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment