WSL/SLF GitLab Repository

Commit 37b40c54 authored by Thomas Egger's avatar Thomas Egger
Browse files

Speed optimizations for the SNIO plugin: 20-30% performance gain.

Basically two features were added:
- saving the file pointer, thus less searches are necessary within the file parsed
- not parsing the whole line if the date is not within the interval we search for
parent fd9e9cda
......@@ -280,6 +280,8 @@ void SNIO::readMeteoData(const Date& dateStart, const Date& dateEnd,
vecMeteo.clear();
vecMeteo.insert(vecMeteo.begin(), vecAllStations.size(), vector<MeteoData>());
if (vec_streampos.size() == 0) //the vec_streampos save file pointers for certain dates
vec_streampos = vector< map<Date, std::streampos> >(vecAllStations.size());
for (unsigned int ii=0; ii<vecAllStations.size(); ii++){
string filename="", line="";
......@@ -315,7 +317,14 @@ void SNIO::readMeteoData(const Date& dateStart, const Date& dateEnd,
unsigned int linenr = 0;
//The following 4 lines are an optimization to jump to the correct position in the file
streampos current_fpointer = -1; //the filepointer for the current valid date
map<Date,streampos>::const_iterator it = vec_streampos.at(ii).find(dateStart);
if (it != vec_streampos.at(ii).end())
fin.seekg(it->second); //jump to position in the file
while (!fin.eof()){
streampos tmp_fpointer = fin.tellg();
getline(fin, line, eoln); //read complete line of data
stringstream ss;
......@@ -327,11 +336,12 @@ void SNIO::readMeteoData(const Date& dateStart, const Date& dateEnd,
if (ncols >= nr_meteoData){
MeteoData md;
md.meta = vecAllStations[ii];
parseMeteoLine(tmpvec, file_with_path.str() + ":" + ss.str(), md);
parseMeteoLine(tmpvec, file_with_path.str() + ":" + ss.str(), dateStart, dateEnd, md);
if ((md.date >= dateStart) && (md.date <= dateEnd)){//check date and add to vectors
convertUnits(md);
vecMeteo[ii].push_back(md);
current_fpointer = tmp_fpointer; //save this file pointer, it's a valid one for sure
}
} else if (ncols == 1){
if (tmpvec.at(0) == "END") {
......@@ -345,6 +355,9 @@ void SNIO::readMeteoData(const Date& dateStart, const Date& dateEnd,
throw InvalidFormatException(file_with_path.str() + ":line " + ss.str() + " premature end of line", AT);
}
}
//save stream position and the corresponding end date
if (current_fpointer != -1) vec_streampos.at(ii)[dateEnd] = current_fpointer;
} catch (std::exception& e){
cleanup();
throw;
......@@ -353,7 +366,8 @@ void SNIO::readMeteoData(const Date& dateStart, const Date& dateEnd,
}
}
void SNIO::parseMeteoLine(const std::vector<std::string>& vecLine, const std::string& filepos, MeteoData& md)
void SNIO::parseMeteoLine(const std::vector<std::string>& vecLine, const std::string& filepos,
const Date& dateStart, const Date& dateEnd, MeteoData& md)
{
/*
* This function takes a meteo line, extracts the date (ignores Julian) and then converts
......@@ -375,6 +389,9 @@ void SNIO::parseMeteoLine(const std::vector<std::string>& vecLine, const std::st
if (!IOUtils::convertString(md.date, year+"-"+month+"-"+day+"T"+vecLine[2], in_tz, std::dec))
throw InvalidFormatException("Reading station "+md.meta.stationID+", at "+filepos+": invalid date format", AT);
if ((md.date < dateStart) || (md.date > dateEnd)) //stop parsing data for dates out of the scope
return;
//Extract all data as double values
vector<double> tmpdata = vector<double>(vecLine.size());
for (unsigned int ii=4; ii<vecLine.size(); ii++) {
......
......@@ -71,7 +71,8 @@ class SNIO : public IOInterface {
void convertUnits(MeteoData& meteo);
void convertUnitsBack(MeteoData& meteo);
double cloudiness_to_ilwr (const double& RH, const double& TA, const double& cloudiness );
void parseMeteoLine(const std::vector<std::string>& vecLine, const std::string& filepos, MeteoData& md);
void parseMeteoLine(const std::vector<std::string>& vecLine, const std::string& filepos,
const Date& dateStart, const Date& dateEnd, MeteoData& md);
bool readStationMetaData(const std::string& metafile, const std::string& stationname, StationData& sd);
void readMetaData(unsigned int& nrOfStations);
void parseMetaDataLine(const std::vector<std::string>& vecLine, StationData& sd);
......@@ -88,6 +89,7 @@ class SNIO : public IOInterface {
unsigned int nr_meteoData; // number of parameters on data input lines, excluding optional ones
std::string coordin, coordinparam, coordout, coordoutparam; //projection parameters
std::vector<StationData> vecAllStations;
std::vector< std::map <Date, std::streampos> > vec_streampos; //in order to save file pointers
};
} //namespace
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment