WSL/SLF GitLab Repository

Commit ebddecb6 authored by Mathias Bavay's avatar Mathias Bavay
Browse files

The FileIndexer class has been expanded to cater to the needs of SMET (it is...

The FileIndexer class has been expanded to cater to the needs of SMET (it is not the most efficient, but for now...). A bug has been fixed that was leading to segfault. The explanation for the remaining slowness of SNIO has been found: the Config keys were re-read for every parsed line... It is now 10x faster, ie up to par with SMET (more or less).
parent f531e8cd
......@@ -756,6 +756,19 @@ void FileIndexer::setIndex(const Date& i_date, const std::streampos& i_pos)
else vecIndex.insert(vecIndex.begin()+insertIdx, tmp_idx); //insertion is at the proper place -> remains ordered
}
void FileIndexer::setIndex(const std::string& i_date, const std::streampos& i_pos)
{
Date tmpdate;
convertString(tmpdate, i_date, 0.);
setIndex(tmpdate, i_pos);
}
void FileIndexer::setIndex(const double& i_date, const std::streampos& i_pos)
{
Date tmpdate(i_date, 0.);
setIndex(tmpdate, i_pos);
}
std::streampos FileIndexer::getIndex(const Date& i_date)
{
const size_t foundIdx = binarySearch(i_date);
......@@ -763,12 +776,25 @@ std::streampos FileIndexer::getIndex(const Date& i_date)
else return vecIndex[foundIdx].pos;
}
std::streampos FileIndexer::getIndex(const std::string& i_date)
{
Date tmpdate;
convertString(tmpdate, i_date, 0.);
return getIndex(tmpdate);
}
std::streampos FileIndexer::getIndex(const double& i_date)
{
Date tmpdate(i_date, 0.);
return getIndex(tmpdate);
}
size_t FileIndexer::binarySearch(const Date& soughtdate)
{//perform binary search
const size_t vec_size = vecIndex.size();
if(vec_size==0) return (size_t)-1;
if(soughtdate<vecIndex[0].date) return (size_t)-1;
if(soughtdate>=vecIndex[vec_size-1].date) return vec_size;
if(soughtdate>=vecIndex[vec_size-1].date) return vec_size-1;
const file_index value(soughtdate, 0);
const std::vector< struct file_index >::iterator it = std::upper_bound(vecIndex.begin(), vecIndex.end(), value); //returns the first element that is GREATER than the provided value
......
......@@ -385,6 +385,8 @@ namespace IOUtils {
* @param[in] i_pos streampos position
*/
void setIndex(const Date& i_date, const std::streampos& i_pos);
void setIndex(const std::string& i_date, const std::streampos& i_pos);
void setIndex(const double& i_date, const std::streampos& i_pos);
/**
* @brief Get the file position suitable for a given date
......@@ -393,6 +395,8 @@ namespace IOUtils {
* -1 if nothing could be found (empty index)
*/
std::streampos getIndex(const Date& i_date);
std::streampos getIndex(const std::string& i_date);
std::streampos getIndex(const double& i_date);
friend std::ostream& operator<<(std::ostream &os, const FileIndexer& index);
......
......@@ -83,6 +83,7 @@ SNIO::SNIO(const std::string& configfile)
vecAllStations(), vecIndex(), fin(), fout(),
coordin(), coordinparam(), coordout(), coordoutparam(),
in_tz(0.), out_tz(0.), nr_meteoData(min_nr_meteoData),
number_meas_temperatures(0), number_of_solutes (0), vw_drift(false), rho_hn(false),
iswr_inp(true), rswr_inp(true)
{
IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
......@@ -92,6 +93,10 @@ SNIO::SNIO(const std::string& configfile)
cfg.getValue("RSWR_INP","Input",rswr_inp,Config::nothrow);
if (!iswr_inp || !rswr_inp)
nr_meteoData = min_nr_meteoData - 1;
cfg.getValue("NUMBER_MEAS_TEMPERATURES", "Input", number_meas_temperatures, Config::nothrow);
cfg.getValue("NUMBER_OF_SOLUTES", "Input", number_of_solutes, Config::nothrow);
cfg.getValue("VW_DRIFT", "Input", vw_drift, Config::nothrow);
cfg.getValue("RHO_HN", "Input", rho_hn, Config::nothrow);
}
SNIO::SNIO(const Config& cfgreader)
......@@ -99,6 +104,7 @@ SNIO::SNIO(const Config& cfgreader)
vecAllStations(), vecIndex(), fin(), fout(),
coordin(), coordinparam(), coordout(), coordoutparam(),
in_tz(0.), out_tz(0.), nr_meteoData(min_nr_meteoData),
number_meas_temperatures(0), number_of_solutes (0), vw_drift(false), rho_hn(false),
iswr_inp(true), rswr_inp(true)
{
IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
......@@ -108,6 +114,10 @@ SNIO::SNIO(const Config& cfgreader)
cfg.getValue("RSWR_INP","Input",rswr_inp,Config::nothrow);
if (!iswr_inp || !rswr_inp)
nr_meteoData = min_nr_meteoData - 1;
cfg.getValue("NUMBER_MEAS_TEMPERATURES", "Input", number_meas_temperatures, Config::nothrow);
cfg.getValue("NUMBER_OF_SOLUTES", "Input", number_of_solutes, Config::nothrow);
cfg.getValue("VW_DRIFT", "Input", vw_drift, Config::nothrow);
cfg.getValue("RHO_HN", "Input", rho_hn, Config::nothrow);
}
SNIO::~SNIO() throw()
......@@ -341,7 +351,6 @@ void SNIO::readMeteoData(const Date& dateStart, const Date& dateEnd,
*/
vector<string> tmpvec;
string inpath = "";
cfg.getValue("METEOPATH", "Input", inpath);
if (vecAllStations.size() == 0)
......@@ -508,8 +517,6 @@ bool SNIO::parseMeteoLine(const std::vector<std::string>& vecLine, const std::st
// Read optional values
// TS[]: snow temperatures
size_t number_meas_temperatures = 0;
cfg.getValue("NUMBER_MEAS_TEMPERATURES", "Input", number_meas_temperatures, Config::nothrow);
if (vecLine.size() < nr_meteoData + number_meas_temperatures)
throw InvalidFormatException("Reading station "+md.meta.stationID+", at "+file_pos(filename, linenr)+": not enough measured temperatures data", AT);
......@@ -520,8 +527,6 @@ bool SNIO::parseMeteoLine(const std::vector<std::string>& vecLine, const std::st
md(ss.str()) = tmpdata[ii++];
}
// CONC[]: solute concentrations
size_t number_of_solutes = 0;
cfg.getValue("NUMBER_OF_SOLUTES", "Input", number_of_solutes, Config::nothrow);
if (vecLine.size() < nr_meteoData + number_meas_temperatures + number_of_solutes)
throw InvalidFormatException("Reading station "+md.meta.stationID+", at "+file_pos(filename, linenr)+": not enough solute data", AT);
......@@ -532,8 +537,6 @@ bool SNIO::parseMeteoLine(const std::vector<std::string>& vecLine, const std::st
md(ss.str()) = tmpdata[ii++];
}
// VW_DRIFT: optional wind velocity for blowing and drifting snow
bool vw_drift = false;
cfg.getValue("VW_DRIFT", "Input", vw_drift, Config::nothrow);
if (vw_drift) {
if (vecLine.size() < ii+1)
throw InvalidFormatException("Reading station "+md.meta.stationID+", at "+file_pos(filename, linenr)+": no data for vw_drift", AT);
......@@ -541,8 +544,6 @@ bool SNIO::parseMeteoLine(const std::vector<std::string>& vecLine, const std::st
md("VW_DRIFT") = tmpdata[ii++];
}
// RHO_HN: measured new snow density
bool rho_hn = false;
cfg.getValue("RHO_HN", "Input", rho_hn, Config::nothrow);
if (rho_hn) {
if (vecLine.size() < ii+1)
throw InvalidFormatException("Reading station "+md.meta.stationID+", at "+file_pos(filename, linenr)+": no data for rho_hn", AT);
......@@ -743,22 +744,22 @@ void SNIO::writeStationMeteo(const std::vector<MeteoData>& vecmd, const std::str
}
// VW_DRIFT: optional wind velocity for blowing and drifting snow
if (vecmd[jj].param_exists("VW_DRIFT")) {
const double vw_drift = vecmd[jj]("VW_DRIFT");
if (vw_drift == IOUtils::nodata) {
const double vw_drift_val = vecmd[jj]("VW_DRIFT");
if (vw_drift_val == IOUtils::nodata) {
optional_failure_count++;
fout << setw(4) << setprecision(0) << vw_drift << " ";
fout << setw(4) << setprecision(0) << vw_drift_val << " ";
} else {
fout << setw(4) << setprecision(1) << vw_drift << " ";
fout << setw(4) << setprecision(1) << vw_drift_val << " ";
}
}
// RHO_HN: measured new snow density
if (vecmd[jj].param_exists("RHO_HN")) {
const double rho_hn = vecmd[jj]("RHO_HN");
if (rho_hn == IOUtils::nodata) {
const double rho_hn_val = vecmd[jj]("RHO_HN");
if (rho_hn_val == IOUtils::nodata) {
optional_failure_count++;
fout << setw(6) << setprecision(0) << rho_hn << " ";
fout << setw(6) << setprecision(0) << rho_hn_val << " ";
} else {
fout << setw(6) << setprecision(1) << rho_hn << " ";
fout << setw(6) << setprecision(1) << rho_hn_val << " ";
}
}
......
......@@ -92,6 +92,8 @@ class SNIO : public IOInterface {
static const size_t min_nr_meteoData; // minimal number of parameters on data input lines
static const size_t streampos_every_n_lines; //save current stream pos every n lines of data
size_t nr_meteoData; // number of parameters on data input lines, excluding optional ones
size_t number_meas_temperatures, number_of_solutes;
bool vw_drift, rho_hn;
bool iswr_inp, rswr_inp;
};
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment