WSL/SLF GitLab Repository

Commit d4021fad authored by Mathias Bavay's avatar Mathias Bavay
Browse files

Some of theOSDH file names have changed. Some of the fields are not available...

Some of theOSDH file names have changed. Some of the fields are not available anymore. Code cleanup in DBO.
parent c5a1c16e
...@@ -242,6 +242,8 @@ std::vector<double> getDoubles(const std::string& path, picojson::value& v) ...@@ -242,6 +242,8 @@ std::vector<double> getDoubles(const std::string& path, picojson::value& v)
//converts C to Kelvin, converts RH to [0,1], HS to m //converts C to Kelvin, converts RH to [0,1], HS to m
double convertUnits(const MeteoData::Parameters& param, const double& value) double convertUnits(const MeteoData::Parameters& param, const double& value)
{ {
if (value==IOUtils::nodata) return value;
switch (param) { switch (param) {
case MeteoData::TA: case MeteoData::TSG: case MeteoData::TSS: case MeteoData::TA: case MeteoData::TSG: case MeteoData::TSS:
return IOUtils::C_TO_K(value); return IOUtils::C_TO_K(value);
...@@ -266,8 +268,11 @@ bool parseTsPoint(const picojson::value& v, Date& datum, double& value) ...@@ -266,8 +268,11 @@ bool parseTsPoint(const picojson::value& v, Date& datum, double& value)
if (array[1].is<double>()) if (array[1].is<double>())
value = array[1].get<double>(); value = array[1].get<double>();
else else {
return false; if (!array[1].is<picojson::null>()) return false;
value = IOUtils::nodata;
return true;
}
return true; return true;
} }
...@@ -285,6 +290,7 @@ void parseTimeSerie(const std::string& tsID, const MeteoData::Parameters& param, ...@@ -285,6 +290,7 @@ void parseTimeSerie(const std::string& tsID, const MeteoData::Parameters& param,
Date datum; Date datum;
double value; double value;
if (!parseTsPoint(vecRaw[ii], datum, value)) { if (!parseTsPoint(vecRaw[ii], datum, value)) {
printJSON(vecRaw[ii], 1);
std::ostringstream ss; ss << "Error parsing element " << ii << " of timeserie " << tsID; std::ostringstream ss; ss << "Error parsing element " << ii << " of timeserie " << tsID;
throw InvalidFormatException(ss.str(), AT); throw InvalidFormatException(ss.str(), AT);
} }
...@@ -298,13 +304,21 @@ void parseTimeSerie(const std::string& tsID, const MeteoData::Parameters& param, ...@@ -298,13 +304,21 @@ void parseTimeSerie(const std::string& tsID, const MeteoData::Parameters& param,
Date datum; Date datum;
double value; double value;
if (!parseTsPoint(vecRaw[ii], datum, value)) { if (!parseTsPoint(vecRaw[ii], datum, value)) {
printJSON(vecRaw[ii], 1);
std::ostringstream ss; ss << "Error parsing element " << ii << " of timeserie " << tsID; std::ostringstream ss; ss << "Error parsing element " << ii << " of timeserie " << tsID;
throw InvalidFormatException(ss.str(), AT); throw InvalidFormatException(ss.str(), AT);
} }
/*MeteoData md(datum, sd); //easy case: all TS have the same indices
md(param) = convertUnits(param, value); if (ii<vecMeteo.size()) {
vecMeteo.push_back( md );*/ if (datum==vecMeteo[ii].date) {
vecMeteo[ii](param) = convertUnits(param, value);
continue;
}
}
//hard case: the TS don't match HACK
} }
} }
} }
...@@ -330,7 +344,7 @@ std::map<std::string, std::vector<DBO::tsMeta> > getTsProperties(picojson::value ...@@ -330,7 +344,7 @@ std::map<std::string, std::vector<DBO::tsMeta> > getTsProperties(picojson::value
for (size_t jj=0; jj<array.size(); jj++) { for (size_t jj=0; jj<array.size(); jj++) {
if (! array[jj].is<picojson::null>()) { if (! array[jj].is<picojson::null>()) {
std::string code, device_code, agg_type; std::string code, device_code, agg_type;
double ts_id; double id;
unsigned int interval; unsigned int interval;
Date since, until; Date since, until;
...@@ -338,7 +352,7 @@ std::map<std::string, std::vector<DBO::tsMeta> > getTsProperties(picojson::value ...@@ -338,7 +352,7 @@ std::map<std::string, std::vector<DBO::tsMeta> > getTsProperties(picojson::value
for (picojson::value::object::const_iterator it = obj.begin(); it != obj.end(); ++it) { for (picojson::value::object::const_iterator it = obj.begin(); it != obj.end(); ++it) {
if (it->first=="code" && it->second.is<std::string>()) code = it->second.get<std::string>(); if (it->first=="code" && it->second.is<std::string>()) code = it->second.get<std::string>();
if (it->first=="deviceCode" && it->second.is<std::string>()) device_code = it->second.get<std::string>(); if (it->first=="deviceCode" && it->second.is<std::string>()) device_code = it->second.get<std::string>();
if (it->first=="id" && it->second.is<double>()) ts_id = it->second.get<double>(); if (it->first=="id" && it->second.is<double>()) id = it->second.get<double>();
if (it->first=="since" && it->second.is<std::string>()) IOUtils::convertString(since, it->second.get<std::string>(), 0.); if (it->first=="since" && it->second.is<std::string>()) IOUtils::convertString(since, it->second.get<std::string>(), 0.);
if (it->first=="until" && it->second.is<std::string>()) IOUtils::convertString(until, it->second.get<std::string>(), 0.); if (it->first=="until" && it->second.is<std::string>()) IOUtils::convertString(until, it->second.get<std::string>(), 0.);
if (it->first=="aggregationType" && it->second.is<std::string>()) agg_type = it->second.get<std::string>(); if (it->first=="aggregationType" && it->second.is<std::string>()) agg_type = it->second.get<std::string>();
...@@ -348,9 +362,8 @@ std::map<std::string, std::vector<DBO::tsMeta> > getTsProperties(picojson::value ...@@ -348,9 +362,8 @@ std::map<std::string, std::vector<DBO::tsMeta> > getTsProperties(picojson::value
if (device_code=="BATTERY" || device_code=="LOGGER") break; if (device_code=="BATTERY" || device_code=="LOGGER") break;
if (agg_type=="SD") break; //we don't care about standard deviation anyway if (agg_type=="SD") break; //we don't care about standard deviation anyway
std::string param_str( code.substr(0, code.find('_')) ); const std::string param_str( IOUtils::strToUpper( code.substr(0, code.find('_')) ) );
IOUtils::toUpper( param_str ); tsMap[param_str].push_back( DBO::tsMeta(since, until, agg_type, id, interval) );
tsMap[param_str].push_back( DBO::tsMeta(since, until, agg_type, ts_id, interval) );
} }
} }
} }
...@@ -452,7 +465,11 @@ void DBO::fillStationMeta() ...@@ -452,7 +465,11 @@ void DBO::fillStationMeta()
throw IOException("Could not retrieve data for station " + station_id, AT); throw IOException("Could not retrieve data for station " + station_id, AT);
} }
} }
}
//read all data for the given station
void DBO::readData(const Date& dateStart, const Date& dateEnd, std::vector<MeteoData>& vecMeteo, const size_t& stationindex)
{
//debug info //debug info
/*for(size_t ii=0; ii<vecStationName.size(); ii++) { /*for(size_t ii=0; ii<vecStationName.size(); ii++) {
for (std::map<std::string, std::vector<DBO::tsMeta> >::iterator it = vecTsMeta[ii].begin(); it != vecTsMeta[ii].end(); ++it) { for (std::map<std::string, std::vector<DBO::tsMeta> >::iterator it = vecTsMeta[ii].begin(); it != vecTsMeta[ii].end(); ++it) {
...@@ -460,19 +477,27 @@ void DBO::fillStationMeta() ...@@ -460,19 +477,27 @@ void DBO::fillStationMeta()
std::cout << it->first << " " << it->second[jj].toString() << "\n"; std::cout << it->first << " " << it->second[jj].toString() << "\n";
} }
}*/ }*/
}
//read all data for the given station
void DBO::readData(const Date& dateStart, const Date& dateEnd, std::vector<MeteoData>& vecMeteo, const size_t& stationindex)
{
//TODO: for station stationindex, loop over the timeseries that cover [dateStart, dateEnd] for the current station //TODO: for station stationindex, loop over the timeseries that cover [dateStart, dateEnd] for the current station
//vecTsMeta[ stationindex ] //vecTsMeta[ stationindex ]
const unsigned int ts_id = 39; /*for (std::map<std::string, std::vector<DBO::tsMeta> >::iterator it = vecTsMeta[stationindex].begin(); it != vecTsMeta[stationindex].end(); ++it) {
readTimeSerie(ts_id, dateStart, dateEnd, vecMeta[stationindex], vecMeteo); for(size_t jj=0; jj<it->second.size(); jj++)
std::cout << it->first << " " << it->second[jj].toString() << "\n";
}*/
readTimeSerie(15, MeteoData::TA, dateStart, dateEnd, vecMeta[stationindex], vecMeteo);
readTimeSerie(23, MeteoData::RH, dateStart, dateEnd, vecMeta[stationindex], vecMeteo);
readTimeSerie(93, MeteoData::RSWR, dateStart, dateEnd, vecMeta[stationindex], vecMeteo);
readTimeSerie(31, MeteoData::HS, dateStart, dateEnd, vecMeta[stationindex], vecMeteo);
readTimeSerie(46, MeteoData::TSG, dateStart, dateEnd, vecMeta[stationindex], vecMeteo);
/*for (size_t param=MeteoData::firstparam; param<=MeteoData::lastparam; param++) {
}*/
} }
void DBO::readTimeSerie(const unsigned int& ts_id, const Date& dateStart, const Date& dateEnd, const StationData& sd, std::vector<MeteoData>& vecMeteo) void DBO::readTimeSerie(const unsigned int& ts_id, const MeteoData::Parameters& param, const Date& dateStart, const Date& dateEnd, const StationData& sd, std::vector<MeteoData>& vecMeteo)
{ {
std::ostringstream ss_ID; ss_ID << ts_id; std::ostringstream ss_ID; ss_ID << ts_id;
const std::string base_url( data_endpoint + ss_ID.str() ); const std::string base_url( data_endpoint + ss_ID.str() );
...@@ -482,12 +507,12 @@ void DBO::readTimeSerie(const unsigned int& ts_id, const Date& dateStart, const ...@@ -482,12 +507,12 @@ void DBO::readTimeSerie(const unsigned int& ts_id, const Date& dateStart, const
std::stringstream ss; std::stringstream ss;
if (curl_read(request, ss)) { if (curl_read(request, ss)) {
if (ss.str().empty()) throw UnknownValueException("Timeserie not found: '"+ss_ID.str()+"'", AT); if (ss.str().empty()) throw UnknownValueException("Timeseries not found: '"+ss_ID.str()+"'", AT);
picojson::value v; picojson::value v;
const std::string err( picojson::parse(v, ss.str()) ); const std::string err( picojson::parse(v, ss.str()) );
if (!err.empty()) throw IOException("Error while parsing JSON: "+err, AT); if (!err.empty()) throw IOException("Error while parsing JSON: "+err, AT);
parseTimeSerie(ss_ID.str(), MeteoData::RH, sd, v, vecMeteo); parseTimeSerie(ss_ID.str(), param, sd, v, vecMeteo);
} else { } else {
if (dbo_debug) if (dbo_debug)
std::cout << "****\nRequest: " << request << "\n****\n"; std::cout << "****\nRequest: " << request << "\n****\n";
......
...@@ -54,8 +54,8 @@ class DBO : public IOInterface { ...@@ -54,8 +54,8 @@ class DBO : public IOInterface {
std::string toString() { std::string toString() {
std::ostringstream os; std::ostringstream os;
os << ts_id << " ["; os << ts_id << " [";
if (since.isUndef()) os << " -∞ - "; else os << since.toString(Date::ISO) << " - "; os << ((since.isUndef())? "-∞" : since.toString(Date::ISO)) << " - ";
if (until.isUndef()) os << " ∞] "; else os << until.toString(Date::ISO) << "] "; os << ((until.isUndef())? "∞" : until.toString(Date::ISO)) << "] ";
os << agg_type << " - " << interval << " s"; os << agg_type << " - " << interval << " s";
return os.str(); return os.str();
} }
...@@ -68,7 +68,7 @@ class DBO : public IOInterface { ...@@ -68,7 +68,7 @@ class DBO : public IOInterface {
private: private:
void fillStationMeta(); void fillStationMeta();
void readData(const Date& dateStart, const Date& dateEnd, std::vector<MeteoData>& vecMeteo, const size_t& stationindex); void readData(const Date& dateStart, const Date& dateEnd, std::vector<MeteoData>& vecMeteo, const size_t& stationindex);
void readTimeSerie(const unsigned int& ts_id, const Date& dateStart, const Date& dateEnd, const StationData& sd, std::vector<MeteoData>& vecMeteo); void readTimeSerie(const unsigned int& ts_id, const MeteoData::Parameters& param, const Date& dateStart, const Date& dateEnd, const StationData& sd, std::vector<MeteoData>& vecMeteo);
void initDBOConnection(); void initDBOConnection();
static size_t data_write(void* buf, size_t size, size_t nmemb, void* userp); static size_t data_write(void* buf, size_t size, size_t nmemb, void* userp);
......
...@@ -277,12 +277,12 @@ void OshdIO::parseInputOutputSection() ...@@ -277,12 +277,12 @@ void OshdIO::parseInputOutputSection()
} }
//fill the params mapping vector //fill the params mapping vector
params_map.push_back( std::make_pair(MeteoData::ILWR, "ilwr") ); params_map.push_back( std::make_pair(MeteoData::ILWR, "ilwc") );
params_map.push_back( std::make_pair(MeteoData::P, "pair") ); params_map.push_back( std::make_pair(MeteoData::P, "pair") );
params_map.push_back( std::make_pair(MeteoData::PSUM, "prec") ); //in mm/ts params_map.push_back( std::make_pair(MeteoData::PSUM, "prec") ); //in mm/ts
params_map.push_back( std::make_pair(MeteoData::RH, "rhum") ); params_map.push_back( std::make_pair(MeteoData::RH, "rcor") ); //old: rhum
params_map.push_back( std::make_pair(MeteoData::TA, "tcor") ); //old:tair params_map.push_back( std::make_pair(MeteoData::TA, "tcor") ); //old:tair
params_map.push_back( std::make_pair(MeteoData::VW, "wind") ); //upcoming: wcor params_map.push_back( std::make_pair(MeteoData::VW, "wcor") ); //old: wind
params_map.push_back( std::make_pair(MeteoData::DW, "wdir") ); params_map.push_back( std::make_pair(MeteoData::DW, "wdir") );
} }
...@@ -413,41 +413,47 @@ void OshdIO::readSWRad(const Date& station_date, const std::string& path, const ...@@ -413,41 +413,47 @@ void OshdIO::readSWRad(const Date& station_date, const std::string& path, const
{ {
std::vector<double> vecDir; std::vector<double> vecDir;
vecDir.resize( nrIDs, IOUtils::nodata ); vecDir.resize( nrIDs, IOUtils::nodata );
const std::string filename_dir( path + "/" + "idir" + file_suffix ); const std::string filename_dir( path + "/" + "idrc" + file_suffix );
readFromFile(filename_dir, MeteoData::ISWR, station_date, vecDir); readFromFile(filename_dir, MeteoData::ISWR, station_date, vecDir);
std::vector<double> vecDiff; std::vector<double> vecDiff;
vecDiff.resize( nrIDs, IOUtils::nodata ); vecDiff.resize( nrIDs, IOUtils::nodata );
const std::string filename_diff( path + "/" + "idif" + file_suffix ); const std::string filename_diff( path + "/" + "idfc" + file_suffix );
readFromFile(filename_diff, MeteoData::ISWR, station_date, vecDiff); readFromFile(filename_diff, MeteoData::ISWR, station_date, vecDiff);
std::vector<double> vecAlbd; std::vector<double> vecAlbd;
vecAlbd.resize( nrIDs, IOUtils::nodata );
const std::string filename_albd( path + "/" + "albd" + file_suffix ); const std::string filename_albd( path + "/" + "albd" + file_suffix );
readFromFile(filename_albd, MeteoData::RSWR, station_date, vecAlbd); //We read ALBD and use it to build RSWR if (FileUtils::fileExists(filename_albd)) {
vecAlbd.resize( nrIDs, IOUtils::nodata );
readFromFile(filename_albd, MeteoData::RSWR, station_date, vecAlbd); //We read ALBD and use it to build RSWR
}
const double albedo = !vecAlbd.empty();
for (size_t jj=0; jj<nrIDs; jj++) { for (size_t jj=0; jj<nrIDs; jj++) {
vecMeteo[jj].back()( MeteoData::ISWR ) = vecDir[jj]+vecDiff[jj]; vecMeteo[jj].back()( MeteoData::ISWR ) = vecDir[jj]+vecDiff[jj];
vecMeteo[jj].back()( MeteoData::RSWR ) = (vecDir[jj]+vecDiff[jj])*vecAlbd[jj]; if (albedo) vecMeteo[jj].back()( MeteoData::RSWR ) = (vecDir[jj]+vecDiff[jj])*vecAlbd[jj];
} }
} }
void OshdIO::readPPhase(const Date& station_date, const std::string& path, const std::string& file_suffix, const size_t& nrIDs, std::vector< std::vector<MeteoData> >& vecMeteo) const void OshdIO::readPPhase(const Date& station_date, const std::string& path, const std::string& file_suffix, const size_t& nrIDs, std::vector< std::vector<MeteoData> >& vecMeteo) const
{ {
const double half_elevation_band = 50.; //we consider that there are mixed precip in the elevation range snow_line ± half_elevation_band const std::string filename( path + "/" + "snfl" + file_suffix );
std::vector<double> vecSnowLine;
vecSnowLine.resize( nrIDs, IOUtils::nodata );
const std::string filename_dir( path + "/" + "snfl" + file_suffix );
readFromFile(filename_dir, MeteoData::PSUM_PH, station_date, vecSnowLine);
for (size_t jj=0; jj<nrIDs; jj++) { if (FileUtils::fileExists(filename)) {
const double altitude = vecMeteo[jj].front().meta.getAltitude(); const double half_elevation_band = 50.; //we consider that there are mixed precip in the elevation range snow_line ± half_elevation_band
if (altitude>(vecSnowLine[jj]+half_elevation_band)) std::vector<double> vecSnowLine;
vecMeteo[jj].back()( MeteoData::PSUM_PH ) = 0.; vecSnowLine.resize( nrIDs, IOUtils::nodata );
else if (altitude<(vecSnowLine[jj]-half_elevation_band)) readFromFile(filename, MeteoData::PSUM_PH, station_date, vecSnowLine);
vecMeteo[jj].back()( MeteoData::PSUM_PH ) = 1.;
else for (size_t jj=0; jj<nrIDs; jj++) {
vecMeteo[jj].back()( MeteoData::PSUM_PH ) = .5; const double altitude = vecMeteo[jj].front().meta.getAltitude();
if (altitude>(vecSnowLine[jj]+half_elevation_band))
vecMeteo[jj].back()( MeteoData::PSUM_PH ) = 0.;
else if (altitude<(vecSnowLine[jj]-half_elevation_band))
vecMeteo[jj].back()( MeteoData::PSUM_PH ) = 1.;
else
vecMeteo[jj].back()( MeteoData::PSUM_PH ) = .5;
}
} }
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment