WSL/SLF GitLab Repository

Commit a6b5d8f3 authored by Mathias Bavay's avatar Mathias Bavay
Browse files

It is now possible to write timestamps as ISO_WEEK (this is YYYY-Www-dd). This...

It is now possible to write timestamps as ISO_WEEK (this is YYYY-Www-dd). This is now used for the reports generated by DATA_QA in order to ease data aggregation. The GSN plugin has a new key: GSN_TIMEOUT. A bug in error handling was fixed in the GSN plugin.
parent 32a15430
......@@ -85,7 +85,9 @@ void DataGenerator::fillMissing(METEO_SET& vecMeteo) const
#ifdef DATA_QA
double old_val = vecMeteo[station](param);
const string statName = vecMeteo[station].meta.getStationName();
const string statID = vecMeteo[station].meta.getStationID();
const string stat = (!statID.empty())? statID : statName;
#endif
bool status = false;
......@@ -97,7 +99,8 @@ void DataGenerator::fillMissing(METEO_SET& vecMeteo) const
if (vecMeteo[station](param) != old_val) {
const string parname = it->first;
const string algo_name = vecGenerators[jj-1]->getAlgo();
cout << "[DATA_QA] Generating " << statID << "::" << parname << "::" << algo_name << " " << vecMeteo[station].date.toString(Date::ISO_TZ) << "\n";
const Date date( vecMeteo[station].date );
cout << "[DATA_QA] Generating " << stat << "::" << parname << "::" << algo_name << " " << date.toString(Date::ISO_TZ) << " [" << date.toString(Date::ISO_WEEK) << "]\n";
}
#endif
}
......@@ -126,7 +129,9 @@ void DataGenerator::fillMissing(std::vector<METEO_SET>& vecVecMeteo) const
#ifdef DATA_QA
METEO_SET old_val = vecVecMeteo[station];
const string statName = old_val[0].meta.getStationName();
const string statID = old_val[0].meta.getStationID();
const string stat = (!statID.empty())? statID : statName;
#endif
bool status = false;
......@@ -139,7 +144,7 @@ void DataGenerator::fillMissing(std::vector<METEO_SET>& vecVecMeteo) const
const string algo_name = vecGenerators[jj-1]->getAlgo();
for (size_t kk=0; kk<old_val.size(); kk++) {
if (old_val[kk](param) != vecVecMeteo[station][kk](param)) {
cout << "[DATA_QA] Generating " << statID << "::" << parname << "::" << algo_name << " " << old_val[kk].date.toString(Date::ISO_TZ) << "\n";
cout << "[DATA_QA] Generating " << stat << "::" << parname << "::" << algo_name << " " << old_val[kk].date.toString(Date::ISO_TZ) << "\n";
}
}
#endif
......
......@@ -102,9 +102,11 @@ bool Meteo1DInterpolator::resampleData(const Date& date, const std::vector<Meteo
#ifdef DATA_QA
const map< string, ResamplingAlgorithms* >::const_iterator it2 = mapAlgorithms.find(parname); //we have to re-find it in order to handle extra parameters
if ((index != IOUtils::npos) && vecM[index](ii)!=md(ii)) {
const string statName = md.meta.getStationName();
const string statID = md.meta.getStationID();
const string stat = (!statID.empty())? statID : statName;
const string algo_name = it2->second->getAlgo();
cout << "[DATA_QA] Resampling " << statID << "::" << parname << "::" << algo_name << " " << md.date.toString(Date::ISO_TZ) << "\n";
cout << "[DATA_QA] Resampling " << stat << "::" << parname << "::" << algo_name << " " << md.date.toString(Date::ISO_TZ) << " [" << md.date.toString(Date::ISO_WEEK) << "]\n";
}
#endif
}
......
......@@ -1089,6 +1089,12 @@ const string Date::toString(FORMATS type, const bool& gmt) const
<< setw(2) << setfill('0') << minute_out << ":"
<< setw(2) << setfill('0') << second_out;
break;
case(ISO_WEEK):
tmpstr
<< setw(4) << setfill('0') << year_out << "-W"
<< setw(2) << setfill('0') << getISOWeekNr(gmt) << "-"
<< setw(2) << setfill('0') << getDayOfWeek(gmt);
break;
default:
throw InvalidArgumentException("Wrong date conversion format requested", AT);
}
......
......@@ -62,7 +62,8 @@ class Date {
ISO_TZ, ///< ISO 8601 format (same as ISO) but with time zone specification
FULL, ///< ISO 8601 followed by the julian date (in parenthesis)
NUM, ///< ISO 8601 basic format date: YYYYMMDDHHmmSS (fields might be dropped, in the least to the most significant order)
DIN ///<DIN5008 format: DD.MM.YYYY HH:MM
DIN, ///<DIN5008 format: DD.MM.YYYY HH:MM
ISO_WEEK ///< ISO 8601 week date: YYYY-Www-D (for example: 2014-W41-1)
} FORMATS;
///Keywords for selecting rounding strategy
......
......@@ -108,9 +108,11 @@ void ProcessingStack::process(const std::vector< std::vector<MeteoData> >& ivec,
const double orig = tmp[kk](param);
const double filtered = ovec[ii][kk](param);
if (orig!=filtered) {
const string statName = ovec[ii][kk].meta.getStationName();
const string statID = ovec[ii][kk].meta.getStationID();
const string stat = (!statID.empty())? statID : statName;
const string filtername = (*filter_stack[jj]).getName();
cout << "[DATA_QA] Filtering " << statID << "::" << param_name << "::" << filtername << " " << tmp[kk].date.toString(Date::ISO_TZ) << "\n";
cout << "[DATA_QA] Filtering " << stat << "::" << param_name << "::" << filtername << " " << tmp[kk].date.toString(Date::ISO_TZ) << " [" << tmp[kk].date.toString(Date::ISO_WEEK) << "]\n";
}
}
#endif
......
......@@ -81,6 +81,7 @@ namespace mio {
* - GSN_USER: The username to access the service (optional)
* - GSN_PASS: The password to authenticate the USER (optional)
* - STATION#: station code for the given number #, e. g. la_fouly_1034 (case sensitive!)
* - GSN_TIMEOUT: timeout (in seconds) for the connection to the server (default: 60s)
* - GSN_DEBUG: print the full requests/answers from the server when something does not work as expected
*
* If no STATION keys are given, the full list of ALL stations available to the user in GSN will be used!
......@@ -96,14 +97,15 @@ namespace mio {
*
*/
const int GSNIO::http_timeout = 60; // seconds until connect time out for libcurl
const int GSNIO::http_timeout_dflt = 60; // seconds until connect time out for libcurl
const std::string GSNIO::sensors_endpoint = "sensors";
const std::string GSNIO::sensors_format = "format=csv";
const std::string GSNIO::null_string = "null";
GSNIO::GSNIO(const std::string& configfile)
: cfg(configfile), vecStationName(), multiplier(), offset(), coordin(),
coordinparam(), coordout(), coordoutparam(), endpoint(), userid(), passwd(), default_timezone(1.), gsn_debug(false)
coordinparam(), coordout(), coordoutparam(), endpoint(), userid(), passwd(), default_timezone(1.),
http_timeout(http_timeout_dflt), gsn_debug(false)
{
IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
initGSNConnection();
......@@ -113,7 +115,8 @@ GSNIO::GSNIO(const std::string& configfile)
GSNIO::GSNIO(const Config& cfgreader)
: cfg(cfgreader), vecStationName(), multiplier(), offset(), coordin(),
coordinparam(), coordout(), coordoutparam(), endpoint(), userid(), passwd(), default_timezone(1.), gsn_debug(false)
coordinparam(), coordout(), coordoutparam(), endpoint(), userid(), passwd(), default_timezone(1.),
http_timeout(http_timeout_dflt), gsn_debug(false)
{
IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
initGSNConnection();
......@@ -124,6 +127,7 @@ GSNIO::GSNIO(const Config& cfgreader)
void GSNIO::initGSNConnection() {
curl_global_init(CURL_GLOBAL_ALL);
cfg.getValue("GSN_TIMEOUT", "Input", http_timeout, IOUtils::nothrow);
default_timezone = IOUtils::nodata;
cfg.getValue("TIME_ZONE", "Input", default_timezone, IOUtils::nothrow);
......@@ -315,7 +319,7 @@ void GSNIO::readData(const Date& dateStart, const Date& dateEnd, std::vector<Met
} else {
if (gsn_debug)
std::cout << "****\nRequest: " << request << "\n****\n";
throw IOException("Could not retrieve data for station " + vecMeteo[stationindex].meta.stationID, AT);
throw IOException("Could not retrieve data for station " + station_id, AT);
}
}
......
......@@ -86,9 +86,10 @@ class GSNIO : public IOInterface {
std::string coordin, coordinparam, coordout, coordoutparam; //projection parameters
std::string endpoint, userid, passwd; ///< Variables for endpoint configuration
double default_timezone;
int http_timeout; //time out for http connections
bool gsn_debug;
static const int http_timeout; //time out for http connections
static const int http_timeout_dflt;
static const std::string sensors_endpoint, sensors_format, null_string;
};
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment