WSL/SLF GitLab Repository

Commit 9a2344ad authored by Mathias Bavay's avatar Mathias Bavay
Browse files

Better reporting for DATA_QA, including the data generators.

parent a255f307
......@@ -83,8 +83,24 @@ void DataGenerator::fillMissing(METEO_SET& vecMeteo) const
const size_t param = vecMeteo[station].getParameterIndex(it->first);
if(param==IOUtils::npos) continue;
#ifdef DATA_QA
double old_val = vecMeteo[station](param);
const string statID = vecMeteo[station].meta.getStationID();
#endif
bool status = false;
size_t jj=0;
while (jj<vecGenerators.size() && vecGenerators[jj]->generate(param, vecMeteo[station]) != true) jj++;
while (jj<vecGenerators.size() && status != true) { //loop over the generators
status = vecGenerators[jj]->generate(param, vecMeteo[station]);
jj++;
#ifdef DATA_QA
if (vecMeteo[station](param) != old_val) {
const string parname = it->first;
const string algo_name = vecGenerators[jj-1]->getAlgo();
cout << "[DATA_QA] Generating " << statID << "::" << parname << "::" << algo_name << " " << vecMeteo[station].date.toString(Date::ISO_TZ) << "\n";
}
#endif
}
}
}
}
......@@ -108,8 +124,26 @@ void DataGenerator::fillMissing(std::vector<METEO_SET>& vecVecMeteo) const
const size_t param = vecVecMeteo[station][0].getParameterIndex(it->first);
if(param==IOUtils::npos) continue;
#ifdef DATA_QA
METEO_SET old_val = vecVecMeteo[station];
const string statID = old_val[0].meta.getStationID();
#endif
bool status = false;
size_t jj=0;
while (jj<vecGenerators.size() && vecGenerators[jj]->generate(param, vecVecMeteo[station]) != true) jj++;
while (jj<vecGenerators.size() && status != true) { //loop over the generators
status = vecGenerators[jj]->generate(param, vecVecMeteo[station]);
jj++;
#ifdef DATA_QA
const string parname = it->first;
const string algo_name = vecGenerators[jj-1]->getAlgo();
for (size_t kk=0; kk<old_val.size(); kk++) {
if (old_val[kk](param) != vecVecMeteo[station][kk](param)) {
cout << "[DATA_QA] Generating " << statID << "::" << parname << "::" << algo_name << " " << old_val[kk].date.toString(Date::ISO_TZ) << "\n";
}
}
#endif
}
}
}
}
......
......@@ -44,7 +44,7 @@ Meteo1DInterpolator::Meteo1DInterpolator(const Config& in_cfg)
Meteo1DInterpolator::~Meteo1DInterpolator()
{
map< string, ResamplingAlgorithms* >::iterator it;
for(it=mapAlgorithms.begin(); it!=mapAlgorithms.end(); ++it)
for (it=mapAlgorithms.begin(); it!=mapAlgorithms.end(); ++it)
delete it->second;
}
......@@ -87,10 +87,10 @@ bool Meteo1DInterpolator::resampleData(const Date& date, const std::vector<Meteo
}
//now, perform the resampling
for(size_t ii=0; ii<md.getNrOfParameters(); ii++) {
for (size_t ii=0; ii<md.getNrOfParameters(); ii++) {
const std::string parname = md.getNameForParameter(ii); //Current parameter name
const map< string, ResamplingAlgorithms* >::const_iterator it = mapAlgorithms.find(parname);
if(it!=mapAlgorithms.end()) {
if (it!=mapAlgorithms.end()) {
it->second->resample(index, elementpos, ii, vecM, md);
} else { //we are dealing with an extra parameter, we need to add it to the map first, so it will exist next time...
vector<string> vecArgs;
......@@ -100,9 +100,11 @@ bool Meteo1DInterpolator::resampleData(const Date& date, const std::vector<Meteo
}
#ifdef DATA_QA
if((index != IOUtils::npos) && vecM[index](ii)!=md(ii)) {
const string algo_name = it->second->getAlgo();
cout << "[DATA_QA] Resampling " << parname << "::" << algo_name << " " << md.date.toString(Date::ISO_TZ) << "\n";
const map< string, ResamplingAlgorithms* >::const_iterator it2 = mapAlgorithms.find(parname); //we have to re-find it in order to handle extra parameters
if ((index != IOUtils::npos) && vecM[index](ii)!=md(ii)) {
const string statID = md.meta.getStationID();
const string algo_name = it2->second->getAlgo();
cout << "[DATA_QA] Resampling " << statID << "::" << parname << "::" << algo_name << " " << md.date.toString(Date::ISO_TZ) << "\n";
}
#endif
}
......@@ -124,14 +126,14 @@ string Meteo1DInterpolator::getInterpolationForParameter(const std::string& parn
cfg.getValue(parname+"::"+algo_name, "Interpolations1D", vecArguments, IOUtils::nothrow);
if(cfg.keyExists(parname+"::"+"args", "Interpolations1D")) //HACK: temporary until we consider everybody has migrated
if (cfg.keyExists(parname+"::"+"args", "Interpolations1D")) //HACK: temporary until we consider everybody has migrated
throw InvalidArgumentException("The syntax for Interpolations1D arguments has been changed. Please check the documentation and update your configuration file \""+cfg.getSourceName()+"\"", AT);
return algo_name;
}
Meteo1DInterpolator& Meteo1DInterpolator::operator=(const Meteo1DInterpolator& source) {
if(this != &source) {
if (this != &source) {
window_size = source.window_size;
mapAlgorithms= source.mapAlgorithms;
}
......@@ -145,7 +147,7 @@ const std::string Meteo1DInterpolator::toString() const
os << "Config& cfg = " << hex << &cfg << dec <<"\n";
os << "Resampling algorithms:\n";
map< string, ResamplingAlgorithms* >::const_iterator it;
for(it=mapAlgorithms.begin(); it!=mapAlgorithms.end(); ++it) {
for (it=mapAlgorithms.begin(); it!=mapAlgorithms.end(); ++it) {
//os << setw(10) << it->first << "::" << it->second->getAlgo() << "\n";
os << it->second->toString() << "\n";
}
......
......@@ -100,7 +100,7 @@ class ResamplingAlgorithms {
virtual ~ResamplingAlgorithms() {};
std::string getAlgo() const {return algo;};
const std::string getAlgo() const {return algo;};
virtual void resample(const size_t& index, const ResamplingPosition& position, const size_t& paramindex,
const std::vector<MeteoData>& vecM, MeteoData& md) = 0;
......
......@@ -89,15 +89,6 @@ void ProcessingStack::process(const std::vector< std::vector<MeteoData> >& ivec,
//since all filters start with ovec=ivec, maybe we could just swap pointers instead for copying
std::vector<MeteoData> tmp( ivec[ii] );
#ifdef DATA_QA
for (size_t kk=0; kk<ivec[ii].size(); kk++) {
const double orig = ivec[ii][kk](param);
if (orig==IOUtils::nodata) {
cout << "[DATA_QA] Filtering " << param_name << " is nodata " << ivec[ii][kk].date.toString(Date::ISO_TZ) << "\n";
}
}
#endif
//Now call the filters one after another for the current station and parameter
bool appliedFilter = false;
for (size_t jj=0; jj<nr_of_filters; jj++){
......@@ -117,8 +108,9 @@ void ProcessingStack::process(const std::vector< std::vector<MeteoData> >& ivec,
const double orig = tmp[kk](param);
const double filtered = ovec[ii][kk](param);
if (orig!=filtered) {
const string statID = ovec[ii][kk].meta.getStationID();
const string filtername = (*filter_stack[jj]).getName();
cout << "[DATA_QA] Filtering " << param_name << "::" << filtername << " " << tmp[kk].date.toString(Date::ISO_TZ) << "\n";
cout << "[DATA_QA] Filtering " << statID << "::" << param_name << "::" << filtername << " " << tmp[kk].date.toString(Date::ISO_TZ) << "\n";
}
}
#endif
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment