WSL/SLF GitLab Repository

Commit f884c607 authored by Mathias Bavay's avatar Mathias Bavay
Browse files

Some docuemntation has been written in the PSUDistribute code to help with...

Some docuemntation has been written in the PSUDistribute code to help with debuging *for next time*. The Accumulate resampling now fills the first nodata with 0 (since we know that we don't have enough information to re-accumulate at the first point). The unused "isGzip" booleans have been removed from the smet code since this is currently not supported the way we want it.
parent 957498bf
......@@ -35,15 +35,15 @@ Meteo1DInterpolator::Meteo1DInterpolator(const Config& in_cfg)
//MeteoData::Parameters parameter (i.e. each member variable like ta, p, psum, ...)
for (size_t ii=0; ii<MeteoData::nrOfParameters; ii++){ //loop over all MeteoData member variables
const std::string parname = MeteoData::getParameterName(ii); //Current parameter name
vector<string> vecArgs;
const string algo_name = getInterpolationForParameter(parname, vecArgs);
std::vector<std::string> vecArgs;
const std::string algo_name = getInterpolationForParameter(parname, vecArgs);
mapAlgorithms[parname] = ResamplingAlgorithmsFactory::getAlgorithm(algo_name, parname, window_size, vecArgs);
}
}
Meteo1DInterpolator::~Meteo1DInterpolator()
{
map< string, ResamplingAlgorithms* >::iterator it;
std::map< std::string, ResamplingAlgorithms* >::iterator it;
for (it=mapAlgorithms.begin(); it!=mapAlgorithms.end(); ++it)
delete it->second;
}
......@@ -89,23 +89,23 @@ bool Meteo1DInterpolator::resampleData(const Date& date, const std::vector<Meteo
//now, perform the resampling
for (size_t ii=0; ii<md.getNrOfParameters(); ii++) {
const std::string parname = md.getNameForParameter(ii); //Current parameter name
const map< string, ResamplingAlgorithms* >::const_iterator it = mapAlgorithms.find(parname);
const std::map< std::string, ResamplingAlgorithms* >::const_iterator it = mapAlgorithms.find(parname);
if (it!=mapAlgorithms.end()) {
it->second->resample(index, elementpos, ii, vecM, md);
} else { //we are dealing with an extra parameter, we need to add it to the map first, so it will exist next time...
vector<string> vecArgs;
const string algo_name = getInterpolationForParameter(parname, vecArgs);
std::vector<std::string> vecArgs;
const std::string algo_name = getInterpolationForParameter(parname, vecArgs);
mapAlgorithms[parname] = ResamplingAlgorithmsFactory::getAlgorithm(algo_name, parname, window_size, vecArgs);;
mapAlgorithms[parname]->resample(index, elementpos, ii, vecM, md);
}
#ifdef DATA_QA
const map< string, ResamplingAlgorithms* >::const_iterator it2 = mapAlgorithms.find(parname); //we have to re-find it in order to handle extra parameters
const std::map< std::string, ResamplingAlgorithms* >::const_iterator it2 = mapAlgorithms.find(parname); //we have to re-find it in order to handle extra parameters
if ((index != IOUtils::npos) && vecM[index](ii)!=md(ii)) {
const string statName = md.meta.getStationName();
const string statID = md.meta.getStationID();
const string stat = (!statID.empty())? statID : statName;
const string algo_name = it2->second->getAlgo();
const std::string statName = md.meta.getStationName();
const std::string statID = md.meta.getStationID();
const std::string stat = (!statID.empty())? statID : statName;
const std::string algo_name = it2->second->getAlgo();
cout << "[DATA_QA] Resampling " << stat << "::" << parname << "::" << algo_name << " " << md.date.toString(Date::ISO_TZ) << " [" << md.date.toString(Date::ISO_WEEK) << "]\n";
}
#endif
......
......@@ -462,7 +462,7 @@ Accumulate::Accumulate(const std::string& i_algoname, const std::string& i_parna
throw InvalidArgumentException(ss.str(), AT);
}
period_read = true;
} else if (vecArgs[ii]=="strict" && !strict) {
} else if (vecArgs[ii]=="strict") {
if (strict) //do not set strict more than once!
throw InvalidArgumentException("Do not provide \"strict\" more than once for \""+i_parname+"::"+i_algoname+"\"", AT);
strict = true;
......@@ -484,8 +484,7 @@ size_t Accumulate::findStartOfPeriod(const std::vector<MeteoData>& vecM, const s
{
size_t start_idx = IOUtils::npos;
for (size_t idx=index; idx--> 0; ) {
const Date curr_date = vecM[idx].date;
if (curr_date <= dateStart) {
if ( vecM[idx].date <= dateStart) {
start_idx = idx;
break;
}
......@@ -540,29 +539,31 @@ void Accumulate::resample(const size_t& index, const ResamplingPosition& positio
{
if (index >= vecM.size())
throw IOException("The index of the element to be resampled is out of bounds", AT);
if (position==ResamplingAlgorithms::begin || position==ResamplingAlgorithms::end)
if (position==ResamplingAlgorithms::begin || index==0) {
if (!strict) md(paramindex) = 0.;
return;
}
md(paramindex) = IOUtils::nodata;
const Date resampling_date = md.date;
const Date dateStart(resampling_date.getJulian() - accumulate_period, resampling_date.getTimeZone());
const size_t start_idx = findStartOfPeriod(vecM, index, dateStart);
if (start_idx==IOUtils::npos) {//No acceptable starting point found
cerr << "[W] Could not accumulate " << vecM.at(0).getNameForParameter(paramindex) << ": ";
cerr << "not enough data for accumulation period at date " << resampling_date.toString(Date::ISO) << "\n";
std::cerr << "[W] Could not accumulate " << vecM.at(0).getNameForParameter(paramindex) << ": ";
std::cerr << "not enough data for accumulation period at date " << resampling_date.toString(Date::ISO) << "\n";
return;
}
if ((index - start_idx) <= 1) {//easy upsampling when start & stop are in the same input time step
//upsampling (for example, generate 15min values from hourly data)
const double sum = easySampling(vecM, paramindex, index, start_idx, dateStart, resampling_date);
md(paramindex) = sum; //if resampling was unsuccesful, sum==IOUtils::nodata
const double sum = easySampling(vecM, paramindex, index, start_idx, dateStart, resampling_date); //if resampling was unsuccesful, sum==IOUtils::nodata
md(paramindex) = sum;
} else {
//downsampling (for example, generate daily values from hourly data)
//and upsampling when resampled period falls accross a measurement timestamp
const double sum = complexSampling(vecM, paramindex, index, start_idx, dateStart, resampling_date);
md(paramindex) = sum; //if resampling was unsuccesful, sum==IOUtils::nodata
const double sum = complexSampling(vecM, paramindex, index, start_idx, dateStart, resampling_date); //if resampling was unsuccesful, sum==IOUtils::nodata
md(paramindex) = sum;
}
}
......
......@@ -250,7 +250,7 @@ class Solar : public ResamplingAlgorithms {
std::string toString() const;
private:
typedef struct POINTS {
POINTS(): jul1(0.), loss1(IOUtils::nodata), jul2(0), loss2(IOUtils::nodata) {};
POINTS(): jul1(0.), loss1(IOUtils::nodata), jul2(0), loss2(IOUtils::nodata) {}
double jul1, loss1, jul2, loss2;
} Points;
......
......@@ -31,6 +31,22 @@ ProcPSUMDistribute::ProcPSUMDistribute(const std::vector<std::string>& vec_args,
parse_args(vec_args);
}
/**
* @brief Distributes a precipitation sum over the most probable preceeding time steps.
* This works by looking at various criterias (like relative humidity, TA-TSS, etc) and picking up
* the time steps that show the highest scores.
* About this index usage in the code below: When looking into the ovec vector, they are:
* \verbatim
___▒_____________▒_____________▒____
↑ ↑
ii endIdx
\endverbatim
* The goal is that the only "hard" references we can have are the accumulated value (ie the only values != nodata)
* and therefore their index are always kept (as ii and endIdx). This leads to the many "ii+1" in the code below since
* the period over which the data has to be distributed starts at ii+1 and goes until endIdx (which will become the new ii for the
* next iteration).
*
*/
void ProcPSUMDistribute::process(const unsigned int& param, const std::vector<MeteoData>& ivec,
std::vector<MeteoData>& ovec)
{
......
......@@ -80,7 +80,7 @@ SMETIO::SMETIO(const std::string& configfile)
: cfg(configfile),
coordin(), coordinparam(), coordout(), coordoutparam(),
vec_smet_reader(), vecFiles(), outpath(), in_dflt_TZ(0.), out_dflt_TZ(0.),
plugin_nodata(IOUtils::nodata), nr_stations(0), outputIsAscii(true), outputIsGzipped(false)
plugin_nodata(IOUtils::nodata), nr_stations(0), outputIsAscii(true)
{
parseInputOutputSection();
}
......@@ -89,7 +89,7 @@ SMETIO::SMETIO(const Config& cfgreader)
: cfg(cfgreader),
coordin(), coordinparam(), coordout(), coordoutparam(),
vec_smet_reader(), vecFiles(), outpath(), in_dflt_TZ(0.), out_dflt_TZ(0.),
plugin_nodata(IOUtils::nodata), nr_stations(0), outputIsAscii(true), outputIsGzipped(false)
plugin_nodata(IOUtils::nodata), nr_stations(0), outputIsAscii(true)
{
parseInputOutputSection();
}
......@@ -141,7 +141,6 @@ void SMETIO::parseInputOutputSection()
//Parse output section: extract info on whether to write ASCII or BINARY format, gzipped or not
outpath.clear();
outputIsAscii = true;
outputIsGzipped = false;
vector<string> vecArgs;
cfg.getValue("METEOPATH", "Output", outpath, IOUtils::nothrow);
......@@ -153,7 +152,7 @@ void SMETIO::parseInputOutputSection()
if (vecArgs.empty())
vecArgs.push_back("ASCII");
if (vecArgs.size() > 2)
if (vecArgs.size() > 1)
throw InvalidFormatException("Too many values for key METEOPARAM", AT);
if (vecArgs[0] == "BINARY")
......@@ -162,14 +161,6 @@ void SMETIO::parseInputOutputSection()
outputIsAscii = true;
else
throw InvalidFormatException("The first value for key METEOPARAM may only be ASCII or BINARY", AT);
if (vecArgs.size() == 2){
if (vecArgs[1] != "GZIP")
throw InvalidFormatException("The second value for key METEOPARAM may only be GZIP", AT);
outputIsGzipped = true;
}
}
void SMETIO::identify_fields(const std::vector<std::string>& fields, std::vector<size_t>& indexes,
......@@ -432,7 +423,7 @@ void SMETIO::writeMeteoData(const std::vector< std::vector<MeteoData> >& vecMete
try {
const smet::SMETType type = (outputIsAscii)? smet::ASCII : smet::BINARY;
smet::SMETWriter mywriter(filename, type, outputIsGzipped);
smet::SMETWriter mywriter(filename, type);
generateHeaderInfo(sd, outputIsAscii, isConsistent, timezone,
nr_of_parameters, vecParamInUse, vecColumnName, mywriter);
......
......@@ -85,7 +85,7 @@ class SMETIO : public IOInterface {
double in_dflt_TZ, out_dflt_TZ; //default time zones
double plugin_nodata;
size_t nr_stations; //number of stations to read from
bool outputIsAscii, outputIsGzipped;//read from the Config [Output] section
bool outputIsAscii;//read from the Config [Output] section
};
} //namespace
......
......@@ -263,10 +263,10 @@ size_t SMETCommon::readLineToVec(const std::string& line_in, std::vector<std::st
return vec_string.size();
}
SMETWriter::SMETWriter(const std::string& in_filename, const SMETType& in_type, const bool& in_gzip)
SMETWriter::SMETWriter(const std::string& in_filename, const SMETType& in_type)
: other_header_keys(), ascii_precision(), ascii_width(), header(), mandatory_header_keys(),
filename(in_filename), nodata_string(), smet_type(in_type), nodata_value(-999.), nr_of_fields(0),
julian_field(0), timestamp_field(0), location_wgs84(0), location_epsg(0), gzip(in_gzip),
julian_field(0), timestamp_field(0), location_wgs84(0), location_epsg(0),
location_in_header(false), location_in_data_wgs84(false), location_in_data_epsg(false),
timestamp_present(false), julian_present(false), file_is_binary(false) {}
......
......@@ -260,9 +260,8 @@ class SMETWriter {
* @brief The constructor allows to set the filename, the type and whether the file should be gzipped
* @param[in] in_fname The filename of the SMET file to be written
* @param[in] in_type The type of the SMET file, i.e. smet::ASCII or smet::BINARY (default: ASCII)
* @param[in] in_gzip Whether the file should be zipped or not (default: false)
*/
SMETWriter(const std::string& in_fname, const SMETType& in_type=ASCII, const bool& in_gzip=false);
SMETWriter(const std::string& in_fname, const SMETType& in_type=ASCII);
/**
* @brief Set a key, value pair in the SMET header (both strings)
......@@ -331,7 +330,6 @@ class SMETWriter {
double nodata_value;
size_t nr_of_fields, julian_field, timestamp_field;
char location_wgs84, location_epsg;
bool gzip;
bool location_in_header, location_in_data_wgs84, location_in_data_epsg;
bool timestamp_present, julian_present;
bool file_is_binary;
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment