WSL/SLF GitLab Repository

Commit 664fcd45 authored by Mathias Bavay's avatar Mathias Bavay
Browse files

The SUPPR filter can now take as argument a file containing a list of station...

The SUPPR filter can now take as argument a file containing a list of station IDs and timesteps where the parameter should be deleted
parent b51c3e4f
......@@ -17,6 +17,8 @@
*/
#include <ctime>
#include <cstdlib>
#include <cstring>
#include <errno.h>
#include <meteoio/meteoFilters/FilterSuppr.h>
......@@ -24,8 +26,8 @@ using namespace std;
namespace mio {
FilterSuppr::FilterSuppr(const std::vector<std::string>& vec_args, const std::string& name)
: FilterBlock(name), range(IOUtils::nodata)
FilterSuppr::FilterSuppr(const std::vector<std::string>& vec_args, const std::string& name, const std::string& i_root_path, const double& i_TZ)
: FilterBlock(name), suppr_dates(), root_path(i_root_path), TZ(i_TZ), range(IOUtils::nodata)
{
parse_args(vec_args);
properties.stage = ProcessingProperties::first; //for the rest: default values
......@@ -35,8 +37,20 @@ void FilterSuppr::process(const unsigned int& param, const std::vector<MeteoData
std::vector<MeteoData>& ovec)
{
ovec = ivec;
if (ovec.empty()) return;
if (range==IOUtils::nodata) { //remove all
if (!suppr_dates.empty()) {
const std::string station_ID = ivec[0].meta.stationID;
const std::map< std::string, std::set<Date> >::const_iterator station_it = suppr_dates.find( station_ID );
if (station_it==suppr_dates.end()) return;
for (size_t ii=0; ii<ovec.size(); ii++){
const std::set<Date>::const_iterator it = station_it->second.find( ovec[ii].date );
if (it!=station_it->second.end()) {
ovec[ii](param) = IOUtils::nodata;
}
}
} else if (range==IOUtils::nodata) { //remove all
for (size_t ii=0; ii<ovec.size(); ii++){
ovec[ii](param) = IOUtils::nodata;
}
......@@ -56,6 +70,54 @@ void FilterSuppr::process(const unsigned int& param, const std::vector<MeteoData
}
}
void FilterSuppr::fillSuppr_dates(const std::string& filename)
{
if (!IOUtils::validFileAndPath(filename)) throw InvalidNameException(filename, AT);
if (!IOUtils::fileExists(filename)) throw NotFoundException(filename, AT);
std::ifstream fin(filename.c_str());
if (fin.fail()) {
std::ostringstream ss;
ss << "Filter " << block_name << ": ";
ss << "error opening file \"" << filename << "\", possible reason: " << std::strerror(errno);
throw AccessException(ss.str(), AT);
}
const char eoln = IOUtils::getEoln(fin); //get the end of line character for the file
try {
size_t lcount=0;
do {
lcount++;
std::string line;
getline(fin, line, eoln); //read complete line
IOUtils::stripComments(line);
IOUtils::trim(line);
if (line.empty()) continue;
std::vector<std::string> vecString;
const size_t nrElems = IOUtils::readLineToVec(line, vecString);
if (nrElems!=2) {
std::ostringstream os;
os << "Invalid syntax for filter " << block_name << " in file \"" << filename << "\": expecting 2 arguments, got " << nrElems;
throw InvalidFormatException(os.str(), AT);
}
Date d1;
if (!IOUtils::convertString(d1, vecString[1], TZ))
throw InvalidFormatException("Could not process date "+vecString[1]+" in file \""+filename+"\"", AT);
const std::string station_ID = vecString[0];
suppr_dates[ station_ID ].insert( d1 );
} while (!fin.eof());
fin.close();
} catch (const std::exception&){
if (fin.is_open()) {//close fin if open
fin.close();
}
throw;
}
}
void FilterSuppr::parse_args(std::vector<std::string> vec_args)
{
const size_t nrArgs = vec_args.size();
......@@ -64,10 +126,19 @@ void FilterSuppr::parse_args(std::vector<std::string> vec_args)
throw InvalidArgumentException("Wrong number of arguments for filter " + getName(), AT);
if (nrArgs==1) {
if (!IOUtils::convertString(range, vec_args[0]))
throw InvalidArgumentException("Invalid range \""+vec_args[0]+"\" specified for the "+getName()+" filter.", AT);
if (range<0. || range>1.)
throw InvalidArgumentException("Wrong range for filter " + getName() + ", it should be between 0 and 1", AT);
if (IOUtils::isNumeric(vec_args[0])) {
if (!IOUtils::convertString(range, vec_args[0]))
throw InvalidArgumentException("Invalid range \""+vec_args[0]+"\" specified for the "+getName()+" filter.", AT);
if (range<0. || range>1.)
throw InvalidArgumentException("Wrong range for filter " + getName() + ", it should be between 0 and 1", AT);
} else {
const std::string in_filename( vec_args[0] );
const std::string prefix = ( IOUtils::isAbsolutePath(in_filename) )? "" : root_path+"/";
const std::string path = IOUtils::getPath(prefix+in_filename, true); //clean & resolve path
const std::string filename = path + "/" + IOUtils::getFilename(in_filename);
fillSuppr_dates(filename);
}
}
}
......
......@@ -31,29 +31,45 @@ namespace mio {
* @date 2013-12-06
* @brief Suppression filter.
* Normally, this filter simply reject all values. This is convenient to quickly turn a parameter off
* without modifying the original data.
* But it is also possible to suppress a given fraction of the data at random by providing
* without modifying the original data. It is also possible to provide a list of station ID's and timesteps
* where the parameter should be suppressed.
*
* Finally, it is also possible to suppress a given fraction of the data at random by providing
* such fraction as an argument. For example, <i>0.5</i> would ensure that at least <i>50%</i> of the
* data set contains <i>nodata</i> for this parameter.
* @code
* ILWR::filter1 = suppr
*
* PSUM::filter1 = suppr
* PSUM::arg1 = ./input/meteo/psum_suppr.dat
*
* TA::filter1 = suppr
* TA::arg1 = 0.5
* @endcode
*
* In the second example (PSUM), the file <i>psum_suppr.dat</i> would look like this (the time is given in the timezone declared in Input::TIME_ZONE):
* @code
* *WFJ 2015-10-01T12:00
* *DAV 2015-10-02T15:00
* *WFJ 2015-11-10T06:00
* STB2 2015-10-01T21:30
* @endcode
*/
class FilterSuppr : public FilterBlock {
public:
FilterSuppr(const std::vector<std::string>& vec_args, const std::string& name);
FilterSuppr(const std::vector<std::string>& vec_args, const std::string& name, const std::string& i_root_path, const double& i_TZ);
virtual void process(const unsigned int& param, const std::vector<MeteoData>& ivec,
std::vector<MeteoData>& ovec);
private:
void fillSuppr_dates(const std::string& filename);
void parse_args(std::vector<std::string> vec_args);
double range;
std::map< std::string, std::set<Date> > suppr_dates;
std::string root_path;
double TZ, range;
};
} //end namespace
......
......@@ -85,7 +85,7 @@ void ProcAdd::parse_args(const std::vector<std::string>& vec_args)
throw InvalidArgumentException("Invalid period \""+type_str+"\" specified for the "+getName()+" filter", AT);
//if this is a relative path, prefix the path with the current path
const std::string in_filename = vec_args[1];
const std::string in_filename( vec_args[1] );
const std::string prefix = ( IOUtils::isAbsolutePath(in_filename) )? "" : root_path+"/";
const std::string path = IOUtils::getPath(prefix+in_filename, true); //clean & resolve path
const std::string filename = path + "/" + IOUtils::getFilename(in_filename);
......
......@@ -99,7 +99,7 @@ namespace mio {
* - SNOS: detection of grass growing under the snow height sensor, see FilterSnowNosnow
*
* Some data transformations are also supported besides filtering, both very basic and generic data transformations:
* - SUPPR: delete all data, see FilterSuppr
* - SUPPR: delete all or some data, see FilterSuppr
* - ADD: adds a given offset to the data, see ProcAdd
* - MULT: multiply the data by a given factor, see ProcMult
*
......@@ -120,7 +120,7 @@ namespace mio {
ProcessingBlock* BlockFactory::getBlock(const std::string& blockname, const std::vector<std::string>& vec_args, const Config& cfg)
{
if (blockname == "SUPPR"){
return new FilterSuppr(vec_args, blockname);
return new FilterSuppr(vec_args, blockname, cfg.getConfigRootDir(), cfg.get("TIME_ZONE", "Input"));
} else if (blockname == "MIN"){
return new FilterMin(vec_args, blockname);
} else if (blockname == "MAX"){
......@@ -225,7 +225,7 @@ void ProcessingBlock::readCorrections(const std::string& filter, const std::stri
const size_t maxIndex = corrections.size();
const size_t minIndex = (c_type=='h')? 0 : 1;
char eoln = IOUtils::getEoln(fin); //get the end of line character for the file
const char eoln = IOUtils::getEoln(fin); //get the end of line character for the file
try {
size_t index, lcount=0;
......
......@@ -264,6 +264,7 @@ void PNGIO::setFile(const std::string& filename, png_structp& png_ptr, png_infop
{
// Open file for writing (binary mode)
if (!IOUtils::validFileAndPath(filename)) throw InvalidNameException(filename, AT);
if (!IOUtils::fileExists(filename)) throw NotFoundException(filename, AT);
errno=0;
fp = fopen(filename.c_str(), "wb");
if (fp == NULL) {
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment