WSL/SLF GitLab Repository

Commit 82b555ad authored by Mathias Bavay's avatar Mathias Bavay
Browse files

The METEO_TIMSERIES and STATION_TIMESERIES have been renamed in METEO_SET and...

The METEO_TIMSERIES and STATION_TIMESERIES have been renamed in METEO_SET and STATIONS_SET in order to better reflect their content. 

The concept of data generators is introduced. A new section has been created [Generators] to define how to generate data for each parameter when everything else failed. Soon, you will be able to generate ILWR directly by declaring it in your io.ini!
parent b8e00441
......@@ -163,16 +163,16 @@ void BufferedIOHandler::readAssimilationData(const Date& in_date, Grid2DObject&
}
}
void BufferedIOHandler::readStationData(const Date& date, STATION_TIMESERIE& vecStation)
void BufferedIOHandler::readStationData(const Date& date, STATIONS_SET& vecStation)
{
iohandler.readStationData(date, vecStation);
}
#ifdef _POPC_
void BufferedIOHandler::writeMeteoData(std::vector< METEO_TIMESERIE >& vecMeteo,
void BufferedIOHandler::writeMeteoData(std::vector< METEO_SET >& vecMeteo,
const std::string& name)
#else
void BufferedIOHandler::writeMeteoData(const std::vector< METEO_TIMESERIE >& vecMeteo,
void BufferedIOHandler::writeMeteoData(const std::vector< METEO_SET >& vecMeteo,
const std::string& name)
#endif
{
......@@ -253,7 +253,7 @@ double BufferedIOHandler::getAvgSamplingRate()
return IOUtils::nodata;
}
const std::vector< METEO_TIMESERIE >& BufferedIOHandler::get_complete_buffer(Date& start, Date& end)
const std::vector< METEO_SET >& BufferedIOHandler::get_complete_buffer(Date& start, Date& end)
{
start = buffer_start;
end = buffer_end;
......@@ -262,7 +262,7 @@ const std::vector< METEO_TIMESERIE >& BufferedIOHandler::get_complete_buffer(Dat
}
void BufferedIOHandler::readMeteoData(const Date& date_start, const Date& date_end,
std::vector< METEO_TIMESERIE >& vecMeteo,
std::vector< METEO_SET >& vecMeteo,
const size_t& /*stationindex*/)
{
vecMeteo.clear();
......@@ -339,7 +339,7 @@ void BufferedIOHandler::readMeteoData(const Date& date_start, const Date& date_e
}
}
void BufferedIOHandler::bufferData(const Date& date_start, const Date& date_end, std::vector< METEO_TIMESERIE >& vecvecMeteo){
void BufferedIOHandler::bufferData(const Date& date_start, const Date& date_end, std::vector< METEO_SET >& vecvecMeteo){
vecvecMeteo.clear(); //the plugins do it internally anyway, but this is cheap and safe...
iohandler.readMeteoData(date_start, date_end, vecvecMeteo);
buffer_start = date_start;
......@@ -355,7 +355,7 @@ void BufferedIOHandler::bufferData(const Date& date_start, const Date& date_end,
* @param vecMeteo The actual data being pushed into vec_buffer_meteo
*/
void BufferedIOHandler::push_meteo_data(const Date& date_start, const Date& date_end,
const std::vector< METEO_TIMESERIE >& vecMeteo)
const std::vector< METEO_SET >& vecMeteo)
{
//perform check on date_start and date_end
if (date_end < date_start)
......
......@@ -89,7 +89,7 @@ class BufferedIOHandler : public IOInterface {
* @param date date for which to read the metadata
* @param vecStation vector of metadata
*/
virtual void readStationData(const Date& date, STATION_TIMESERIE& vecStation);
virtual void readStationData(const Date& date, STATIONS_SET& vecStation);
/**
* @brief Clear all buffers in BufferedIOHandler and hence force rebuffering
......@@ -103,13 +103,13 @@ class BufferedIOHandler : public IOInterface {
virtual void readLanduse(Grid2DObject& landuse_out);
virtual void readSpecialPoints(std::vector<Coords>& pts);
virtual void readMeteoData(const Date& dateStart, const Date& dateEnd,
std::vector< METEO_TIMESERIE >& vecMeteo,
std::vector< METEO_SET >& vecMeteo,
const size_t& stationindex=IOUtils::npos);
#ifdef _POPC_
virtual void writeMeteoData(std::vector< METEO_TIMESERIE >& vecMeteo,
virtual void writeMeteoData(std::vector< METEO_SET >& vecMeteo,
const std::string& name="");
#else
virtual void writeMeteoData(const std::vector< METEO_TIMESERIE >& vecMeteo,
virtual void writeMeteoData(const std::vector< METEO_SET >& vecMeteo,
const std::string& name="");
#endif
virtual void write2DGrid(const Grid2DObject& grid_in, const std::string& options="");
......@@ -142,20 +142,20 @@ class BufferedIOHandler : public IOInterface {
private:
//private methods
const std::vector<METEO_TIMESERIE>& get_complete_buffer(Date& start, Date& end);
const std::vector<METEO_SET>& get_complete_buffer(Date& start, Date& end);
void push_meteo_data(const Date& date_start, const Date& date_end,
const std::vector< METEO_TIMESERIE >& vecMeteo);
const std::vector< METEO_SET >& vecMeteo);
void setDfltBufferProperties();
void bufferData(const Date& date_start, const Date& date_end, std::vector< METEO_TIMESERIE >& vecvecMeteo);
void bufferData(const Date& date_start, const Date& date_end, std::vector< METEO_SET >& vecvecMeteo);
void bufferGrid(const Grid2DObject& in_grid2Dobj, const std::string& in_filename);
//private members
IOHandler& iohandler;
const Config& cfg;
std::vector< METEO_TIMESERIE > vec_buffer_meteo;
std::vector< METEO_SET > vec_buffer_meteo;
std::map<std::string, Grid2DObject> mapBufferedGrids;
std::vector<std::string> IndexBufferedGrids;
......
......@@ -48,6 +48,8 @@ SET(meteoio_sources
ResamplingAlgorithms2D.cc
Meteo1DInterpolator.cc
InterpolationAlgorithms.cc
DataGenerator.cc
GeneratorAlgorithms.cc
Date.cc
Timer.cc
Grid2DObject.cc
......
/***********************************************************************************/
/* Copyright 2009 WSL Institute for Snow and Avalanche Research SLF-DAVOS */
/***********************************************************************************/
/* This file is part of MeteoIO.
MeteoIO is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
MeteoIO is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with MeteoIO. If not, see <http://www.gnu.org/licenses/>.
*/
#include <meteoio/DataGenerator.h>
using namespace std;
namespace mio {
/**
*
*/ //explain how to use the generators for the end user
DataGenerator::DataGenerator(const Config& i_cfg)
: cfg(i_cfg), mapAlgorithms(), generators_defined(false)
{
setAlgorithms();
}
DataGenerator& DataGenerator::operator=(const DataGenerator& source)
{
if(this != &source) {
mapAlgorithms = source.mapAlgorithms;
generators_defined = source.generators_defined;
}
return *this;
}
/**
* @brief generate data to fill missing data points.
* This relies on data generators defined by the user for each meteo parameters.
* This loops over the defined generators and stops as soon as all missing points
* have been successfully replaced.
* @param vecMeteo vector containing one point for each station
*/
void DataGenerator::fillMissing(METEO_SET& vecMeteo) const
{
if(!generators_defined) return; //no generators defined by the end user
for(size_t param=0; param < MeteoData::nrOfParameters; param++) { //loop over all possible meteo parameters
const std::map< size_t, std::vector<GeneratorAlgorithm*> >::const_iterator it = mapAlgorithms.find(param);
if (it != mapAlgorithms.end()){ //there is a data generator for this parameter
const std::vector<GeneratorAlgorithm*> vecGenerators = it->second;
for(size_t station=0; station<vecMeteo.size(); station++) { //process this parameter on all stations
size_t jj=0;
while (jj<vecGenerators.size() && vecGenerators[jj]->generate(param, vecMeteo[station]) != true) jj++;
}
}
}
}
/**
* @brief generate data to fill missing data points.
* This relies on data generators defined by the user for each meteo parameters.
* This loops over the defined generators and stops as soon as all missing points
* have been successfully replaced.
* @param vecVecMeteo vector containing a timeserie for each station
*/
void DataGenerator::fillMissing(std::vector<METEO_SET>& vecVecMeteo) const
{
if(!generators_defined) return; //no generators defined by the end user
for(size_t param=0; param < MeteoData::nrOfParameters; param++) { //loop over all possible meteo parameters
const std::map< size_t, std::vector<GeneratorAlgorithm*> >::const_iterator it = mapAlgorithms.find(param);
if (it != mapAlgorithms.end()){ //there is a data generator for this parameter
const std::vector<GeneratorAlgorithm*> vecGenerators = it->second;
for(size_t station=0; station<vecVecMeteo.size(); station++) { //process this parameter on all stations
size_t jj=0;
while (jj<vecGenerators.size() && vecGenerators[jj]->generate(param, vecVecMeteo[station]) != true) jj++;
}
}
}
}
/** @brief build the generators for each meteo parameter
* By reading the Config object build up a list of user configured algorithms
* for each MeteoData::Parameters parameter (i.e. each member variable of MeteoData like ta, p, hnw, ...)
* Concept of this constructor: loop over all MeteoData::Parameters and then look
* for configuration of interpolation algorithms within the Config object.
*/
void DataGenerator::setAlgorithms()
{
for (size_t ii=0; ii < MeteoData::nrOfParameters; ii++){ //loop over all MeteoData member variables
std::vector<std::string> tmpAlgorithms;
const std::string& parname = MeteoData::getParameterName(ii); //Current parameter name
const size_t nrOfAlgorithms = getAlgorithmsForParameter(parname, tmpAlgorithms);
std::vector<GeneratorAlgorithm*> vecGenerators(nrOfAlgorithms);
for(size_t jj=0; jj<nrOfAlgorithms; jj++) {
std::vector<std::string> vecArgs;
getArgumentsForAlgorithm(parname, tmpAlgorithms[jj], vecArgs);
vecGenerators[jj] = GeneratorAlgorithmFactory::getAlgorithm( tmpAlgorithms[jj], vecArgs);
}
if(nrOfAlgorithms>0) {
mapAlgorithms[ii] = vecGenerators;
generators_defined = true;
}
}
}
size_t DataGenerator::getAlgorithmsForParameter(const std::string& parname, std::vector<std::string>& vecAlgorithms)
{
// This function retrieves the user defined generator algorithms for
// parameter 'parname' by querying the Config object
vecAlgorithms.clear();
std::vector<std::string> vecKeys;
cfg.findKeys(vecKeys, parname+"::generators", "Generators");
if (vecKeys.size() > 1)
throw IOException("Multiple definitions of " + parname + "::generators in config file", AT);;
if (vecKeys.empty())
return 0;
cfg.getValue(vecKeys.at(0), "Generators", vecAlgorithms, IOUtils::nothrow);
return vecAlgorithms.size();
}
size_t DataGenerator::getArgumentsForAlgorithm(const std::string& parname,
const std::string& algorithm,
std::vector<std::string>& vecArgs) const
{
vecArgs.clear();
cfg.getValue(parname+"::"+algorithm, "Generators", vecArgs, IOUtils::nothrow);
return vecArgs.size();
}
std::ostream& operator<<(std::ostream &os, const DataGenerator &mi) {
os << "<DataGenerator>\n";
os << "Config& cfg = " << hex << &mi.cfg << dec << "\n";
os << "User list of generators:\n";
std::map< size_t, std::vector<GeneratorAlgorithm*> >::const_iterator iter = mi.mapAlgorithms.begin();
for (; iter != mi.mapAlgorithms.end(); ++iter) {
os << setw(10) << MeteoData::getParameterName(iter->first) << " :: ";
for(unsigned int jj=0; jj<iter->second.size(); jj++) {
os << iter->second[jj]->getAlgo() << " ";
}
os << "\n";
}
os << "</DataGenerator>\n";
return os;
}
#ifdef _POPC_
#include "marshal_meteoio.h"
using namespace mio; //HACK for POPC
void DataGenerator::Serialize(POPBuffer &buf, bool pack)
{
/*if (pack)
{
}
else
{
}*/
}
#endif
} //namespace
/***********************************************************************************/
/* Copyright 2013 WSL Institute for Snow and Avalanche Research SLF-DAVOS */
/***********************************************************************************/
/* This file is part of MeteoIO.
MeteoIO is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
MeteoIO is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with MeteoIO. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __DATAGENERATOR_H__
#define __DATAGENERATOR_H__
#include <meteoio/Config.h>
#include <meteoio/MeteoData.h>
#include <meteoio/GeneratorAlgorithms.h>
#include <vector>
#include <map>
namespace mio {
/**
*
*/ //explain how to write a new generator algorithm here
/**
* @class DataGenerator
* @brief A class to generate meteo data from user-selected models or parametrizations.
* This class sits in between the actual implementation of the various methods and the IOManager in
* order to offer some high level interface. It basically reads the arguments and creates the objects for
* the various data generators in its constructor and loop through the parameters and stations when called to fill the data.
*
* @ingroup meteolaws
* @author Mathias Bavay
* @date 2013-03-20
*/
#ifdef _POPC_
#include <paroc_base.h>
class DataGenerator : POPBase {
public:
void Serialize(POPBuffer &buf, bool pack);
#else
class DataGenerator {
#endif
public:
DataGenerator(const Config& i_cfg);
DataGenerator(const DataGenerator& c) : cfg(c.cfg), mapAlgorithms(c.mapAlgorithms), generators_defined(c.generators_defined) {};
void fillMissing(METEO_SET& vecMeteo) const;
void fillMissing(std::vector<METEO_SET>& vecVecMeteo) const;
DataGenerator& operator=(const DataGenerator& source);
friend std::ostream& operator<<(std::ostream& os, const DataGenerator& mi);
private:
void setAlgorithms();
size_t getAlgorithmsForParameter(const std::string& parname, std::vector<std::string>& vecAlgorithms);
size_t getArgumentsForAlgorithm(const std::string& parname,
const std::string& algorithm,
std::vector<std::string>& vecArgs) const;
const Config& cfg; ///< Reference to Config object, initialized during construction
std::map< size_t, std::vector<GeneratorAlgorithm*> > mapAlgorithms; //per parameter interpolation algorithms
bool generators_defined; //if true, there are some generators to run. if false, nothing to do
};
} //end namespace
#endif
/***********************************************************************************/
/* Copyright 2013 WSL Institute for Snow and Avalanche Research SLF-DAVOS */
/***********************************************************************************/
/* This file is part of MeteoIO.
MeteoIO is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
MeteoIO is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with MeteoIO. If not, see <http://www.gnu.org/licenses/>.
*/
#include <meteoio/GeneratorAlgorithms.h>
#include <meteoio/MathOptim.h>
#include <meteoio/meteolaws/Atmosphere.h>
using namespace std;
namespace mio {
GeneratorAlgorithm* GeneratorAlgorithmFactory::getAlgorithm(const std::string& i_algoname, const std::vector<std::string>& vecArgs)
{
std::string algoname(i_algoname);
IOUtils::toUpper(algoname);
if (algoname == "CST"){
return new ConstGenerator(vecArgs, i_algoname);
} else if (algoname == "STD_PRESS"){
return new StandardPressureGenerator(vecArgs, i_algoname);
} else {
throw IOException("The generator algorithm '"+algoname+"' is not implemented" , AT);
}
}
std::string GeneratorAlgorithm::getAlgo() const {
return algo;
}
////////////////////////////////////////////////////////////////////////
void ConstGenerator::parse_args(const std::vector<std::string>& vecArgs)
{
//Get the optional arguments for the algorithm: constant value to use
if(vecArgs.size()==1) {
IOUtils::convertString(constant, vecArgs[0]);
} else { //incorrect arguments, throw an exception
throw InvalidArgumentException("Wrong number of arguments supplied for the "+algo+" generator", AT);
}
}
bool ConstGenerator::generate(const size_t& param, MeteoData& md) const
{
double &value = md(param);
if(value == IOUtils::nodata)
value = constant;
return true; //all missing values could be filled
}
bool ConstGenerator::generate(const size_t& param, std::vector<MeteoData>& vecMeteo) const
{
if(vecMeteo.empty()) return true;
for(size_t ii=0; ii<vecMeteo.size(); ii++) {
generate(param, vecMeteo[ii]);
}
return true; //all missing values could be filled
}
void StandardPressureGenerator::parse_args(const std::vector<std::string>& vecArgs)
{
//Get the optional arguments for the algorithm: constant value to use
if(!vecArgs.empty()) { //incorrect arguments, throw an exception
throw InvalidArgumentException("Wrong number of arguments supplied for the "+algo+" generator", AT);
}
}
bool StandardPressureGenerator::generate(const size_t& param, MeteoData& md) const
{
if(param!=MeteoData::P) {
stringstream ss;
ss << "Can not use " << algo << " generator on " << MeteoData::getParameterName(param);
throw InvalidArgumentException(ss.str(), AT);
}
double &value = md(param);
if(value == IOUtils::nodata) {
const double altitude = md.meta.position.getAltitude();
if(altitude==IOUtils::nodata) return false;
value = Atmosphere::stdAirPressure(altitude);
}
return true; //all missing values could be filled
}
bool StandardPressureGenerator::generate(const size_t& param, std::vector<MeteoData>& vecMeteo) const
{
if(param!=MeteoData::P) {
stringstream ss;
ss << "Can not use " << algo << " generator on " << MeteoData::getParameterName(param);
throw InvalidArgumentException(ss.str(), AT);
}
if(vecMeteo.empty()) return true;
const double altitude = vecMeteo[0].meta.position.getAltitude(); //if the stations move, this has to be in the loop
if(altitude==IOUtils::nodata) return false;
for(size_t ii=0; ii<vecMeteo.size(); ii++) {
double &value = vecMeteo[ii](param);
if(value == IOUtils::nodata)
value = Atmosphere::stdAirPressure(altitude);
}
return true; //all missing values could be filled
}
} //namespace
/***********************************************************************************/
/* Copyright 2013 WSL Institute for Snow and Avalanche Research SLF-DAVOS */
/***********************************************************************************/
/* This file is part of MeteoIO.
MeteoIO is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
MeteoIO is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with MeteoIO. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __GENERATORALGORITHMS_H__
#define __GENERATORALGORITHMS_H__
#include <meteoio/MeteoData.h>
#include <vector>
#include <set>
#include <string>
namespace mio {
/**
* @page generators Data generators
* Once the data has been read, filtered and resampled, it can be that some data points are still missing.
* These are either a few isolated periods (a sensor was not functioning) that are too large for performing
* a statistical temporal interpolation or that a meteorological parameter was not even measured. In such a case,
* we generate data, generally relying on some parametrization using other meteorological parameters. In a few
* cases, even fully arbitrary data might be helpful (replacing missing value by a given constant so a model can
* run over the data gap).
*
* @section generators_section Data generators section
* The data generators are defined per meteorological parameter. They are applied to all stations
* (if using multiple meteorological stations). If multiple dat generators are specified for each parameter,
* they would be used in the order of declaration, meaning that only the data points that could not be
* generated by the first generator would be tentatively generated by the second generator, etc
* @code
* [Generators]
* RH::generators = CST
* RH::Cst = .7
*
* P::generators = STD_PRESS
* @endcode
*
* @section keywords Available generators
* The keywords defining the algorithms are the following:
* - STD_PRESS: standard atmospheric pressure as a function of the elevation of each station (see StandardPressureGenerator)
* - CST: constant value as provided in argument (see ConstGenerator)
*
* @author Mathias Bavay
* @date 2013-03-20
*/
/**
* @class GeneratorAlgorithm
* @brief Interface class for the generator models.
* These models generate data for a specific parameter when all other options failed (the resampling could not help).
* Therefore, there is nothing more that could be done with the temporal history of the data, we have to use
* a totally different approach: either generic data (constant value, etc) or generate the data from other
* meteorological parameters (relying on a parametrization, like clear sky for ILWR).
*
* @ingroup meteolaws
* @author Mathias Bavay
* @date 2013-03-20
*/
class GeneratorAlgorithm {
public:
GeneratorAlgorithm(const std::vector<std::string>& /*vecArgs*/, const std::string& i_algo) : algo(i_algo) {};
virtual ~GeneratorAlgorithm() {}
//fill one MeteoData, for one station
virtual bool generate(const size_t& param, MeteoData& md) const = 0;
//fill one time series of MeteoData for one station
virtual bool generate(const size_t& param, std::vector<MeteoData>& vecMeteo) const = 0;
std::string getAlgo() const;
protected:
virtual void parse_args(const std::vector<std::string>& /*i_vecArgs*/) = 0;
const std::string algo;
};
class GeneratorAlgorithmFactory {
public:
static GeneratorAlgorithm* getAlgorithm(const std::string& i_algoname, const std::vector<std::string>& vecArgs);
};
/**
* @class ConstGenerator
* @brief Constant value generator.
* Generate a constant value for this parameter, as provided in argument.
* @code
* RH::generators = Cst