WSL/SLF GitLab Repository

Skip to content
GitLab
Commit 31424f54 authored by Thomas Egger's avatar Thomas Egger
Browse files

Removed legacy code from BufferedIOHandler and MeteoProcessor, added a few comments to IOManager

parent 67c032b1
Hide whitespace changes
Inline Side-by-side
meteoio/BufferedIOHandler.cc
View file @ 31424f54
......@@ -23,10 +23,10 @@ namespace mio {
#ifdef _POPC_
BufferedIOHandler::BufferedIOHandler(IOHandler& _iohandler, const Config& _cfg)
: iohandler(_iohandler), cfg(_cfg), meteoprocessor(_cfg), meteoBuffer(), startDateBuffer(), endDateBuffer(), mapBufferedGrids()
: iohandler(_iohandler), cfg(_cfg), mapBufferedGrids()
#else
BufferedIOHandler::BufferedIOHandler(IOHandler& _iohandler, const Config& _cfg)
: IOInterface(NULL), iohandler(_iohandler), cfg(_cfg), meteoprocessor(_cfg), meteoBuffer(), startDateBuffer(), endDateBuffer(), mapBufferedGrids()
: IOInterface(NULL), iohandler(_iohandler), cfg(_cfg), mapBufferedGrids()
#endif
{
setDfltBufferProperties();
......@@ -126,134 +126,6 @@ void BufferedIOHandler::writeMeteoData(const std::vector< std::vector<MeteoData>
void BufferedIOHandler::setDfltBufferProperties()
{
always_rebuffer = true;
bufferbefore = Date(2.0); //minus 2 days
bufferafter = Date(20.0); //plus 20 days
}
void BufferedIOHandler::readMeteoData(const Date& i_date, std::vector<MeteoData>& vecMeteo){
/* For every station:
* 1) See whether data is already buffered
* 2) Filter - includes resampling
* 3) Return the values
*/
vecMeteo.clear();
if (meteoBuffer.size() == 0){ //init
bufferAllData(i_date);
}
//loop through all meteo buffers, there is one for each station
for (unsigned int ii=0; ii<meteoBuffer.size(); ii++) {
unsigned int index = IOUtils::npos;
if (meteoBuffer[ii].size() > 0) {//check whether meteo data for the date exists in buffer
index = IOUtils::seek(i_date, meteoBuffer[ii], false);
}
if (index == IOUtils::npos) { //not in buffer
//Check buffering strategy
bool rebuffer = false;
if ((startDateBuffer.at(ii) > i_date) || (endDateBuffer.at(ii) <= i_date)){
rebuffer = true;
} else {
if (always_rebuffer) rebuffer = true;
}
if (rebuffer){
//cout << "[I] Station " << ii << "(" << stationID
// << ") data for date " << i_date.toString(Date::FULL) << " not in buffer ..." << endl;
bool dataexists = bufferData(i_date, ii);
if (dataexists) {//i_date is contained in buffer
index = IOUtils::seek(i_date, meteoBuffer[ii], false);
}
}
}
//APPLY FILTERS
MeteoData md;
if (index != IOUtils::npos)
meteoprocessor.processData(i_date, meteoBuffer[ii], md);
//Check whether StationData is meaningful, try to get meaningful meta data
if (index != IOUtils::npos) {
vecMeteo.push_back(md);
} else {
cout << "[I] No data found for station " << ii+1 << " at date " << i_date.toString(Date::FULL)
<< endl;
vecMeteo.push_back(MeteoData());
vecMeteo[ii].date = i_date; //set correct date
}
}
if (vecMeteo.size() == 0) {//No data found - return one object set to i_date and nodata in all other fields
vecMeteo.push_back(MeteoData());
vecMeteo[0].date = i_date; //set correct date
}
}
void BufferedIOHandler::getNextMeteoData(const Date& _date, std::vector<MeteoData>& vecMeteo)
{
//TODO: check whether there is something in the buffer!
//Try to rebuffer!
vecMeteo.clear();
std::vector< std::vector<MeteoData> > meteoTmpBuffer;
legacy_readMeteoData(_date, (_date-Date(1900,1,2,0,0)), meteoTmpBuffer);
unsigned int emptycounter = 0;
for (unsigned int ii=0; ii<meteoTmpBuffer.size(); ii++){//stations
if (meteoTmpBuffer[ii].size() > 0){
vecMeteo.push_back(meteoTmpBuffer[ii][0]);
} else {
emptycounter++;
}
}
if (emptycounter == meteoTmpBuffer.size())
vecMeteo.clear();
}
void BufferedIOHandler::bufferAllData(const Date& _date){
Date fromDate = _date - bufferbefore;
Date toDate = _date + bufferafter;
legacy_readMeteoData(fromDate, toDate, meteoBuffer);
for (unsigned int ii=0; ii<meteoBuffer.size(); ii++){
//set the start and the end date of the interval requested for each station
startDateBuffer.push_back(fromDate);
endDateBuffer.push_back(toDate);
}
}
bool BufferedIOHandler::bufferData(const Date& _date, const unsigned int& stationindex)
{
Date fromDate = _date - bufferbefore;
Date toDate = _date + bufferafter;
legacy_readMeteoData(fromDate, toDate, meteoBuffer, stationindex);
startDateBuffer.at(stationindex) = fromDate;
endDateBuffer.at(stationindex) = toDate;
if (meteoBuffer.size() == 0) {
return false;
}
if (meteoBuffer[stationindex].size() == 0) {
return false;
}
if ((!((_date >= meteoBuffer[stationindex][0].date)
&& (meteoBuffer[stationindex][meteoBuffer[stationindex].size()-1].date >= _date)))) {
meteoBuffer[stationindex].clear();
return false;
}
//If we reach this point: Date is definitely covered
return true;
}
void BufferedIOHandler::setBufferPolicy(const buffer_policy& policy)
......@@ -265,22 +137,6 @@ void BufferedIOHandler::setBufferPolicy(const buffer_policy& policy)
}
}
void BufferedIOHandler::setBufferDuration(const Date& _beforeDate, const Date& _afterDate)
{
bufferbefore = _beforeDate;
bufferafter = _afterDate;
}
void BufferedIOHandler::legacy_readMeteoData(const Date& date_start, const Date& date_end,
std::vector< std::vector<MeteoData> >& vecMeteo,
const unsigned int& stationindex)
{
iohandler.readMeteoData(date_start, date_end, vecMeteo, stationindex);
if (&meteoBuffer != &vecMeteo)
meteoBuffer = vecMeteo; //copy by value
}
double BufferedIOHandler::getAvgSamplingRate()
{
if (vec_buffer_meteo.size() > 0){
......@@ -394,9 +250,6 @@ void BufferedIOHandler::write2DGrid(const Grid2DObject& _grid2Dobj, const std::s
}
void BufferedIOHandler::clearBuffer(){
meteoBuffer.clear();
startDateBuffer.clear();
endDateBuffer.clear();
mapBufferedGrids.clear();
}
......@@ -405,18 +258,19 @@ std::ostream& operator<<(std::ostream& os, const BufferedIOHandler& data)
os << "<BufferedIOHandler>\n";
os << "Config cfg = " << hex << &data.cfg << "\n";
os << "IOHandler &iohandler = " << hex << &data.iohandler << "\n";
os << data.meteoprocessor;
os << "Rebuffer if not found: " << data.always_rebuffer << "\n";
os << "Buffer span: -" << data.bufferbefore.getJulianDate() << " days, +" << data.bufferafter.getJulianDate() << " days\n";
os << "Current buffer content (" << data.meteoBuffer.size() << " stations, " << data.mapBufferedGrids.size() << " grids):\n";
for(unsigned int i=0; i<data.meteoBuffer.size(); i++) {
os << std::setw(10) << data.meteoBuffer[i][0].meta.stationID << " = ";
os << data.startDateBuffer[i].toString(Date::ISO) << " - ";
os << data.endDateBuffer[i].toString(Date::ISO) << ", ";
os << data.meteoBuffer[i].size() << " timesteps\n";
os << "Current buffer content (" << data.vec_buffer_meteo.size() << " stations, "
<< data.mapBufferedGrids.size() << " grids):\n";
for(unsigned int ii=0; ii<data.vec_buffer_meteo.size(); ii++) {
if (data.vec_buffer_meteo[ii].size() > 0){
os << std::setw(10) << data.vec_buffer_meteo[ii][0].meta.stationID << " = "
<< data.vec_buffer_meteo[ii][0].date.toString(Date::ISO) << " - "
<< data.vec_buffer_meteo[ii][data.vec_buffer_meteo[ii].size()-1].date.toString(Date::ISO) << ", "
<< data.vec_buffer_meteo[ii].size() << " timesteps" << endl;
}
}
std::map<std::string, Grid2DObject>::const_iterator it1;
......
meteoio/BufferedIOHandler.h
View file @ 31424f54
......@@ -25,7 +25,6 @@
#endif
#include <meteoio/Config.h>
#include <meteoio/MeteoProcessor.h>
#include <map>
#include <vector>
......@@ -86,37 +85,10 @@ class BufferedIOHandler : public IOInterface {
* @param _date start date of the data search for each station
* @param vecMeteo A vector of MeteoData objects to be filled with data
*/
void getNextMeteoData(const Date& _date, std::vector<MeteoData>& vecMeteo);
//void getNextMeteoData(const Date& _date, std::vector<MeteoData>& vecMeteo);
virtual void readStationData(const Date& date, std::vector<StationData>& vecStation);
/**
* @brief Fill vector<MeteoData> and vector<StationData> objects with multiple datasets
* corresponding to the time indicated by the Date object.
* Matching rule: Find first data set for every station which has an event time (measurement time)
* that is greater (newer) or equal to the time represented by the Date object parameter. The
* vector<StationData> object holds multiple StationData objects representing meta information
* about the meteo stations that recorded the meteo data.
*
* NOTE:
* - vecMeteo and vecStation will contain nodata objects if an exact time match is impossible
* and resampling is turned off. If resampling is turned on a resampled value is returned
* if resampling is possible (enough measurements), otherwise nodata objects will be returned
* - is there is absolutely no data to be found, and hence not even station data, vecMeteo and vecStation
* will be filled with only one nodata obejct of MeteoData and StationData respectively
*
* Example Usage:
* @code
* vector<MeteoData> vecMeteo; //empty vector
* vector<StationData> vecStation; //empty vector
* BufferedIOHandler bio(A3DIO("io.ini"), Config("io.ini"));
* bio.readMeteoData(Date(2008,06,21,11,00), vecMeteo, vecStation); //21.6.2008 11:00
* @endcode
* @param _date A Date object representing the date/time for the sought MeteoData objects
* @param vecMeteo A vector of MeteoData objects to be filled with data
*/
void readMeteoData(const Date& _date, std::vector<MeteoData>& vecMeteo);
/**
* @brief Clear all buffers in BufferedIOHandler and hence force rebuffering
*/
......@@ -160,27 +132,16 @@ class BufferedIOHandler : public IOInterface {
friend std::ostream& operator<<(std::ostream& os, const BufferedIOHandler& data);
private:
void legacy_readMeteoData(const Date& dateStart, const Date& dateEnd,
std::vector< std::vector<MeteoData> >& vecMeteo,
const unsigned int& stationindex=IOUtils::npos);
bool bufferData(const Date& _date, const unsigned int& stationindex);
void bufferAllData(const Date& _date);
void setDfltBufferProperties();
void bufferAllData(const Date& date_start, const Date& date_end);
IOHandler& iohandler;
Config cfg;
MeteoProcessor meteoprocessor;
bool always_rebuffer;
Date bufferbefore, bufferafter; //NrOfDays to buffer before and after a given date
Date buffer_start, buffer_end;
std::vector< std::vector<MeteoData> > meteoBuffer;
std::vector< std::vector<MeteoData> > vec_buffer_meteo;
std::vector< Date > startDateBuffer;
std::vector< Date > endDateBuffer;
std::map<std::string, Grid2DObject> mapBufferedGrids;
};
} //end namespace
......
meteoio/IOManager.h
View file @ 31424f54
......@@ -51,7 +51,30 @@ class IOManager {
unsigned int getMeteoData(const Date& dateStart, const Date& dateEnd,
std::vector< std::vector<MeteoData> >& vecMeteo);
//data can be raw or processed (filtered, resampled)
/**
* @brief Fill vector<MeteoData> object with multiple datasets
* corresponding to the time indicated by the Date object.
* Matching rule: Find first data set for every station which has an event time (measurement time)
* that is greater (newer) or equal to the time represented by the Date object parameter. The
* vector<StationData> object holds multiple StationData objects representing meta information
* about the meteo stations that recorded the meteo data.
*
* NOTE:
* - vecMeteo will contain nodata objects if an exact time match is impossible
* and resampling is turned off. If resampling is turned on a resampled value is returned
* if resampling is possible (enough measurements), otherwise nodata objects will be returned
* - is there is absolutely no data to be found, and hence not even station data, vecMeteo and vecStation
* will be filled with only one nodata obejct of MeteoData and StationData respectively
*
* Example Usage:
* @code
* vector<MeteoData> vecMeteo; //empty vector
* IOManager iomanager(Config("io.ini"));
* iomanager.getMeteoData(Date(2008,06,21,11,00), vecMeteo); //21.6.2008 11:00
* @endcode
* @param i_date A Date object representing the date/time for the sought MeteoData objects
* @param vecMeteo A vector of MeteoData objects to be filled with data
*/
unsigned int getMeteoData(const Date& i_date, std::vector<MeteoData>& vecMeteo);
void interpolate(const Date& date, const DEMObject& dem, const MeteoData::Parameters& meteoparam,
......
meteoio/MeteoProcessor.cc
View file @ 31424f54
......@@ -21,9 +21,7 @@ using namespace std;
namespace mio {
const unsigned int MeteoProcessor::window_half_size = 40; //org: 4
MeteoProcessor::MeteoProcessor(const Config& cfg) : mf(cfg), mi1d(cfg)
MeteoProcessor::MeteoProcessor(const Config& cfg) : mi1d(cfg)
{
//Parse [Filters] section, create processing stack for each configured parameter
set<string> set_of_used_parameters;
......@@ -109,47 +107,14 @@ unsigned int MeteoProcessor::resample(const Date& date, std::vector<MeteoData>&
return mi1d.resampleData(date, ivec);
}
void MeteoProcessor::processData(const Date& date, const std::vector<MeteoData>& vecM, MeteoData& md)
{
unsigned int currentpos = IOUtils::seek(date, vecM, false);
unsigned int startindex = IOUtils::npos, endindex = IOUtils::npos;
//No need to operate on the raw data, a copy of relevant data will be stored in these vectors:
std::vector<MeteoData> vecWindowM;
/*
* Cut out a window of data, on which the filtering and the resampling will occur
*/
bool windowexists = false;
for (int ii=(int)(currentpos-window_half_size-1); ii<=(int)(currentpos+window_half_size); ii++){
if ((ii>=0) && (ii<(int)vecM.size())){
if (!windowexists){
windowexists = true;
startindex = ii;
}
endindex = ii;
vecWindowM.push_back(vecM.at(ii));
//cout << "Added " << vecM[ii].date.toString(Date::ISO) << endl;
}
}
mf.filterData(vecM, vecWindowM, false); //first pass
unsigned int position = mi1d.resampleData(date, vecWindowM); //resampling
mf.filterData(vecM, vecWindowM, true); //checkonly, second filter pass
md = vecWindowM[position];
}
std::ostream& operator<<(std::ostream& os, const MeteoProcessor& data)
{
os << "<MeteoProcessor>\n";
os << data.mf;
os << data.mi1d;
os << "window_half_size = " << data.window_half_size << "\n";
for (map<string, ProcessingStack*>::const_iterator it=data.processing_stack.begin();
it != data.processing_stack.end(); it++){
//os << (*it->second) << endl;
}
os << "</MeteoProcessor>\n";
return os;
}
......
meteoio/MeteoProcessor.h
View file @ 31424f54
......@@ -21,7 +21,6 @@
#include <meteoio/MeteoData.h>
#include <meteoio/StationData.h>
#include <meteoio/Config.h>
#include <meteoio/MeteoFilter.h>
#include <meteoio/Meteo1DInterpolator.h>
#include <meteoio/meteofilters/ProcessingStack.h>
......@@ -42,19 +41,18 @@ class MeteoProcessor {
MeteoProcessor(const Config& cfg);
~MeteoProcessor();
/**
* @brief A function that executes all the filters for all meteo parameters
* configuered by the user
* @param[in] ivec A dataset of MeteoData
* @param[in] ovec The filtered output of MeteoData
* @param[in] second_pass Whether this is the second pass (check only filters)
*/
void process(const std::vector< std::vector<MeteoData> >& ivec,
std::vector< std::vector<MeteoData> >& ovec, const bool& second_pass=false);
unsigned int resample(const Date& date, std::vector<MeteoData>& ivec);
/**
* @brief A function that executes all the filters that have been setup in the constructor
* @param[in] date The requested date for a MeteoData object
* @param[in] vecM The raw sequence of MeteoData objects for a given station
* @param[out] md The MeteoData object to be returned
*/
void processData(const Date& date, const std::vector<MeteoData>& vecM, MeteoData& md);
void getWindowSize(ProcessingProperties& o_properties);
friend std::ostream& operator<<(std::ostream& os, const MeteoProcessor& data);
......@@ -64,9 +62,7 @@ class MeteoProcessor {
std::map<std::string, ProcessingStack*> processing_stack;
MeteoFilter mf;
Meteo1DInterpolator mi1d;
static const unsigned int window_half_size; //number of elements in filtering window on the left and right
};
} //end namespace
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment