WSL/SLF GitLab Repository

Commit fcc881f0 authored by Thomas Egger's avatar Thomas Egger
Browse files

Finalization of the new filtering infrastructure: speed optimizations...

Finalization of the new filtering infrastructure: speed optimizations (IOManager, MeteoProcessor), clean up
parent 803ea9b2
......@@ -126,6 +126,7 @@ void BufferedIOHandler::writeMeteoData(const std::vector< std::vector<MeteoData>
void BufferedIOHandler::setDfltBufferProperties()
{
always_rebuffer = true;
default_chunk_size = Date(15.0); //15 days
}
void BufferedIOHandler::setBufferPolicy(const buffer_policy& policy)
......@@ -154,6 +155,14 @@ double BufferedIOHandler::getAvgSamplingRate()
return IOUtils::nodata;
}
const std::vector<METEO_DATASET>& BufferedIOHandler::get_complete_buffer(Date& start, Date& end)
{
start = buffer_start;
end = buffer_end;
return vec_buffer_meteo; //return reference
}
void BufferedIOHandler::readMeteoData(const Date& date_start, const Date& date_end,
std::vector< std::vector<MeteoData> >& vecMeteo,
const unsigned int& /*stationindex*/)
......@@ -161,7 +170,6 @@ void BufferedIOHandler::readMeteoData(const Date& date_start, const Date& date_e
{
vecMeteo.clear();
Date default_chunk_size(15.0); //15 days
Date current_buffer_end(date_start + default_chunk_size);
vector< vector<MeteoData> > tmp_meteo_buffer;
......
......@@ -131,7 +131,11 @@ class BufferedIOHandler : public IOInterface {
friend std::ostream& operator<<(std::ostream& os, const BufferedIOHandler& data);
friend class IOManager;
private:
const std::vector<METEO_DATASET>& get_complete_buffer(Date& start, Date& end);
void setDfltBufferProperties();
void bufferAllData(const Date& date_start, const Date& date_end);
......@@ -139,7 +143,7 @@ class BufferedIOHandler : public IOInterface {
Config cfg;
bool always_rebuffer;
Date buffer_start, buffer_end;
Date buffer_start, buffer_end, default_chunk_size;
std::vector< std::vector<MeteoData> > vec_buffer_meteo;
std::map<std::string, Grid2DObject> mapBufferedGrids;
......
......@@ -24,6 +24,10 @@ namespace mio {
IOManager::IOManager(const Config& i_cfg) : cfg(i_cfg), rawio(i_cfg), bufferedio(rawio, i_cfg), meteoprocessor(i_cfg)
{
processing_level = IOManager::filtered | IOManager::resampled;
fcache_start = fcache_end = Date(0.0);
meteoprocessor.getWindowSize(proc_properties);
}
void IOManager::setProcessingLevel(const unsigned int& i_level)
......@@ -63,25 +67,63 @@ unsigned int IOManager::getMeteoData(const Date& dateStart, const Date& dateEnd,
{
vecMeteo.clear();
vector< vector<MeteoData> > tmp_meteo;
if (processing_level == IOManager::raw){
rawio.readMeteoData(dateStart, dateEnd, vecMeteo);
} else {
bufferedio.readMeteoData(dateStart, dateEnd, tmp_meteo);
//now it needs to secured that the data is actually filtered, if configured
if ((IOManager::filtered & processing_level) == IOManager::filtered){
//cout << "Now filtering ..." << endl;
meteoprocessor.process(tmp_meteo, vecMeteo);
} else {
vecMeteo = tmp_meteo;
bool success = read_filtered_cache(dateStart, dateEnd, vecMeteo);
if (!success){
vector< vector<MeteoData> > tmp_meteo;
bufferedio.readMeteoData(dateStart, dateEnd, tmp_meteo);
//now it needs to be secured that the data is actually filtered, if configured
if ((IOManager::filtered & processing_level) == IOManager::filtered){
fill_filtered_cache();
read_filtered_cache(dateStart, dateEnd, vecMeteo);
} else {
vecMeteo = tmp_meteo;
}
}
}
return vecMeteo.size(); //equivalent with the number of stations that have data
}
void IOManager::fill_filtered_cache()
{
if ((IOManager::filtered & processing_level) == IOManager::filtered){
//ask the bufferediohandler for the whole buffer
const vector<METEO_DATASET>& buffer = bufferedio.get_complete_buffer(fcache_start, fcache_end);
//cout << "Now filtering ..." << endl;
meteoprocessor.process(buffer, filtered_cache);
}
}
bool IOManager::read_filtered_cache(const Date& start_date, const Date& end_date,
std::vector<METEO_DATASET>& vec_meteo)
{
if ((start_date >= fcache_start) && (end_date <= fcache_end)){
//it's already in the filtered_cache, so just copy the requested slice
for (unsigned int ii=0; ii<filtered_cache.size(); ii++){
unsigned int startpos = IOUtils::seek(start_date, filtered_cache[ii], false);
if (startpos != IOUtils::npos){
vec_meteo.push_back(vector<MeteoData>());
unsigned int index = vec_meteo.size()-1;
for (unsigned int jj=startpos; jj<filtered_cache[ii].size(); jj++){
const MeteoData& md = filtered_cache[ii][jj];
if (md.date <= end_date)
vec_meteo[index].push_back(md);
}
}
}
return true;
}
return false;
}
void IOManager::add_to_cache(const Date& i_date, const std::vector<MeteoData>& vecMeteo)
{
//Check cache size, delete oldest elements if necessary
......@@ -100,9 +142,6 @@ unsigned int IOManager::getMeteoData(const Date& i_date, std::vector<MeteoData>&
vector< vector<MeteoData> > vec_cache;
ProcessingProperties properties;
meteoprocessor.getWindowSize(properties);
//1. Check whether user wants raw data or processed data
if (processing_level == IOManager::raw){
rawio.readMeteoData(i_date-Date(0.001), i_date+Date(0.001), vec_cache);
......@@ -123,9 +162,8 @@ unsigned int IOManager::getMeteoData(const Date& i_date, std::vector<MeteoData>&
return vecMeteo.size();
}
// request an appropriate window of data from bufferedio
// Hand window of data over to meteo processor
getMeteoData(i_date-properties.time_before, i_date+properties.time_after, vec_cache);
//request an appropriate window of filtered or unfiltered data
getMeteoData(i_date-proc_properties.time_before, i_date+proc_properties.time_after, vec_cache);
//vec_cache is either filtered or unfiltered, in any case it is wise to buffer it
for (unsigned int ii=0; ii<vec_cache.size(); ii++){//resampling for every station
......
......@@ -93,13 +93,19 @@ class IOManager {
private:
void add_to_cache(const Date& i_date, const std::vector<MeteoData>& vecMeteo);
void fill_filtered_cache();
bool read_filtered_cache(const Date& start_date, const Date& end_date,
std::vector<METEO_DATASET>& vec_meteo);
Config cfg;
IOHandler rawio;
BufferedIOHandler bufferedio;
MeteoProcessor meteoprocessor;
ProcessingProperties proc_properties;
std::map<Date, std::vector<MeteoData> > meteo_cache; ///< stores already fetched data points
std::map<Date, std::vector<MeteoData> > meteo_cache; ///< stores already fetched data points
std::vector< std::vector<MeteoData> > filtered_cache; ///< stores already filtered data intervals
Date fcache_start, fcache_end;
unsigned int processing_level;
};
} //end namespace
......
......@@ -38,7 +38,7 @@ Meteo1DInterpolator::Meteo1DInterpolator(const Config& _cfg) : cfg(_cfg) {
taskargs.push_back(vecResamplingArguments);
}
window_size = 0.5; //default size is half a day
window_size = 0.5; //default size is 0.5 julian days
string tmp = cfg.get("WINDOW_SIZE", "Interpolations1D", Config::nothrow);
if (tmp != "") {
window_size = cfg.get("WINDOW_SIZE", "Interpolations1D");
......
......@@ -46,12 +46,11 @@
#include <meteoio/Meteo1DInterpolator.h>
#include <meteoio/Meteo2DInterpolator.h>
#include <meteoio/MeteoData.h>
//#include <meteoio/MeteoFilter.h> //HACK: is it obsolete?
#include <meteoio/MeteoProcessor.h> //HACK: is it obsolete?
#include <meteoio/MeteoProcessor.h>
#include <meteoio/meteofilters/FilterBlock.h>
#include <meteoio/meteofilters/ProcessingBlock.h>
#include <meteoio/meteofilters/ProcessingStack.h>
#include <meteoio/ResamplingAlgorithms.h> //HACK: is it obsolete?
#include <meteoio/ResamplingAlgorithms.h>
#include <meteoio/StationData.h>
#include <meteoio/IOManager.h>
......
......@@ -90,11 +90,15 @@ void MeteoProcessor::process(const std::vector< std::vector<MeteoData> >& ivec,
std::vector< std::vector<MeteoData> >& ovec, const bool& second_pass)
{
//call the different processing stacks
std::vector< std::vector<MeteoData> > vec_tmp = ivec;
std::vector< std::vector<MeteoData> > vec_tmp;
//cout << "Calling processing stacks for all parameters" << endl;
for (map<string, ProcessingStack*>::const_iterator it=processing_stack.begin(); it != processing_stack.end(); it++){
//cout << "Calling processing stack for parameter " << it->first << endl;
(*(it->second)).process(vec_tmp, ovec, second_pass);
if (it==processing_stack.begin()){
(*(it->second)).process(ivec, ovec, second_pass);
} else {
(*(it->second)).process(vec_tmp, ovec, second_pass);
}
vec_tmp = ovec;
}
......
......@@ -60,6 +60,7 @@ class MeteoProcessor {
unsigned int resample(const Date& date, std::vector<MeteoData>& ivec);
void getWindowSize(ProcessingProperties& o_properties);
friend std::ostream& operator<<(std::ostream& os, const MeteoProcessor& data);
private:
......
......@@ -36,6 +36,7 @@ void FilterMinMax::process(const unsigned int& index, const std::vector<MeteoDat
ovec.push_back(ivec[ii]);
double& tmp = ovec[ii].param(index);
if (tmp == IOUtils::nodata) continue; //preserve nodata values
if (tmp < min_val){
if (is_soft){
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment