Finalization of the new filtering infrastructure: speed optimizations...

Finalization of the new filtering infrastructure: speed optimizations (IOManager, MeteoProcessor), clean up

meteoio/BufferedIOHandler.cc

@@ -126,6 +126,7 @@ void BufferedIOHandler::writeMeteoData(const std::vector< std::vector<MeteoData>
 void BufferedIOHandler::setDfltBufferProperties()
 {
 	always_rebuffer = true;
+	default_chunk_size = Date(15.0); //15 days
 }
 
 void BufferedIOHandler::setBufferPolicy(const buffer_policy& policy)
@@ -154,6 +155,14 @@ double BufferedIOHandler::getAvgSamplingRate()
 	return IOUtils::nodata;
 }
 
+const std::vector<METEO_DATASET>& BufferedIOHandler::get_complete_buffer(Date& start, Date& end)
+{
+	start = buffer_start;
+	end   = buffer_end;
+
+	return vec_buffer_meteo; //return reference
+}
+
 void BufferedIOHandler::readMeteoData(const Date& date_start, const Date& date_end, 
 							   std::vector< std::vector<MeteoData> >& vecMeteo,
 							   const unsigned int& /*stationindex*/)
@@ -161,7 +170,6 @@ void BufferedIOHandler::readMeteoData(const Date& date_start, const Date& date_e
 {
 	vecMeteo.clear();
 
-	Date default_chunk_size(15.0); //15 days
 	Date current_buffer_end(date_start + default_chunk_size);
 	vector< vector<MeteoData> > tmp_meteo_buffer;
 

meteoio/BufferedIOHandler.h

@@ -131,7 +131,11 @@ class BufferedIOHandler : public IOInterface {
 
 		friend std::ostream& operator<<(std::ostream& os, const BufferedIOHandler& data);
 
+		friend class IOManager;
+
 	private:
+		const std::vector<METEO_DATASET>& get_complete_buffer(Date& start, Date& end);
+
 		void setDfltBufferProperties();
 		void bufferAllData(const Date& date_start, const Date& date_end);
 
@@ -139,7 +143,7 @@ class BufferedIOHandler : public IOInterface {
 		Config cfg;
 
 		bool always_rebuffer;
-		Date buffer_start, buffer_end;
+		Date buffer_start, buffer_end, default_chunk_size;
 
 		std::vector< std::vector<MeteoData> > vec_buffer_meteo;
 		std::map<std::string, Grid2DObject> mapBufferedGrids;

meteoio/IOManager.cc

@@ -24,6 +24,10 @@ namespace mio {
 IOManager::IOManager(const Config& i_cfg) : cfg(i_cfg), rawio(i_cfg), bufferedio(rawio, i_cfg), meteoprocessor(i_cfg)
 {
 	processing_level = IOManager::filtered | IOManager::resampled;
+
+	fcache_start = fcache_end = Date(0.0);
+
+	meteoprocessor.getWindowSize(proc_properties);
 }
 
 void IOManager::setProcessingLevel(const unsigned int& i_level)
@@ -63,25 +67,63 @@ unsigned int IOManager::getMeteoData(const Date& dateStart, const Date& dateEnd,
 {
 	vecMeteo.clear();
 
-	vector< vector<MeteoData> > tmp_meteo;
-
 	if (processing_level == IOManager::raw){
 		rawio.readMeteoData(dateStart, dateEnd, vecMeteo);
 	} else {
-		bufferedio.readMeteoData(dateStart, dateEnd, tmp_meteo);
-		//now it needs to secured that the data is actually filtered, if configured
-
-		if ((IOManager::filtered & processing_level) == IOManager::filtered){
-			//cout << "Now filtering ..." << endl;
-			meteoprocessor.process(tmp_meteo, vecMeteo);
-		} else {
-			vecMeteo = tmp_meteo;
+		bool success = read_filtered_cache(dateStart, dateEnd, vecMeteo);
+
+		if (!success){
+			vector< vector<MeteoData> > tmp_meteo;
+			bufferedio.readMeteoData(dateStart, dateEnd, tmp_meteo);
+
+			//now it needs to be secured that the data is actually filtered, if configured
+			if ((IOManager::filtered & processing_level) == IOManager::filtered){
+				fill_filtered_cache();
+				read_filtered_cache(dateStart, dateEnd, vecMeteo);
+			} else {
+				vecMeteo = tmp_meteo;
+			}
 		}
 	}
 
 	return vecMeteo.size(); //equivalent with the number of stations that have data
 }
 
+void IOManager::fill_filtered_cache()
+{
+	if ((IOManager::filtered & processing_level) == IOManager::filtered){
+		//ask the bufferediohandler for the whole buffer
+		const vector<METEO_DATASET>& buffer = bufferedio.get_complete_buffer(fcache_start, fcache_end);
+
+		//cout << "Now filtering ..." << endl;
+		meteoprocessor.process(buffer, filtered_cache);
+	}
+}
+
+bool IOManager::read_filtered_cache(const Date& start_date, const Date& end_date,
+                                    std::vector<METEO_DATASET>& vec_meteo)
+{
+	if ((start_date >= fcache_start) && (end_date <= fcache_end)){
+		//it's already in the filtered_cache, so just copy the requested slice
+		for (unsigned int ii=0; ii<filtered_cache.size(); ii++){
+			unsigned int startpos = IOUtils::seek(start_date, filtered_cache[ii], false);
+			if (startpos != IOUtils::npos){
+				vec_meteo.push_back(vector<MeteoData>());
+				unsigned int index = vec_meteo.size()-1;
+				for (unsigned int jj=startpos; jj<filtered_cache[ii].size(); jj++){
+					const MeteoData& md = filtered_cache[ii][jj];
+					if (md.date <= end_date)
+						vec_meteo[index].push_back(md);
+				}
+			}
+		}
+
+		return true;
+	}
+	
+	return false;
+}
+
 void IOManager::add_to_cache(const Date& i_date, const std::vector<MeteoData>& vecMeteo)
 {
 	//Check cache size, delete oldest elements if necessary
@@ -100,9 +142,6 @@ unsigned int IOManager::getMeteoData(const Date& i_date, std::vector<MeteoData>&
 
 	vector< vector<MeteoData> > vec_cache;
 
-	ProcessingProperties properties;
-	meteoprocessor.getWindowSize(properties);
-
 	//1. Check whether user wants raw data or processed data
 	if (processing_level == IOManager::raw){
 		rawio.readMeteoData(i_date-Date(0.001), i_date+Date(0.001), vec_cache);
@@ -123,9 +162,8 @@ unsigned int IOManager::getMeteoData(const Date& i_date, std::vector<MeteoData>&
 		return vecMeteo.size();
 	}
 
-	//    request an appropriate window of data from bufferedio
-	//    Hand window of data over to meteo processor
-	getMeteoData(i_date-properties.time_before, i_date+properties.time_after, vec_cache);
+	//request an appropriate window of filtered or unfiltered data
+	getMeteoData(i_date-proc_properties.time_before, i_date+proc_properties.time_after, vec_cache);
 	//vec_cache is either filtered or unfiltered, in any case it is wise to buffer it
 
 	for (unsigned int ii=0; ii<vec_cache.size(); ii++){//resampling for every station

meteoio/IOManager.h

@@ -93,13 +93,19 @@ class IOManager {
 
 	private:
 		void add_to_cache(const Date& i_date, const std::vector<MeteoData>& vecMeteo);
+		void fill_filtered_cache();
+		bool read_filtered_cache(const Date& start_date, const Date& end_date,
+							std::vector<METEO_DATASET>& vec_meteo);
 
 		Config cfg;
 		IOHandler rawio;
 		BufferedIOHandler bufferedio;
 		MeteoProcessor meteoprocessor;
+		ProcessingProperties proc_properties;
 
-		std::map<Date, std::vector<MeteoData> > meteo_cache; ///< stores already fetched data points
+		std::map<Date, std::vector<MeteoData> > meteo_cache;  ///< stores already fetched data points
+		std::vector< std::vector<MeteoData> > filtered_cache; ///< stores already filtered data intervals
+		Date fcache_start, fcache_end;
 		unsigned int processing_level;
 };
 } //end namespace

meteoio/Meteo1DInterpolator.cc

@@ -38,7 +38,7 @@ Meteo1DInterpolator::Meteo1DInterpolator(const Config& _cfg) : cfg(_cfg) {
 		taskargs.push_back(vecResamplingArguments);
 	}
 
-	window_size = 0.5; //default size is half a day
+	window_size = 0.5; //default size is 0.5 julian days
 	string tmp = cfg.get("WINDOW_SIZE", "Interpolations1D", Config::nothrow); 
 	if (tmp != "") {
 		window_size = cfg.get("WINDOW_SIZE", "Interpolations1D"); 

meteoio/MeteoIO.h

@@ -46,12 +46,11 @@
 #include <meteoio/Meteo1DInterpolator.h>
 #include <meteoio/Meteo2DInterpolator.h>
 #include <meteoio/MeteoData.h>
-//#include <meteoio/MeteoFilter.h> //HACK: is it obsolete?
-#include <meteoio/MeteoProcessor.h> //HACK: is it obsolete?
+#include <meteoio/MeteoProcessor.h>
 #include <meteoio/meteofilters/FilterBlock.h>
 #include <meteoio/meteofilters/ProcessingBlock.h>
 #include <meteoio/meteofilters/ProcessingStack.h>
-#include <meteoio/ResamplingAlgorithms.h> //HACK: is it obsolete?
+#include <meteoio/ResamplingAlgorithms.h>
 #include <meteoio/StationData.h>
 #include <meteoio/IOManager.h>
 

meteoio/MeteoProcessor.cc

@@ -90,11 +90,15 @@ void MeteoProcessor::process(const std::vector< std::vector<MeteoData> >& ivec,
                              std::vector< std::vector<MeteoData> >& ovec, const bool& second_pass)
 {
 	//call the different processing stacks
-	std::vector< std::vector<MeteoData> > vec_tmp = ivec;
+	std::vector< std::vector<MeteoData> > vec_tmp;
 	//cout << "Calling processing stacks for all parameters" << endl;
 	for (map<string, ProcessingStack*>::const_iterator it=processing_stack.begin(); it != processing_stack.end(); it++){
 		//cout << "Calling processing stack for parameter " << it->first << endl;
-		(*(it->second)).process(vec_tmp, ovec, second_pass);
+		if (it==processing_stack.begin()){
+			(*(it->second)).process(ivec, ovec, second_pass);
+		} else {
+			(*(it->second)).process(vec_tmp, ovec, second_pass);
+		}
 		vec_tmp = ovec;
 	}
 

meteoio/MeteoProcessor.h

@@ -60,6 +60,7 @@ class MeteoProcessor {
 		unsigned int resample(const Date& date, std::vector<MeteoData>& ivec);
 
 		void getWindowSize(ProcessingProperties& o_properties);
+
 		friend std::ostream& operator<<(std::ostream& os, const MeteoProcessor& data);
 
  	private:

meteoio/meteofilters/FilterMinMax.cc

@@ -36,6 +36,7 @@ void FilterMinMax::process(const unsigned int& index, const std::vector<MeteoDat
 		ovec.push_back(ivec[ii]);
 		
 		double& tmp = ovec[ii].param(index);
+		if (tmp == IOUtils::nodata) continue; //preserve nodata values
 
 		if (tmp < min_val){
 			if (is_soft){