A gridBuffer class has finally been implemented (basically, moving the old...

A gridBuffer class has finally been implemented (basically, moving the old grid buffering code into it) and is now in use for the GridsManager. An invalid file name in the examples has been fixed and a toString() debug message has been improved in the TimeSeriesManager.

meteoio/GridsManager.cc

@@ -23,10 +23,12 @@ using namespace std;
 namespace mio {
 
 GridsManager::GridsManager(IOHandler& in_iohandler, const Config& in_cfg)
-             : iohandler(in_iohandler), cfg(in_cfg), mapBufferedGrids(), dem_buffer(), IndexBufferedGrids(), max_grids(10),
+             : iohandler(in_iohandler), cfg(in_cfg), buffer(0),
                processing_level(IOUtils::filtered | IOUtils::resampled | IOUtils::generated)
 {
+	size_t max_grids = 10;
 	cfg.getValue("BUFF_GRIDS", "General", max_grids, IOUtils::nothrow);
+	buffer.setMaxGrids(max_grids);
 }
 
 /**
@@ -55,8 +57,7 @@ void GridsManager::setProcessingLevel(const unsigned int& i_level)
 
 void GridsManager::clear_cache()
 {
-	mapBufferedGrids.clear();
-	IndexBufferedGrids.clear();
+	buffer.clear();
 }
 
 void GridsManager::read2DGrid(Grid2DObject& grid2D, const std::string& filename)
@@ -64,11 +65,11 @@ void GridsManager::read2DGrid(Grid2DObject& grid2D, const std::string& filename)
 	if (processing_level == IOUtils::raw){
 		iohandler.read2DGrid(grid2D, filename);
 	} else {
-		if (getFromBuffer(filename, grid2D))
+		if (buffer.get(grid2D, filename))
 			return;
 
 		iohandler.read2DGrid(grid2D, filename);
-		addToBuffer(grid2D, filename);
+		buffer.push(grid2D, filename);
 	}
 }
 
@@ -77,16 +78,11 @@ void GridsManager::read2DGrid(Grid2DObject& grid2D, const MeteoGrids::Parameters
 	if (processing_level == IOUtils::raw){
 		iohandler.read2DGrid(grid2D, parameter, date);
 	} else {
-		if (max_grids>0) {
-			const string grid_hash = date.toString(Date::ISO)+"::"+MeteoGrids::getParameterName(parameter);
-			if (getFromBuffer(grid_hash, grid2D))
-				return;
-
-			iohandler.read2DGrid(grid2D, parameter, date);
-			addToBuffer(grid2D, grid_hash); //the STL containers make a copy
-		} else {
-			iohandler.read2DGrid(grid2D, parameter, date);
-		}
+		if (buffer.get(grid2D, parameter, date))
+			return;
+
+		iohandler.read2DGrid(grid2D, parameter, date);
+		buffer.push(grid2D, parameter, date);
 	}
 }
 
@@ -95,31 +91,11 @@ void GridsManager::readDEM(DEMObject& grid2D)
 	if (processing_level == IOUtils::raw){
 		iohandler.readDEM(grid2D);
 	} else {
-		if (max_grids>0) {
-			if (dem_buffer.size() == 1) { //HACK: properly manage buffering multiple dems!
-				//already in buffer. If the update properties have changed,
-				//we copy the ones given in input and force the update of the object
-				const DEMObject::update_type in_ppt = (DEMObject::update_type)grid2D.getUpdatePpt();
-				const DEMObject::slope_type in_slope_alg = (DEMObject::slope_type)grid2D.getDefaultAlgorithm();
-
-				grid2D = dem_buffer[0];
-				const DEMObject::update_type buff_ppt = (DEMObject::update_type)grid2D.getUpdatePpt();
-				const DEMObject::slope_type buff_slope_alg = (DEMObject::slope_type)grid2D.getDefaultAlgorithm();
-
-				if (in_ppt!=buff_ppt || in_slope_alg!=buff_slope_alg) {
-					grid2D.setDefaultAlgorithm(in_slope_alg);
-					grid2D.setUpdatePpt(in_ppt);
-					grid2D.update();
-				}
-
-				return;
-			}
-
-			iohandler.readDEM(grid2D);
-			dem_buffer.push_back(grid2D); //the STL containers make a copy
-		} else {
-			iohandler.readDEM(grid2D);
-		}
+		if (buffer.get(grid2D))
+			return;
+
+		iohandler.readDEM(grid2D);
+		buffer.push(grid2D);
 	}
 }
 
@@ -128,11 +104,11 @@ void GridsManager::readLanduse(Grid2DObject& grid2D)
 	if (processing_level == IOUtils::raw){
 		iohandler.readLanduse(grid2D);
 	} else {
-		if (getFromBuffer("/:LANDUSE", grid2D))
+		if (buffer.get(grid2D, "/:LANDUSE"))
 			return;
 
 		iohandler.readLanduse(grid2D);
-		addToBuffer(grid2D, "/:LANDUSE");
+		buffer.push(grid2D, "/:LANDUSE");
 	}
 }
 
@@ -141,16 +117,12 @@ void GridsManager::readAssimilationData(const Date& date, Grid2DObject& grid2D)
 	if (processing_level == IOUtils::raw){
 		iohandler.readAssimilationData(date, grid2D);
 	} else {
-		if(max_grids>0) {
-			const string grid_hash = "/:ASSIMILATIONDATA"+date.toString(Date::ISO);
-			if (getFromBuffer(grid_hash, grid2D))
-				return;
-
-			iohandler.readAssimilationData(date, grid2D);
-			addToBuffer(grid2D, grid_hash); //the STL containers make a copy
-		} else {
-			iohandler.readAssimilationData(date, grid2D);
-		}
+		const string grid_hash = "/:ASSIMILATIONDATA"+date.toString(Date::ISO);
+		if (buffer.get(grid2D, grid_hash))
+			return;
+
+		iohandler.readAssimilationData(date, grid2D);
+		buffer.push(grid2D, grid_hash);
 	}
 }
 
@@ -164,55 +136,13 @@ void GridsManager::write2DGrid(const Grid2DObject& grid2D, const MeteoGrids::Par
 	iohandler.write2DGrid(grid2D, parameter, date);
 }
 
-void GridsManager::addToBuffer(const Grid2DObject& in_grid2Dobj, const std::string& grid_hash)
-{
-	if (max_grids==0) return;
-
-	if(IndexBufferedGrids.size() >= max_grids) { //we need to remove the oldest grid
-		mapBufferedGrids.erase( mapBufferedGrids.find( IndexBufferedGrids.front() ) );
-		IndexBufferedGrids.erase( IndexBufferedGrids.begin() );
-	}
-	mapBufferedGrids[ grid_hash ] = in_grid2Dobj;
-	IndexBufferedGrids.push_back( grid_hash  );
-}
-
-bool GridsManager::getFromBuffer(const std::string& grid_hash, Grid2DObject& grid) const
-{
-	if (IndexBufferedGrids.empty())
-		return false;
-
-	const std::map<std::string, Grid2DObject>::const_iterator it = mapBufferedGrids.find( grid_hash );
-	if (it != mapBufferedGrids.end()) { //already in map
-		grid = (*it).second;
-		return true;
-	}
-
-	return false;
-}
-
 const std::string GridsManager::toString() const {
 	ostringstream os;
 	os << "<GridsManager>\n";
 	os << "Config& cfg = " << hex << &cfg << dec << "\n";
 	os << "IOHandler& iohandler = " << hex << &iohandler << dec << "\n";
 	os << "Processing level = " << processing_level << "\n";
-	os << "Max buffered grids = " << max_grids << "\n";
-
-	//cache content
-	os << "Current buffer content (" << mapBufferedGrids.size() << " grids):\n";
-	std::map<std::string, Grid2DObject>::const_iterator it1;
-	for (it1=mapBufferedGrids.begin(); it1 != mapBufferedGrids.end(); ++it1){
-		os << setw(10) << "Grid " << it1->first << "\n";
-	}
-
-	//dem buffer
-	os << "Dem buffer content (" << dem_buffer.size() << " grids):\n";
-	for (size_t ii=0; ii<dem_buffer.size(); ++ii){
-		std::ostringstream ss;
-		ss << dem_buffer[ii].llcorner.printLatLon() << " " << dem_buffer[ii].getNx() << "x" << dem_buffer[ii].getNy() << " @" << dem_buffer[ii].cellsize << "m";
-		os << setw(10) << "Dem " << ss.str() << "\n";
-	}
-
+	os << buffer.toString();
 	os << "</GridsManager>\n";
 	return os.str();
 }

meteoio/GridsManager.h

@@ -19,6 +19,7 @@
 #define __GRIDSMANAGER_H__
 
 #include <meteoio/DataGenerator.h>
+#include <meteoio/dataClasses/Buffer.h>
 #include <meteoio/dataClasses/MeteoData.h>
 #include <meteoio/dataClasses/Coords.h>
 #include <meteoio/IOHandler.h>
@@ -51,11 +52,8 @@ class GridsManager {
 
 		IOHandler& iohandler;
 		const Config& cfg;
+		GridBuffer buffer;
 
-		std::map<std::string, Grid2DObject> mapBufferedGrids;
-		std::vector<DEMObject> dem_buffer;
-		std::vector<std::string> IndexBufferedGrids; // this is required in order to know which grid is the oldest one
-		size_t max_grids; ///< How many grids to buffer (grids, dem, landuse and assimilation grids together)
 		unsigned int processing_level;
 };
 } //end namespace

meteoio/TimeSeriesManager.cc

@@ -317,8 +317,8 @@ const std::string TimeSeriesManager::toString() const {
 	os << "Processing level = " << processing_level << "\n";
 	os << dataGenerator.toString();
 
-	os << "RawBuffer:" << raw_buffer.toString();
-	os << "Filteredcache:" << raw_buffer.toString();
+	os << "RawBuffer:\n" << raw_buffer.toString();
+	os << "Filteredcache:\n" << raw_buffer.toString();
 
 	//display point_cache
 	size_t count=0;

meteoio/dataClasses/Buffer.cc

@@ -221,4 +221,113 @@ const std::string MeteoBuffer::toString() const
 	return os.str();
 }
 
+
+/****************************** GridBuffer class ***********************************************/
+
+bool GridBuffer::get(Grid2DObject& grid, const std::string& grid_hash) const
+{
+	if (IndexBufferedGrids.empty())
+		return false;
+
+	const std::map<std::string, Grid2DObject>::const_iterator it = mapBufferedGrids.find( grid_hash );
+	if (it != mapBufferedGrids.end()) { //already in map
+		grid = (*it).second;
+		return true;
+	}
+
+	return false;
+}
+
+bool GridBuffer::get(Grid2DObject& grid, const MeteoGrids::Parameters& parameter, const Date& date) const
+{
+	const string grid_hash = date.toString(Date::ISO)+"::"+MeteoGrids::getParameterName(parameter);
+	return get(grid, grid_hash);
+}
+
+bool GridBuffer::get(DEMObject& grid) const
+{
+	if (max_grids==0) return false;
+	
+	if (!dem_buffer.empty()) {
+		//HACK: properly manage buffering multiple dems!
+		//already in buffer. If the update properties have changed,
+		//we copy the ones given in input and force the update of the object
+		const DEMObject::update_type in_ppt = (DEMObject::update_type)grid.getUpdatePpt();
+		const DEMObject::slope_type in_slope_alg = (DEMObject::slope_type)grid.getDefaultAlgorithm();
+
+		grid = dem_buffer[0];
+		const DEMObject::update_type buff_ppt = (DEMObject::update_type)grid.getUpdatePpt();
+		const DEMObject::slope_type buff_slope_alg = (DEMObject::slope_type)grid.getDefaultAlgorithm();
+
+		if (in_ppt!=buff_ppt || in_slope_alg!=buff_slope_alg) {
+			grid.setDefaultAlgorithm(in_slope_alg);
+			grid.setUpdatePpt(in_ppt);
+			grid.update();
+		}
+
+		return true;
+	}
+	
+	return false;
+}
+
+void GridBuffer::push(const DEMObject& in_grid)
+{
+	//HACK: properly manage buffering multiple dems!
+	//std::ostringstream grid_hash;
+	//grid_hash << dem_buffer[ii].llcorner.printLatLon() << " " << dem_buffer[ii].getNx() << "x" << dem_buffer[ii].getNy() << " @" << dem_buffer[ii].cellsize << "m";
+	if (max_grids==0) return;
+	
+	if (!dem_buffer.empty())
+		dem_buffer.clear();
+	
+	dem_buffer.push_back( in_grid );
+}
+
+void GridBuffer::push(const Grid2DObject& in_grid, const std::string& grid_hash)
+{
+	if (max_grids==0) return;
+
+	if(IndexBufferedGrids.size() >= max_grids) { //we need to remove the oldest grid
+		mapBufferedGrids.erase( mapBufferedGrids.find( IndexBufferedGrids.front() ) );
+		IndexBufferedGrids.erase( IndexBufferedGrids.begin() );
+	}
+	mapBufferedGrids[ grid_hash ] = in_grid;
+	IndexBufferedGrids.push_back( grid_hash  );
+}
+
+void GridBuffer::push(const Grid2DObject& in_grid, const MeteoGrids::Parameters& parameter, const Date& date)
+{
+	const string grid_hash = date.toString(Date::ISO)+"::"+MeteoGrids::getParameterName(parameter);
+	push(in_grid, grid_hash);
+}
+
+const std::string GridBuffer::toString() const
+{
+	ostringstream os;
+	os << "<GridBuffer>\n";
+	os << "Max buffered grids = " << max_grids << "\n";
+
+	//cache content
+	os << "Current buffer content (" << mapBufferedGrids.size() << " grids):\n";
+	std::map<std::string, Grid2DObject>::const_iterator it1;
+	for (it1=mapBufferedGrids.begin(); it1 != mapBufferedGrids.end(); ++it1){
+		os << setw(10) << "Grid " << it1->first << "\n";
+	}
+	
+	//dem buffer
+	if (!dem_buffer.empty()) {
+		os << "Dem buffer content (" << dem_buffer.size() << " grids):\n";
+		for (size_t ii=0; ii<dem_buffer.size(); ++ii){
+			std::ostringstream ss;
+			ss << dem_buffer[ii].llcorner.printLatLon() << " " << dem_buffer[ii].getNx() << "x" << dem_buffer[ii].getNy() << " @" << dem_buffer[ii].cellsize << "m";
+			os << setw(10) << "Dem " << ss.str() << "\n";
+		}
+	}
+
+	os << "</GridBuffer>\n";
+	return os.str();
+}
+
+
 } //end namespace
\ No newline at end of file

meteoio/dataClasses/Buffer.h

@@ -18,6 +18,8 @@
 #ifndef __BUFFER_H__
 #define __BUFFER_H__
 
+#include <meteoio/dataClasses/Grid2DObject.h>
+#include <meteoio/dataClasses/DEMObject.h>
 #include <meteoio/dataClasses/Date.h>
 #include <meteoio/dataClasses/MeteoData.h>
 
@@ -112,5 +114,41 @@ class MeteoBuffer {
 		Date ts_start, ts_end; ///< store the beginning and the end date of the ts_buffer
 };
 
+/**
+ * @class GridBuffer
+ * @brief A class to buffer gridded data.
+ * This class buffers Grid2D objects. It implements a proper ring buffer, thus removing old bufered grids
+ * when necessary.
+ *
+ * @ingroup data_str
+ * @author Mathias Bavay
+ * @date   2015-02-06
+*/
+class GridBuffer {
+	public:
+		GridBuffer(const size_t& in_max_grids) : dem_buffer(), mapBufferedGrids(), IndexBufferedGrids(), max_grids(in_max_grids) {};
+		
+		bool empty() const {return IndexBufferedGrids.empty();};
+		void clear() {mapBufferedGrids.clear(); IndexBufferedGrids.clear();};
+		size_t size() const {return IndexBufferedGrids.size();};
+		
+		void setMaxGrids(const size_t& in_max_grids) {max_grids=in_max_grids;};
+		
+		bool get(Grid2DObject& grid, const std::string& grid_hash) const;
+		bool get(Grid2DObject& grid, const MeteoGrids::Parameters& parameter, const Date& date) const;
+		bool get(DEMObject& grid) const;
+		
+		void push(const Grid2DObject& in_grid, const std::string& grid_hash);
+		void push(const Grid2DObject& in_grid, const MeteoGrids::Parameters& parameter, const Date& date);
+		void push(const DEMObject& in_grid);
+		
+		const std::string toString() const;
+	private:
+		std::vector<DEMObject> dem_buffer;
+		std::map<std::string, Grid2DObject> mapBufferedGrids;
+		std::vector<std::string> IndexBufferedGrids; // this is required in order to know which grid is the oldest one
+		size_t max_grids; ///< How many grids to buffer (grids, dem, landuse and assimilation grids together)
+};
+
 }
 #endif
\ No newline at end of file