WSL/SLF GitLab Repository

Commit ee7edab4 authored by Mathias Bavay's avatar Mathias Bavay
Browse files

A gridBuffer class has finally been implemented (basically, moving the old...

A gridBuffer class has finally been implemented (basically, moving the old grid buffering code into it) and is now in use for the GridsManager. An invalid file name in the examples has been fixed and a toString() debug message has been improved in the TimeSeriesManager.
parent 61df964f
......@@ -23,10 +23,12 @@ using namespace std;
namespace mio {
GridsManager::GridsManager(IOHandler& in_iohandler, const Config& in_cfg)
: iohandler(in_iohandler), cfg(in_cfg), mapBufferedGrids(), dem_buffer(), IndexBufferedGrids(), max_grids(10),
: iohandler(in_iohandler), cfg(in_cfg), buffer(0),
processing_level(IOUtils::filtered | IOUtils::resampled | IOUtils::generated)
{
size_t max_grids = 10;
cfg.getValue("BUFF_GRIDS", "General", max_grids, IOUtils::nothrow);
buffer.setMaxGrids(max_grids);
}
/**
......@@ -55,8 +57,7 @@ void GridsManager::setProcessingLevel(const unsigned int& i_level)
void GridsManager::clear_cache()
{
mapBufferedGrids.clear();
IndexBufferedGrids.clear();
buffer.clear();
}
void GridsManager::read2DGrid(Grid2DObject& grid2D, const std::string& filename)
......@@ -64,11 +65,11 @@ void GridsManager::read2DGrid(Grid2DObject& grid2D, const std::string& filename)
if (processing_level == IOUtils::raw){
iohandler.read2DGrid(grid2D, filename);
} else {
if (getFromBuffer(filename, grid2D))
if (buffer.get(grid2D, filename))
return;
iohandler.read2DGrid(grid2D, filename);
addToBuffer(grid2D, filename);
buffer.push(grid2D, filename);
}
}
......@@ -77,16 +78,11 @@ void GridsManager::read2DGrid(Grid2DObject& grid2D, const MeteoGrids::Parameters
if (processing_level == IOUtils::raw){
iohandler.read2DGrid(grid2D, parameter, date);
} else {
if (max_grids>0) {
const string grid_hash = date.toString(Date::ISO)+"::"+MeteoGrids::getParameterName(parameter);
if (getFromBuffer(grid_hash, grid2D))
return;
iohandler.read2DGrid(grid2D, parameter, date);
addToBuffer(grid2D, grid_hash); //the STL containers make a copy
} else {
iohandler.read2DGrid(grid2D, parameter, date);
}
if (buffer.get(grid2D, parameter, date))
return;
iohandler.read2DGrid(grid2D, parameter, date);
buffer.push(grid2D, parameter, date);
}
}
......@@ -95,31 +91,11 @@ void GridsManager::readDEM(DEMObject& grid2D)
if (processing_level == IOUtils::raw){
iohandler.readDEM(grid2D);
} else {
if (max_grids>0) {
if (dem_buffer.size() == 1) { //HACK: properly manage buffering multiple dems!
//already in buffer. If the update properties have changed,
//we copy the ones given in input and force the update of the object
const DEMObject::update_type in_ppt = (DEMObject::update_type)grid2D.getUpdatePpt();
const DEMObject::slope_type in_slope_alg = (DEMObject::slope_type)grid2D.getDefaultAlgorithm();
grid2D = dem_buffer[0];
const DEMObject::update_type buff_ppt = (DEMObject::update_type)grid2D.getUpdatePpt();
const DEMObject::slope_type buff_slope_alg = (DEMObject::slope_type)grid2D.getDefaultAlgorithm();
if (in_ppt!=buff_ppt || in_slope_alg!=buff_slope_alg) {
grid2D.setDefaultAlgorithm(in_slope_alg);
grid2D.setUpdatePpt(in_ppt);
grid2D.update();
}
return;
}
iohandler.readDEM(grid2D);
dem_buffer.push_back(grid2D); //the STL containers make a copy
} else {
iohandler.readDEM(grid2D);
}
if (buffer.get(grid2D))
return;
iohandler.readDEM(grid2D);
buffer.push(grid2D);
}
}
......@@ -128,11 +104,11 @@ void GridsManager::readLanduse(Grid2DObject& grid2D)
if (processing_level == IOUtils::raw){
iohandler.readLanduse(grid2D);
} else {
if (getFromBuffer("/:LANDUSE", grid2D))
if (buffer.get(grid2D, "/:LANDUSE"))
return;
iohandler.readLanduse(grid2D);
addToBuffer(grid2D, "/:LANDUSE");
buffer.push(grid2D, "/:LANDUSE");
}
}
......@@ -141,16 +117,12 @@ void GridsManager::readAssimilationData(const Date& date, Grid2DObject& grid2D)
if (processing_level == IOUtils::raw){
iohandler.readAssimilationData(date, grid2D);
} else {
if(max_grids>0) {
const string grid_hash = "/:ASSIMILATIONDATA"+date.toString(Date::ISO);
if (getFromBuffer(grid_hash, grid2D))
return;
iohandler.readAssimilationData(date, grid2D);
addToBuffer(grid2D, grid_hash); //the STL containers make a copy
} else {
iohandler.readAssimilationData(date, grid2D);
}
const string grid_hash = "/:ASSIMILATIONDATA"+date.toString(Date::ISO);
if (buffer.get(grid2D, grid_hash))
return;
iohandler.readAssimilationData(date, grid2D);
buffer.push(grid2D, grid_hash);
}
}
......@@ -164,55 +136,13 @@ void GridsManager::write2DGrid(const Grid2DObject& grid2D, const MeteoGrids::Par
iohandler.write2DGrid(grid2D, parameter, date);
}
void GridsManager::addToBuffer(const Grid2DObject& in_grid2Dobj, const std::string& grid_hash)
{
if (max_grids==0) return;
if(IndexBufferedGrids.size() >= max_grids) { //we need to remove the oldest grid
mapBufferedGrids.erase( mapBufferedGrids.find( IndexBufferedGrids.front() ) );
IndexBufferedGrids.erase( IndexBufferedGrids.begin() );
}
mapBufferedGrids[ grid_hash ] = in_grid2Dobj;
IndexBufferedGrids.push_back( grid_hash );
}
bool GridsManager::getFromBuffer(const std::string& grid_hash, Grid2DObject& grid) const
{
if (IndexBufferedGrids.empty())
return false;
const std::map<std::string, Grid2DObject>::const_iterator it = mapBufferedGrids.find( grid_hash );
if (it != mapBufferedGrids.end()) { //already in map
grid = (*it).second;
return true;
}
return false;
}
const std::string GridsManager::toString() const {
ostringstream os;
os << "<GridsManager>\n";
os << "Config& cfg = " << hex << &cfg << dec << "\n";
os << "IOHandler& iohandler = " << hex << &iohandler << dec << "\n";
os << "Processing level = " << processing_level << "\n";
os << "Max buffered grids = " << max_grids << "\n";
//cache content
os << "Current buffer content (" << mapBufferedGrids.size() << " grids):\n";
std::map<std::string, Grid2DObject>::const_iterator it1;
for (it1=mapBufferedGrids.begin(); it1 != mapBufferedGrids.end(); ++it1){
os << setw(10) << "Grid " << it1->first << "\n";
}
//dem buffer
os << "Dem buffer content (" << dem_buffer.size() << " grids):\n";
for (size_t ii=0; ii<dem_buffer.size(); ++ii){
std::ostringstream ss;
ss << dem_buffer[ii].llcorner.printLatLon() << " " << dem_buffer[ii].getNx() << "x" << dem_buffer[ii].getNy() << " @" << dem_buffer[ii].cellsize << "m";
os << setw(10) << "Dem " << ss.str() << "\n";
}
os << buffer.toString();
os << "</GridsManager>\n";
return os.str();
}
......
......@@ -19,6 +19,7 @@
#define __GRIDSMANAGER_H__
#include <meteoio/DataGenerator.h>
#include <meteoio/dataClasses/Buffer.h>
#include <meteoio/dataClasses/MeteoData.h>
#include <meteoio/dataClasses/Coords.h>
#include <meteoio/IOHandler.h>
......@@ -51,11 +52,8 @@ class GridsManager {
IOHandler& iohandler;
const Config& cfg;
GridBuffer buffer;
std::map<std::string, Grid2DObject> mapBufferedGrids;
std::vector<DEMObject> dem_buffer;
std::vector<std::string> IndexBufferedGrids; // this is required in order to know which grid is the oldest one
size_t max_grids; ///< How many grids to buffer (grids, dem, landuse and assimilation grids together)
unsigned int processing_level;
};
} //end namespace
......
......@@ -317,8 +317,8 @@ const std::string TimeSeriesManager::toString() const {
os << "Processing level = " << processing_level << "\n";
os << dataGenerator.toString();
os << "RawBuffer:" << raw_buffer.toString();
os << "Filteredcache:" << raw_buffer.toString();
os << "RawBuffer:\n" << raw_buffer.toString();
os << "Filteredcache:\n" << raw_buffer.toString();
//display point_cache
size_t count=0;
......
......@@ -221,4 +221,113 @@ const std::string MeteoBuffer::toString() const
return os.str();
}
/****************************** GridBuffer class ***********************************************/
bool GridBuffer::get(Grid2DObject& grid, const std::string& grid_hash) const
{
if (IndexBufferedGrids.empty())
return false;
const std::map<std::string, Grid2DObject>::const_iterator it = mapBufferedGrids.find( grid_hash );
if (it != mapBufferedGrids.end()) { //already in map
grid = (*it).second;
return true;
}
return false;
}
bool GridBuffer::get(Grid2DObject& grid, const MeteoGrids::Parameters& parameter, const Date& date) const
{
const string grid_hash = date.toString(Date::ISO)+"::"+MeteoGrids::getParameterName(parameter);
return get(grid, grid_hash);
}
bool GridBuffer::get(DEMObject& grid) const
{
if (max_grids==0) return false;
if (!dem_buffer.empty()) {
//HACK: properly manage buffering multiple dems!
//already in buffer. If the update properties have changed,
//we copy the ones given in input and force the update of the object
const DEMObject::update_type in_ppt = (DEMObject::update_type)grid.getUpdatePpt();
const DEMObject::slope_type in_slope_alg = (DEMObject::slope_type)grid.getDefaultAlgorithm();
grid = dem_buffer[0];
const DEMObject::update_type buff_ppt = (DEMObject::update_type)grid.getUpdatePpt();
const DEMObject::slope_type buff_slope_alg = (DEMObject::slope_type)grid.getDefaultAlgorithm();
if (in_ppt!=buff_ppt || in_slope_alg!=buff_slope_alg) {
grid.setDefaultAlgorithm(in_slope_alg);
grid.setUpdatePpt(in_ppt);
grid.update();
}
return true;
}
return false;
}
void GridBuffer::push(const DEMObject& in_grid)
{
//HACK: properly manage buffering multiple dems!
//std::ostringstream grid_hash;
//grid_hash << dem_buffer[ii].llcorner.printLatLon() << " " << dem_buffer[ii].getNx() << "x" << dem_buffer[ii].getNy() << " @" << dem_buffer[ii].cellsize << "m";
if (max_grids==0) return;
if (!dem_buffer.empty())
dem_buffer.clear();
dem_buffer.push_back( in_grid );
}
void GridBuffer::push(const Grid2DObject& in_grid, const std::string& grid_hash)
{
if (max_grids==0) return;
if(IndexBufferedGrids.size() >= max_grids) { //we need to remove the oldest grid
mapBufferedGrids.erase( mapBufferedGrids.find( IndexBufferedGrids.front() ) );
IndexBufferedGrids.erase( IndexBufferedGrids.begin() );
}
mapBufferedGrids[ grid_hash ] = in_grid;
IndexBufferedGrids.push_back( grid_hash );
}
void GridBuffer::push(const Grid2DObject& in_grid, const MeteoGrids::Parameters& parameter, const Date& date)
{
const string grid_hash = date.toString(Date::ISO)+"::"+MeteoGrids::getParameterName(parameter);
push(in_grid, grid_hash);
}
const std::string GridBuffer::toString() const
{
ostringstream os;
os << "<GridBuffer>\n";
os << "Max buffered grids = " << max_grids << "\n";
//cache content
os << "Current buffer content (" << mapBufferedGrids.size() << " grids):\n";
std::map<std::string, Grid2DObject>::const_iterator it1;
for (it1=mapBufferedGrids.begin(); it1 != mapBufferedGrids.end(); ++it1){
os << setw(10) << "Grid " << it1->first << "\n";
}
//dem buffer
if (!dem_buffer.empty()) {
os << "Dem buffer content (" << dem_buffer.size() << " grids):\n";
for (size_t ii=0; ii<dem_buffer.size(); ++ii){
std::ostringstream ss;
ss << dem_buffer[ii].llcorner.printLatLon() << " " << dem_buffer[ii].getNx() << "x" << dem_buffer[ii].getNy() << " @" << dem_buffer[ii].cellsize << "m";
os << setw(10) << "Dem " << ss.str() << "\n";
}
}
os << "</GridBuffer>\n";
return os.str();
}
} //end namespace
\ No newline at end of file
......@@ -18,6 +18,8 @@
#ifndef __BUFFER_H__
#define __BUFFER_H__
#include <meteoio/dataClasses/Grid2DObject.h>
#include <meteoio/dataClasses/DEMObject.h>
#include <meteoio/dataClasses/Date.h>
#include <meteoio/dataClasses/MeteoData.h>
......@@ -112,5 +114,41 @@ class MeteoBuffer {
Date ts_start, ts_end; ///< store the beginning and the end date of the ts_buffer
};
/**
* @class GridBuffer
* @brief A class to buffer gridded data.
* This class buffers Grid2D objects. It implements a proper ring buffer, thus removing old bufered grids
* when necessary.
*
* @ingroup data_str
* @author Mathias Bavay
* @date 2015-02-06
*/
class GridBuffer {
public:
GridBuffer(const size_t& in_max_grids) : dem_buffer(), mapBufferedGrids(), IndexBufferedGrids(), max_grids(in_max_grids) {};
bool empty() const {return IndexBufferedGrids.empty();};
void clear() {mapBufferedGrids.clear(); IndexBufferedGrids.clear();};
size_t size() const {return IndexBufferedGrids.size();};
void setMaxGrids(const size_t& in_max_grids) {max_grids=in_max_grids;};
bool get(Grid2DObject& grid, const std::string& grid_hash) const;
bool get(Grid2DObject& grid, const MeteoGrids::Parameters& parameter, const Date& date) const;
bool get(DEMObject& grid) const;
void push(const Grid2DObject& in_grid, const std::string& grid_hash);
void push(const Grid2DObject& in_grid, const MeteoGrids::Parameters& parameter, const Date& date);
void push(const DEMObject& in_grid);
const std::string toString() const;
private:
std::vector<DEMObject> dem_buffer;
std::map<std::string, Grid2DObject> mapBufferedGrids;
std::vector<std::string> IndexBufferedGrids; // this is required in order to know which grid is the oldest one
size_t max_grids; ///< How many grids to buffer (grids, dem, landuse and assimilation grids together)
};
}
#endif
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment