WSL/SLF GitLab Repository

Commit 76c7e647 authored by Mathias Bavay's avatar Mathias Bavay
Browse files

Add a setMinBufferRequirements() method to IOManager and BufferedIOHandler....

Add a setMinBufferRequirements() method to IOManager and BufferedIOHandler. This allows an application to specify its buffering needs. For example, when SNOWPACK runs grass detection, it needs the data up to 48 hours before the first time step to compute some averages. Using this call, SNOWPACK will request at least 48 hours of data in pre-buffering. if buff_before is set to something larger, it will be kept, but if it is set to something smaller, it will be increased.
parent fa9d7741
......@@ -161,10 +161,30 @@ void BufferedIOHandler::setDfltBufferProperties()
}
}
//if buff_before>chunk_size, we will have a problem (ie: we won't ever read the whole data we need)
if(buff_before>chunk_size) chunk_size = buff_before;
//BUG: if we do this, we still have the meteo1d window in the way
//-> we end up not reading enough data and rebuffering...
max_grids = 10; //default number of grids to keep in buffer
cfg.getValue("BUFF_GRIDS", "General", max_grids, Config::nothrow);
}
void BufferedIOHandler::setMinBufferRequirements(const double& i_chunk_size, const double& i_buff_before)
{
if(i_buff_before!=IOUtils::nodata) {
const Duration app_buff_before(i_buff_before, 0);
if(app_buff_before>buff_before) buff_before = app_buff_before;
}
if(i_chunk_size!=IOUtils::nodata) {
const Duration app_chunk_size(i_chunk_size, 0);
if(app_chunk_size>chunk_size) chunk_size = app_chunk_size;
}
//if buff_before>chunk_size, we will have a problem (ie: we won't ever read the whole data we need)
if(buff_before>chunk_size) chunk_size = buff_before;
}
double BufferedIOHandler::getAvgSamplingRate()
{
if (vec_buffer_meteo.size() > 0){
......@@ -311,7 +331,8 @@ std::ostream& operator<<(std::ostream& os, const BufferedIOHandler& data)
os << "Config& cfg = " << hex << &data.cfg << dec << "\n";
os << "IOHandler &iohandler = " << hex << &data.iohandler << dec << "\n";
os << "Buffering " << data.chunks << " chunk(s) of " <<data.chunk_size.getJulianDate() << " days\n";
os << "Buffering " << data.chunks << " chunk(s) of " <<data.chunk_size.getJulianDate() << " day(s) with "
<< data.buff_before.getJulianDate() << " day(s) pre-buffering\n";
os << "Current buffer content (" << data.vec_buffer_meteo.size() << " stations, "
<< data.mapBufferedGrids.size() << " grids):\n";
......
......@@ -118,6 +118,16 @@ class BufferedIOHandler : public IOInterface {
friend class IOManager;
/**
* @brief Set buffer window properties requirements as known to the application itself.
* This will compare these requirements with the ones expressed by the end user and keep the max between them.
* The method can be called several times, it will NOT reset the calculated buffer's requirements but keep
* on merging with new submissions. Any parameter given as IOUtils::nodata will be ignored.
* @param i_chunk_size buffer size in days
* @param i_buff_before buffer centering in days
*/
void setMinBufferRequirements(const double& i_chunk_size, const double& i_buff_before);
private:
//private methods
const std::vector<METEO_TIMESERIE>& get_complete_buffer(Date& start, Date& end);
......
......@@ -41,6 +41,10 @@ void IOManager::setProcessingLevel(const unsigned int& i_level)
processing_level = i_level;
}
void IOManager::setMinBufferRequirements(const double& buffer_size, const double& buff_before) {
bufferedio.setMinBufferRequirements(buffer_size, buff_before);
}
double IOManager::getAvgSamplingRate()
{
if (processing_level == IOManager::raw){
......@@ -191,7 +195,7 @@ size_t IOManager::getMeteoData(const Date& i_date, METEO_TIMESERIE& vecMeteo)
//1. Check whether user wants raw data or processed data
if (processing_level == IOManager::raw){
rawio.readMeteoData(i_date-Duration(0.001, 0.), i_date+Duration(0.001, 0.), vec_cache);
rawio.readMeteoData(i_date-Duration(1./(24.*3600.), 0.), i_date+Duration(1./(24.*3600.), 0.), vec_cache);
for (size_t ii=0; ii<vec_cache.size(); ii++){
const size_t index = IOUtils::seek(i_date, vec_cache[ii], true);
if (index != IOUtils::npos)
......
......@@ -53,12 +53,12 @@ class IOManager {
* corresponding to the interval indicated by dateStart and dateEnd.
* Depending on the ProcessingLevel for the instance of the IOManager
* the data returned will be either raw (read directly from the IOHandler)
* or processed (read from an BufferedIOHandler and filtered through the
* or processed (read from an BufferedIOHandler and filtered through the
* MeteoProcessor
*
* vecMeteo will be empty if no datasets were retrieved in the interval defined
* by dateStart and dateEnd
*
*
* Example Usage:
* @code
* vector< vector<MeteoData> > vecMeteo; //empty vector
......@@ -78,8 +78,8 @@ class IOManager {
/**
* @brief Fill vector<MeteoData> object with multiple instances of MeteoData
* corresponding to the instant indicated by a Date object. Each MeteoData
* instance within the vector represents the data for one station at the given
* instant. Depending on the ProcessingLevel configured data will be either
* instance within the vector represents the data for one station at the given
* instant. Depending on the ProcessingLevel configured data will be either
* raw (read directly from the IOHandler)
*
* NOTE:
......@@ -101,21 +101,21 @@ class IOManager {
* @brief Push a vector of time series of MeteoData objects into the IOManager. This overwrites
* any internal buffers that are used and subsequent calls to getMeteoData or interpolate
* will be performed upon this data. This method is a way to bypass the internal reading
* of MeteoData from a certain source and is useful in case the user is only interested
* in data processing and interpolation performed by the IOManager object.
* of MeteoData from a certain source and is useful in case the user is only interested
* in data processing and interpolation performed by the IOManager object.
* @param level Level of processing that has already been performed on the data (raw XOR filtered)
* @param date_start Representing the beginning of the data
* @param date_end Representing the end of the data
* @param vecMeteo The actual data being pushed into the IOManager object
*/
void push_meteo_data(const ProcessingLevel& level, const Date& date_start, const Date& date_end,
void push_meteo_data(const ProcessingLevel& level, const Date& date_start, const Date& date_end,
const std::vector< METEO_TIMESERIE >& vecMeteo);
#ifdef _POPC_ //HACK popc
void interpolate(/*const*/ Date& date, /*const*/ DEMObject& dem, /*const*/ MeteoData::Parameters meteoparam,
Grid2DObject& result);
#else
void interpolate(const Date& date, const DEMObject& dem, const MeteoData::Parameters& meteoparam,
void interpolate(const Date& date, const DEMObject& dem, const MeteoData::Parameters& meteoparam,
Grid2DObject& result);
#endif
......@@ -123,7 +123,7 @@ class IOManager {
void interpolate(/*const*/ Date& date, /*const*/ DEMObject& dem, /*const*/ MeteoData::Parameters meteoparam,
Grid2DObject& result, std::string& info_string);
#else
void interpolate(const Date& date, const DEMObject& dem, const MeteoData::Parameters& meteoparam,
void interpolate(const Date& date, const DEMObject& dem, const MeteoData::Parameters& meteoparam,
Grid2DObject& result, std::string& info_string);
#endif
......@@ -141,6 +141,16 @@ class IOManager {
*/
void setProcessingLevel(const unsigned int& i_level);
/**
* @brief Set buffer window properties requirements as known to the application itself.
* This will compare these requirements with the ones expressed by the end user and keep the max between them.
* The method can be called several times, it will NOT reset the calculated buffer's requirements but keep
* on merging with new submissions. Any parameter given as IOUtils::nodata will be ignored.
* @param buffer_size buffer size in days
* @param buff_before buffer centering in days
*/
void setMinBufferRequirements(const double& buffer_size, const double& buff_before);
double getAvgSamplingRate();
#ifdef _POPC_ //HACK popc
......@@ -162,7 +172,7 @@ class IOManager {
IOHandler rawio;
BufferedIOHandler bufferedio;
MeteoProcessor meteoprocessor;
ProcessingProperties proc_properties;
ProcessingProperties proc_properties; ///< buffer constraints in order to be able to compute the requested values
std::map<Date, METEO_TIMESERIE > resampled_cache; ///< stores already resampled data points
std::vector< METEO_TIMESERIE > filtered_cache; ///< stores already filtered data intervals
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment