WSL/SLF GitLab Repository

Commit 42d06b33 authored by Thomas Egger's avatar Thomas Egger
Browse files

Introducing the new IOManager - the user interface to meteoio. This version is...

Introducing the new IOManager - the user interface to meteoio. This version is fully backwards compatible, but does not incorporate all features of the BufferedIOHandler: the grid reading and writing are currently not supported by this IOManager.
parent b803abc2
......@@ -201,7 +201,7 @@ void BufferedIOHandler::getNextMeteoData(const Date& _date, std::vector<MeteoDat
vecMeteo.clear();
std::vector< std::vector<MeteoData> > meteoTmpBuffer;
readMeteoData(_date, (_date-Date(1900,1,2,0,0)), meteoTmpBuffer);
legacy_readMeteoData(_date, (_date-Date(1900,1,2,0,0)), meteoTmpBuffer);
unsigned int emptycounter = 0;
for (unsigned int ii=0; ii<meteoTmpBuffer.size(); ii++){//stations
......@@ -220,7 +220,7 @@ void BufferedIOHandler::bufferAllData(const Date& _date){
Date fromDate = _date - bufferbefore;
Date toDate = _date + bufferafter;
readMeteoData(fromDate, toDate, meteoBuffer);
legacy_readMeteoData(fromDate, toDate, meteoBuffer);
for (unsigned int ii=0; ii<meteoBuffer.size(); ii++){
//set the start and the end date of the interval requested for each station
......@@ -234,7 +234,7 @@ bool BufferedIOHandler::bufferData(const Date& _date, const unsigned int& statio
Date fromDate = _date - bufferbefore;
Date toDate = _date + bufferafter;
readMeteoData(fromDate, toDate, meteoBuffer, stationindex);
legacy_readMeteoData(fromDate, toDate, meteoBuffer, stationindex);
startDateBuffer.at(stationindex) = fromDate;
endDateBuffer.at(stationindex) = toDate;
......@@ -271,17 +271,100 @@ void BufferedIOHandler::setBufferDuration(const Date& _beforeDate, const Date& _
bufferafter = _afterDate;
}
void BufferedIOHandler::readMeteoData(const Date& dateStart, const Date& dateEnd,
std::vector< std::vector<MeteoData> >& vecMeteo,
const unsigned int& stationindex)
void BufferedIOHandler::legacy_readMeteoData(const Date& date_start, const Date& date_end,
std::vector< std::vector<MeteoData> >& vecMeteo,
const unsigned int& stationindex)
{
iohandler.readMeteoData(dateStart, dateEnd, vecMeteo, stationindex);
iohandler.readMeteoData(date_start, date_end, vecMeteo, stationindex);
if (&meteoBuffer != &vecMeteo)
meteoBuffer = vecMeteo; //copy by value
}
void BufferedIOHandler::readMeteoData(const Date& date_start, const Date& date_end,
std::vector< std::vector<MeteoData> >& vecMeteo,
const unsigned int& /*stationindex*/)
{
vecMeteo.clear();
Date default_chunk_size(15.0); //15 days
Date current_buffer_end(date_start + default_chunk_size);
vector< vector<MeteoData> > tmp_meteo_buffer;
//Read MeteoData for requested interval in chunks, furthermore buffer it
//Try to buffer after the requested chunk for subsequent calls
//0. initialize if not already initialized
if (vec_buffer_meteo.size() == 0) //init
bufferAllData(date_start, current_buffer_end);
unsigned int buffer_size = vec_buffer_meteo.size();
//1. Check whether data is in buffer already, and buffer it if not
if ((date_start >= buffer_start) && (date_end <= buffer_end)){
//copy data and we're done
} else {
//rebuffer data
if (current_buffer_end == buffer_end){
//only append
} else {
//rebuffer for real
bufferAllData(date_start, current_buffer_end);
buffer_size = vec_buffer_meteo.size();
}
while (date_end > current_buffer_end){
iohandler.readMeteoData(current_buffer_end, current_buffer_end+default_chunk_size, tmp_meteo_buffer);
if (tmp_meteo_buffer.size() != buffer_size)
throw IOException("God damn it!", AT);
//Loop through stations and append data
for (unsigned int ii=0; ii<buffer_size; ii++){
unsigned int station_size = vec_buffer_meteo[ii].size();
if ((station_size > 0) && (tmp_meteo_buffer[ii].size() > 0)){
//check if the last element equals the first one
if (vec_buffer_meteo[ii][station_size-1].date >= tmp_meteo_buffer[ii][0].date)
vec_buffer_meteo[ii].pop_back(); //delete the element with the same date
}
vec_buffer_meteo[ii].insert(vec_buffer_meteo[ii].end(), tmp_meteo_buffer[ii].begin(), tmp_meteo_buffer[ii].end());
}
current_buffer_end += default_chunk_size;
buffer_end = current_buffer_end;
}
}
//2. Copy appropriate data into vecMeteo
for (unsigned int ii=0; ii<buffer_size; ii++){
vecMeteo.push_back(vector<MeteoData>()); //insert one empty vector of MeteoData
if (vec_buffer_meteo[ii].size() == 0) continue; //no data in buffer for this station
unsigned int pos_start = IOUtils::seek(date_start, vec_buffer_meteo[ii], false);
if (pos_start == IOUtils::npos) pos_start = 0;
unsigned int pos_end = IOUtils::seek(date_end, vec_buffer_meteo[ii], false);//HACK:: edit IOUtils::seek to accept an offset
if (pos_end == IOUtils::npos) pos_end = vec_buffer_meteo[ii].size() - 1; //just copy until the end of the buffer
//cout << "Station " << ii << ": pos_start=" << pos_start << " pos_end=" << pos_end << endl;
if (vec_buffer_meteo[ii][pos_end].date > date_end){
if (pos_end > pos_start) pos_end--;
} else {
pos_end++;
}
//cout << "Station " << ii << ": pos_start=" << pos_start << " pos_end=" << pos_end << endl;
vecMeteo[ii].insert(vecMeteo[ii].begin(), vec_buffer_meteo[ii].begin()+pos_start, vec_buffer_meteo[ii].begin()+pos_end);
}
}
void BufferedIOHandler::bufferAllData(const Date& date_start, const Date& date_end){
iohandler.readMeteoData(date_start, date_end, vec_buffer_meteo);
buffer_start = date_start;
buffer_end = date_end;
}
void BufferedIOHandler::readSpecialPoints(std::vector<Coords>& _cpa)
{
iohandler.readSpecialPoints(_cpa);
......
......@@ -158,9 +158,14 @@ class BufferedIOHandler : public IOInterface {
friend std::ostream& operator<<(std::ostream& os, const BufferedIOHandler& data);
private:
void legacy_readMeteoData(const Date& dateStart, const Date& dateEnd,
std::vector< std::vector<MeteoData> >& vecMeteo,
const unsigned int& stationindex=IOUtils::npos);
bool bufferData(const Date& _date, const unsigned int& stationindex);
void bufferAllData(const Date& _date);
void setDfltBufferProperties();
void bufferAllData(const Date& date_start, const Date& date_end);
IOHandler& iohandler;
Config cfg;
......@@ -168,8 +173,10 @@ class BufferedIOHandler : public IOInterface {
bool always_rebuffer;
Date bufferbefore, bufferafter; //NrOfDays to buffer before and after a given date
Date buffer_start, buffer_end;
std::vector< std::vector<MeteoData> > meteoBuffer;
std::vector< std::vector<MeteoData> > vec_buffer_meteo;
std::vector< Date > startDateBuffer;
std::vector< Date > endDateBuffer;
std::map<std::string, Grid2DObject> mapBufferedGrids;
......
/***********************************************************************************/
/* Copyright 2009 WSL Institute for Snow and Avalanche Research SLF-DAVOS */
/***********************************************************************************/
/* This file is part of MeteoIO.
MeteoIO is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
MeteoIO is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with MeteoIO. If not, see <http://www.gnu.org/licenses/>.
*/
#include <meteoio/IOManager.h>
using namespace std;
namespace mio {
IOManager::IOManager(const Config& i_cfg) : cfg(i_cfg), rawio(i_cfg), bufferedio(rawio, i_cfg), meteoprocessor(i_cfg)
{
processing_level = IOManager::filtered | IOManager::resampled;
}
void IOManager::setProcessingLevel(const unsigned int& i_level)
{
if (i_level >= IOManager::num_of_levels)
throw InvalidArgumentException("The processing level is invalid", AT);
processing_level = i_level;
}
unsigned int IOManager::getStationData(const Date& date, std::vector<StationData>& vecStation)
{
vecStation.clear();
if (processing_level == IOManager::raw){
rawio.readStationData(date, vecStation);
} else {
bufferedio.readStationData(date, vecStation);
}
return vecStation.size();
}
//for an interval of data: decide whether data should be filtered or raw
unsigned int IOManager::getMeteoData(const Date& dateStart, const Date& dateEnd,
std::vector< std::vector<MeteoData> >& vecMeteo)
{
vecMeteo.clear();
if (processing_level == IOManager::raw){
rawio.readMeteoData(dateStart, dateEnd, vecMeteo);
} else {
bufferedio.readMeteoData(dateStart, dateEnd, vecMeteo);
//now it needs to secured that the data is actually filtered, if configured
}
return vecMeteo.size(); //equivalent with the number of stations that have data
}
//data can be raw or processed (filtered, resampled)
unsigned int IOManager::getMeteoData(const Date& i_date, std::vector<MeteoData>& vecMeteo)
{
vecMeteo.clear();
vector< vector<MeteoData> > vec_cache;
Date time_before, time_after;
unsigned int npoints;
meteoprocessor.getWindowSize(time_before, time_after, npoints);
//1. Check whether user wants raw data or processed data
if (processing_level == IOManager::raw){
rawio.readMeteoData(i_date-Date(0.001), i_date+Date(0.001), vec_cache);
for (unsigned int ii=0; ii<vec_cache.size(); ii++){
unsigned int index = IOUtils::seek(i_date, vec_cache[ii], true);
if (index != IOUtils::npos)
vecMeteo.push_back(vec_cache[ii][index]); //Insert station into vecMeteo
}
return vecMeteo.size();
}
//2. Check which data is available, buffered locally
// request an appropriate window of data from bufferedio
// Hand window of data over to meteo processor
bufferedio.readMeteoData(i_date-time_before, i_date+time_after, vec_cache);
for (unsigned int ii=0; ii<vec_cache.size(); ii++){
MeteoData tmpmd;
meteoprocessor.processData(i_date, vec_cache[ii], tmpmd);
vecMeteo.push_back(tmpmd);
}
return vecMeteo.size();
}
void IOManager::writeMeteoData(const std::vector< std::vector<MeteoData> >& vecMeteo, const std::string& name)
{
if (processing_level == IOManager::raw){
rawio.writeMeteoData(vecMeteo, name);
} else {
bufferedio.writeMeteoData(vecMeteo, name);
}
}
void IOManager::interpolate(const Date& date, const MeteoData::Parameters& meteoparam, Grid2DObject& result)
{
string info_string;
interpolate(date, meteoparam, result, info_string);
}
void IOManager::interpolate(const Date& date, const MeteoData::Parameters& meteoparam,
Grid2DObject& result, std::string& info_string)
{
DEMObject dem;
if (processing_level == IOManager::raw){
rawio.readDEM(dem);
} else {
bufferedio.readDEM(dem);
}
vector<MeteoData> vec_meteo;
getMeteoData(date, vec_meteo);
Meteo2DInterpolator mi(cfg, dem, vec_meteo);
mi.interpolate(meteoparam, result, info_string);
}
} //namespace
/***********************************************************************************/
/* Copyright 2009 WSL Institute for Snow and Avalanche Research SLF-DAVOS */
/***********************************************************************************/
/* This file is part of MeteoIO.
MeteoIO is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
MeteoIO is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with MeteoIO. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __IOMANAGER_H__
#define __IOMANAGER_H__
#include <meteoio/BufferedIOHandler.h>
#include <meteoio/Meteo2DInterpolator.h>
namespace mio {
class IOManager {
public:
enum ProcessingLevel {
raw = 0,
filtered = 1 << 0,
resampled = 1 << 1,
num_of_levels = 1 << 2
};
IOManager(const Config& i_cfg);
unsigned int getStationData(const Date& date, std::vector<StationData>& vecStation);
//for an intervall of data: decide whether data should be filtered or raw
unsigned int getMeteoData(const Date& dateStart, const Date& dateEnd,
std::vector< std::vector<MeteoData> >& vecMeteo);
//data can be raw or processed (filtered, resampled)
unsigned int getMeteoData(const Date& i_date, std::vector<MeteoData>& vecMeteo);
void interpolate(const Date& date, const MeteoData::Parameters& meteoparam,
Grid2DObject& result, std::string& info_string);
void interpolate(const Date& date, const MeteoData::Parameters& meteoparam, Grid2DObject& result);
void setProcessingLevel(const unsigned int& i_level);
void writeMeteoData(const std::vector< std::vector<MeteoData> >& vecMeteo, const std::string& name="");
private:
Config cfg;
IOHandler rawio;
BufferedIOHandler bufferedio;
MeteoProcessor meteoprocessor;
unsigned int processing_level;
};
} //end namespace
#endif
......@@ -48,6 +48,7 @@
#include <meteoio/MeteoProcessor.h>
#include <meteoio/ResamplingAlgorithms.h>
#include <meteoio/StationData.h>
#include <meteoio/IOManager.h>
#ifdef _POPC_
#include <meteoio/IOHandler.ph>
......
......@@ -25,6 +25,14 @@ const unsigned int MeteoProcessor::window_half_size = 40; //org: 4
MeteoProcessor::MeteoProcessor(const Config& cfg) : mf(cfg), mi1d(cfg) {}
void MeteoProcessor::getWindowSize(Date& time_before, Date& time_after, unsigned int& num_of_points)
{
//Plan: parse through io.ini and establish the maximal window size needed
time_before.setDate(1.0); //a day
time_after.setDate(1.0); //a day
num_of_points = 0;
}
void MeteoProcessor::processData(const Date& date, const std::vector<MeteoData>& vecM, MeteoData& md)
{
unsigned int currentpos = IOUtils::seek(date, vecM, false);
......
......@@ -47,6 +47,8 @@ class MeteoProcessor {
*/
void processData(const Date& date, const std::vector<MeteoData>& vecM, MeteoData& md);
void getWindowSize(Date& dateStart, Date& dateEnd, unsigned int& num_of_points);
friend std::ostream& operator<<(std::ostream& os, const MeteoProcessor& data);
private:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment