WSL/SLF GitLab Repository

Commit 8a762fbc authored by Mathias Bavay's avatar Mathias Bavay
Browse files

The method to clear the buffers in BufferedIOHandler was not doing everything...

The method to clear the buffers in BufferedIOHandler was not doing everything it should. Some methods in BufferedIOHandler and IOManager have been made const. The issues 117 and 272 have been adressed: the IOHandler now checks that timestamps are unique and in increasing order. This check has therefore been removed from SMETIO.
parent 377d6e57
......@@ -24,18 +24,10 @@ namespace mio {
BufferedIOHandler::BufferedIOHandler(IOHandler& in_iohandler, const Config& in_cfg)
: iohandler(in_iohandler), cfg(in_cfg), vec_buffer_meteo(), mapBufferedGrids(), IndexBufferedGrids(),
buffer_start(), buffer_end(), chunk_size(), buff_before(), max_grids(10)
{
setDfltBufferProperties();
}
#ifdef _POPC_
BufferedIOHandler::~BufferedIOHandler()
#else
BufferedIOHandler::~BufferedIOHandler() throw()
#endif
{}
BufferedIOHandler& BufferedIOHandler::operator=(const BufferedIOHandler& source) {
if(this != &source) {
iohandler = source.iohandler;
......@@ -221,7 +213,7 @@ void BufferedIOHandler::setMinBufferRequirements(const double& i_chunk_size, con
if(buff_before>chunk_size) chunk_size = buff_before;
}
double BufferedIOHandler::getAvgSamplingRate()
double BufferedIOHandler::getAvgSamplingRate() const
{
if (!vec_buffer_meteo.empty()){
const size_t nr_stations = vec_buffer_meteo.size();
......@@ -372,7 +364,11 @@ void BufferedIOHandler::write2DGrid(const Grid2DObject& grid_in, const MeteoGrid
iohandler.write2DGrid(grid_in, parameter, date);
}
void BufferedIOHandler::clearBuffer(){
void BufferedIOHandler::clearBuffer() {
IndexBufferedGrids.clear();
vec_buffer_meteo.clear();
buffer_start = 0.;
buffer_end = 0.;
mapBufferedGrids.clear();
}
......
......@@ -71,9 +71,9 @@ class BufferedIOHandler : public IOInterface {
*/
BufferedIOHandler(IOHandler& in_iohandler, const Config& in_cfg);
#ifdef _POPC_
virtual ~BufferedIOHandler();
virtual ~BufferedIOHandler() {};
#else
virtual ~BufferedIOHandler() throw();
virtual ~BufferedIOHandler() throw() {};
#endif
BufferedIOHandler& operator=(const BufferedIOHandler&); ///<Assignement operator
......@@ -124,7 +124,7 @@ class BufferedIOHandler : public IOInterface {
* it would return 2 measurements per hour.
* @return average sampling rate in Hz, nodata if the buffer is empty
*/
double getAvgSamplingRate();
double getAvgSamplingRate() const;
const std::string toString() const;
......
......@@ -281,19 +281,29 @@ void IOHandler::readMeteoData(const Date& date, METEO_SET& vecMeteo)
readMeteoData(date, date, meteoTmpBuffer);
vecMeteo.clear();
vecMeteo.reserve(meteoTmpBuffer.size());
vecMeteo.reserve( meteoTmpBuffer.size() );
size_t emptycounter = 0;
for (size_t ii=0; ii<meteoTmpBuffer.size(); ii++){//stations
if (!meteoTmpBuffer[ii].empty()){
vecMeteo.push_back(meteoTmpBuffer[ii].front());
} else {
emptycounter++;
}
for (size_t ii=0; ii<meteoTmpBuffer.size(); ii++) {//stations
if (!meteoTmpBuffer[ii].empty())
vecMeteo.push_back( meteoTmpBuffer[ii].front() );
}
}
if (emptycounter == meteoTmpBuffer.size()){
vecMeteo.clear();
void IOHandler::checkTimestamps(const std::vector<METEO_SET>& vecVecMeteo) const
{
for(size_t stat_idx=0; stat_idx<vecVecMeteo.size(); stat_idx++) {
const size_t nr_timestamps = vecVecMeteo[stat_idx].size();
if(nr_timestamps==0) continue;
Date previous_date( vecVecMeteo[stat_idx][0].date );
for(size_t ii=1; ii<nr_timestamps; ii++) {
const StationData& station = vecVecMeteo[stat_idx][ii].meta;
const Date& current_date = vecVecMeteo[stat_idx][ii].date;
if(current_date<=previous_date) {
throw IOException("Error at time "+current_date.toString(Date::ISO)+" for station \""+station.stationName+"\" ("+station.stationID+") : timestamps must be in increasing order and unique!", AT);
}
previous_date = current_date;
}
}
}
......@@ -303,6 +313,7 @@ void IOHandler::readMeteoData(const Date& dateStart, const Date& dateEnd,
{
IOInterface *plugin = getPlugin("METEO", "Input");
plugin->readMeteoData(dateStart, dateEnd, vecMeteo, stationindex);
checkTimestamps(vecMeteo);
copy_parameters(stationindex, vecMeteo);
}
......
......@@ -83,6 +83,7 @@ class IOHandler : public IOInterface {
void registerPlugins();
IOInterface *getPlugin(const std::string& cfgkey, const std::string& cfgsection="GENERAL");
void parse_copy_config();
void checkTimestamps(const std::vector<METEO_SET>& vecVecMeteo) const;
void copy_parameters(const size_t& stationindex, std::vector< METEO_SET >& vecMeteo) const;
const Config& cfg;
......
......@@ -28,7 +28,6 @@ IOManager::IOManager(const Config& i_cfg) : cfg(i_cfg), rawio(cfg), bufferedio(r
fcache_start(Date(0.0, 0.)), fcache_end(Date(0.0, 0.)), //this should not matter, since 0 is still way back before any real data...
processing_level(IOManager::filtered | IOManager::resampled | IOManager::generated)
{
//setProcessingLevel(IOManager::filtered | IOManager::resampled);
meteoprocessor.getWindowSize(proc_properties);
interpolator.setIOManager(*this); //because "*this" does not necessarily exist in the initialization list...
}
......@@ -49,7 +48,7 @@ void IOManager::setMinBufferRequirements(const double& buffer_size, const double
bufferedio.setMinBufferRequirements(buffer_size, buff_before);
}
double IOManager::getAvgSamplingRate()
double IOManager::getAvgSamplingRate() const
{
if (processing_level == IOManager::raw){
return IOUtils::nodata;
......@@ -101,7 +100,6 @@ size_t IOManager::getStationData(const Date& date, STATIONS_SET& vecStation)
return vecStation.size();
}
//for an interval of data: decide whether data should be filtered or raw
size_t IOManager::getMeteoData(const Date& dateStart, const Date& dateEnd, std::vector< METEO_SET >& vecVecMeteo)
{
......
......@@ -219,7 +219,7 @@ class IOManager {
* it would return 2 measurements per hour.
* @return average sampling rate in Hz, nodata if the buffer is empty
*/
double getAvgSamplingRate();
double getAvgSamplingRate() const;
#ifdef _POPC_ //HACK popc
void writeMeteoData(/*const*/ std::vector< METEO_SET >& vecMeteo, /*const*/ std::string& name/*=""*/);
......
......@@ -324,7 +324,7 @@ void SMETIO::read_meta_data(const smet::SMETReader& myreader, StationData& meta)
}
}
void SMETIO::copy_data(const std::string& filename, const smet::SMETReader& myreader,
void SMETIO::copy_data(const smet::SMETReader& myreader,
const std::vector<std::string>& timestamps,
const std::vector<double>& mydata, std::vector<MeteoData>& vecMeteo)
{
......@@ -359,7 +359,6 @@ void SMETIO::copy_data(const std::string& filename, const smet::SMETReader& myre
const size_t nr_of_fields = indexes.size();
const size_t nr_of_lines = mydata.size() / nr_of_fields;
Date previous_date(0., current_timezone);
double lat=IOUtils::nodata, lon=IOUtils::nodata, east=IOUtils::nodata, north=IOUtils::nodata, alt=IOUtils::nodata;
size_t current_index = 0; //index to vec_data
for (size_t ii = 0; ii<nr_of_lines; ii++){
......@@ -404,9 +403,6 @@ void SMETIO::copy_data(const std::string& filename, const smet::SMETReader& myre
current_index++;
}
if(tmp_md.date<=previous_date)
throw InvalidFormatException("Error at time "+tmp_md.date.toString(Date::ISO)+" in SMET file \""+filename+"\" : timestamps must be in increasing order and unique!", AT);
previous_date=tmp_md.date;
if ((olwr_present) && (tmp_md(MeteoData::TSS) == IOUtils::nodata)) {//HACK
tmp_md(MeteoData::TSS) = olwr_to_tss(tmp_md("OLWR"));
......@@ -462,7 +458,7 @@ void SMETIO::readMeteoData(const Date& dateStart, const Date& dateEnd,
myreader.read(dateStart.getJulian(), dateEnd.getJulian(), mydata);
}
copy_data(filename, myreader, mytimestamps, mydata, vecMeteo[ii]);
copy_data(myreader, mytimestamps, mydata, vecMeteo[ii]);
}
}
......
......@@ -64,7 +64,7 @@ class SMETIO : public IOInterface {
void read_meta_data(const smet::SMETReader& myreader, StationData& meta);
void identify_fields(const std::vector<std::string>& fields, std::vector<size_t>& indexes,
bool& julian_present, MeteoData& md);
void copy_data(const std::string& filename, const smet::SMETReader& myreader, const std::vector<std::string>& timestamps,
void copy_data(const smet::SMETReader& myreader, const std::vector<std::string>& timestamps,
const std::vector<double>& mydata, std::vector<MeteoData>& vecMeteo);
void parseInputOutputSection();
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment