WSL/SLF GitLab Repository

Commit e9cfeddb authored by Mathias Bavay's avatar Mathias Bavay
Browse files

Removing an unnecessary vector copying in BufferedIOHandler. A new private...

Removing an unnecessary vector copying in BufferedIOHandler. A new private method has been added that IOManager can call.
parent 0d204d8f
......@@ -236,7 +236,13 @@ double BufferedIOHandler::getAvgSamplingRate() const
return IOUtils::nodata;
}
const std::vector< METEO_SET >& BufferedIOHandler::get_complete_buffer(Date& start, Date& end)
/**
* @brief return all the buffered data as well as the start and end dates
* @param start start date of the buffer
* @param end end date of the buffer
* @return complete buffer
*/
const std::vector< METEO_SET >& BufferedIOHandler::getFullBuffer(Date& start, Date& end)
{
start = buffer_start;
end = buffer_end;
......@@ -244,11 +250,44 @@ const std::vector< METEO_SET >& BufferedIOHandler::get_complete_buffer(Date& sta
return vec_buffer_meteo; //return reference
}
/**
* @brief return all the buffered data between the given dates
* @param date_start requested start date of the buffer
* @param date_end requested end date of the buffer
* @param data vector to fill with the buffered data
*/
void BufferedIOHandler::getBufferInterval(const Date& date_start, const Date& date_end, std::vector< METEO_SET > &vecMeteo)
{
//1. Prepare the output vector
const size_t buffer_size = vec_buffer_meteo.size();
vecMeteo.clear();
vecMeteo.reserve(buffer_size);
//2. Copy appropriate data into vecMeteo for each station
for (size_t ii=0; ii<buffer_size; ii++){ //loop through stations
vecMeteo.push_back(vector<MeteoData>()); //insert one empty vector of MeteoData
if (vec_buffer_meteo[ii].empty()) continue; //no data in buffer for this station
size_t pos_start = IOUtils::seek(date_start, vec_buffer_meteo[ii], false);
if (pos_start == IOUtils::npos) pos_start = 0;
size_t pos_end = IOUtils::seek(date_end, vec_buffer_meteo[ii], false);//HACK:: edit IOUtils::seek to accept an offset
if (pos_end == IOUtils::npos) pos_end = vec_buffer_meteo[ii].size() - 1; //just copy until the end of the buffer
if (vec_buffer_meteo[ii][pos_end].date > date_end){
if (pos_end > pos_start) pos_end--;
} else {
pos_end++;
}
vecMeteo[ii].reserve(pos_end-pos_start+1); //weird that the "insert" does not handle it internally...
vecMeteo[ii].insert(vecMeteo[ii].begin(), vec_buffer_meteo[ii].begin()+pos_start, vec_buffer_meteo[ii].begin()+pos_end);
}
}
void BufferedIOHandler::readMeteoData(const Date& date_start, const Date& date_end,
std::vector< METEO_SET >& vecMeteo,
const size_t& /*stationindex*/)
{
vecMeteo.clear();
const Date new_buffer_start(date_start-buff_before); //taking centering into account
Date new_buffer_end(new_buffer_start + chunk_size);
vector< vector<MeteoData> > tmp_meteo_buffer; //it must be here -> adresses copied in 2. are still valid
......@@ -298,28 +337,14 @@ void BufferedIOHandler::readMeteoData(const Date& date_start, const Date& date_e
buffer_end = new_buffer_end;
}
}
}
//2. Copy appropriate data into vecMeteo
vecMeteo.reserve(buffer_size);
for (size_t ii=0; ii<buffer_size; ii++){ //loop through stations
vecMeteo.push_back(vector<MeteoData>()); //insert one empty vector of MeteoData
if (vec_buffer_meteo[ii].empty()) continue; //no data in buffer for this station
size_t pos_start = IOUtils::seek(date_start, vec_buffer_meteo[ii], false);
if (pos_start == IOUtils::npos) pos_start = 0;
size_t pos_end = IOUtils::seek(date_end, vec_buffer_meteo[ii], false);//HACK:: edit IOUtils::seek to accept an offset
if (pos_end == IOUtils::npos) pos_end = vec_buffer_meteo[ii].size() - 1; //just copy until the end of the buffer
if (vec_buffer_meteo[ii][pos_end].date > date_end){
if (pos_end > pos_start) pos_end--;
} else {
pos_end++;
}
vecMeteo[ii].reserve(pos_end-pos_start+1); //weird that the "insert" does not handle it internally...
vecMeteo[ii].insert(vecMeteo[ii].begin(), vec_buffer_meteo[ii].begin()+pos_start, vec_buffer_meteo[ii].begin()+pos_end);
}
void BufferedIOHandler::readMeteoData(const Date& date_start, const Date& date_end,
std::vector< METEO_SET >& vecMeteo,
const size_t& stationindex)
{
readMeteoData(date_start, date_end, stationindex);
getBufferInterval(date_start, date_end, vecMeteo);
}
void BufferedIOHandler::bufferData(const Date& date_start, const Date& date_end, std::vector< METEO_SET >& vecvecMeteo){
......
......@@ -142,7 +142,12 @@ class BufferedIOHandler : public IOInterface {
private:
//private methods
const std::vector<METEO_SET>& get_complete_buffer(Date& start, Date& end);
void readMeteoData(const Date& dateStart, const Date& dateEnd,
const size_t& stationindex=IOUtils::npos);
const std::vector<METEO_SET>& getFullBuffer(Date& start, Date& end);
void getBufferInterval(const Date& date_start, const Date& date_end, std::vector< METEO_SET > &vecMeteo);
void push_meteo_data(const Date& date_start, const Date& date_end,
const std::vector< METEO_SET >& vecMeteo);
......
......@@ -142,7 +142,7 @@ void IOManager::fill_filtered_cache()
{
if ((IOManager::filtered & processing_level) == IOManager::filtered){
//ask the bufferediohandler for the whole buffer
const vector< METEO_SET >& buffer = bufferedio.get_complete_buffer(fcache_start, fcache_end);
const vector< METEO_SET >& buffer = bufferedio.getFullBuffer(fcache_start, fcache_end);
meteoprocessor.process(buffer, filtered_cache);
}
}
......@@ -197,8 +197,7 @@ void IOManager::add_to_cache(const Date& i_date, const METEO_SET& vecMeteo)
point_cache[i_date] = vecMeteo;
}
//data can be raw or processed (filtered, resampled)
size_t IOManager::getMeteoData(const Date& i_date, METEO_SET& vecMeteo)
size_t IOManager::getTrueMeteoData(const Date& i_date, METEO_SET& vecMeteo)
{
vecMeteo.clear();
vector< vector<MeteoData> > vec_cache;
......@@ -230,7 +229,7 @@ size_t IOManager::getMeteoData(const Date& i_date, METEO_SET& vecMeteo)
const bool cached = (fcache_start <= i_date-proc_properties.time_before) && (fcache_end >= i_date+proc_properties.time_after);
if (!cached) {
//explicit caching, this forces the bufferediohandler to rebuffer, if necessary
bufferedio.readMeteoData(i_date-proc_properties.time_before, i_date+proc_properties.time_after, vec_cache); //HACK: vec_cache is not used
bufferedio.readMeteoData(i_date-proc_properties.time_before, i_date+proc_properties.time_after);
fill_filtered_cache();
}
data = &filtered_cache;
......@@ -257,6 +256,16 @@ size_t IOManager::getMeteoData(const Date& i_date, METEO_SET& vecMeteo)
dataGenerator.fillMissing(vecMeteo);
}
return vecMeteo.size();
}
//data can be raw or processed (filtered, resampled)
size_t IOManager::getMeteoData(const Date& i_date, METEO_SET& vecMeteo)
{
vecMeteo.clear();
getTrueMeteoData(i_date, vecMeteo);
//Store result in the local cache
add_to_cache(i_date, vecMeteo);
return vecMeteo.size();
......
......@@ -241,6 +241,7 @@ class IOManager {
void fill_filtered_cache();
bool read_filtered_cache(const Date& start_date, const Date& end_date,
std::vector< METEO_SET >& vec_meteo);
size_t getTrueMeteoData(const Date& i_date, METEO_SET& vecMeteo);
const Config cfg; ///< we keep this Config object as full copy, so the original one can get out of scope/be destroyed
IOHandler rawio;
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment