WSL/SLF GitLab Repository

BufferedIOHandler.cc 11.9 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
/***********************************************************************************/
/*  Copyright 2009 WSL Institute for Snow and Avalanche Research    SLF-DAVOS      */
/***********************************************************************************/
/* This file is part of MeteoIO.
    MeteoIO is free software: you can redistribute it and/or modify
    it under the terms of the GNU Lesser General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    MeteoIO is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public License
    along with MeteoIO.  If not, see <http://www.gnu.org/licenses/>.
*/
#include <meteoio/BufferedIOHandler.h>

using namespace std;

namespace mio {

#ifdef _POPC_
25
26
BufferedIOHandler::BufferedIOHandler(IOHandler& in_iohandler, const Config& in_cfg) 
	: iohandler(in_iohandler), cfg(in_cfg), mapBufferedGrids()
27
#else
28
29
BufferedIOHandler::BufferedIOHandler(IOHandler& in_iohandler, const Config& in_cfg) 
	  : IOInterface(NULL), iohandler(in_iohandler), cfg(in_cfg), mapBufferedGrids()
30
31
#endif
{
32
	setDfltBufferProperties();
33
34
35
36
37
38
39
}

#ifdef _POPC_
BufferedIOHandler::~BufferedIOHandler()
#else
BufferedIOHandler::~BufferedIOHandler() throw()
#endif
40
{}
41

42
void BufferedIOHandler::read2DGrid(Grid2DObject& in_grid2Dobj, const std::string& in_filename)
43
{
44
	std::map<std::string, Grid2DObject>::iterator it = mapBufferedGrids.find(in_filename);
45
	if (it != mapBufferedGrids.end()) { //already in map
46
		in_grid2Dobj = (*it).second; 
47
48
49
50
		return;
	}
	
	Grid2DObject tmpgrid2D;
51
52
53
	iohandler.read2DGrid(tmpgrid2D, in_filename);
	mapBufferedGrids[in_filename] = tmpgrid2D;
	in_grid2Dobj = tmpgrid2D;
54
55
}

56
void BufferedIOHandler::readDEM(DEMObject& in_grid2Dobj)
57
58
{
	std::map<std::string, Grid2DObject>::iterator it = mapBufferedGrids.find("/:DEM");
59
60
61
	if (it != mapBufferedGrids.end()) {
		//already in map. If the update properties have changed,
		//we copy the ones given in input and force the update of the object
62
63
64
		const DEMObject::update_type in_ppt = (DEMObject::update_type)in_grid2Dobj.getUpdatePpt();
		in_grid2Dobj = (*it).second;
		const DEMObject::update_type buff_ppt = (DEMObject::update_type)in_grid2Dobj.getUpdatePpt();
65
		if(in_ppt!=buff_ppt) {
66
67
			in_grid2Dobj.setUpdatePpt(in_ppt);
			in_grid2Dobj.update();
68
		}
69
70
71
72
		return;
	}
	
	DEMObject tmpgrid2D;
73
	 //copy the updating policy of the destination
74
	tmpgrid2D.setUpdatePpt((DEMObject::update_type)in_grid2Dobj.getUpdatePpt());
75
76
	iohandler.readDEM(tmpgrid2D);
	mapBufferedGrids["/:DEM"] = tmpgrid2D;
77
	in_grid2Dobj = tmpgrid2D;
78
79
}

80
void BufferedIOHandler::readLanduse(Grid2DObject& in_grid2Dobj)
81
82
83
{
	std::map<std::string, Grid2DObject>::iterator it = mapBufferedGrids.find("/:LANDUSE");
	if (it != mapBufferedGrids.end()) { //already in map
84
		in_grid2Dobj = (*it).second; 
85
86
87
88
89
90
		return;
	}
	
	Grid2DObject tmpgrid2D;
	iohandler.readLanduse(tmpgrid2D);
	mapBufferedGrids["/:LANDUSE"] = tmpgrid2D;
91
	in_grid2Dobj = tmpgrid2D;
92
93
}

94
void BufferedIOHandler::readAssimilationData(const Date& in_date, Grid2DObject& in_grid2Dobj)
95
{
96
	std::map<std::string, Grid2DObject>::iterator it = mapBufferedGrids.find("/:ASSIMILATIONDATA" + in_date.toString(Date::FULL));
97
	if (it != mapBufferedGrids.end()) { //already in map
98
		in_grid2Dobj = (*it).second; 
99
100
101
102
		return;
	}
	
	Grid2DObject tmpgrid2D;
103
104
105
	iohandler.readAssimilationData(in_date, tmpgrid2D);
	mapBufferedGrids["/:ASSIMILATIONDATA" + in_date.toString(Date::FULL)] = tmpgrid2D;
	in_grid2Dobj = tmpgrid2D;
106
107
}

108
void BufferedIOHandler::readStationData(const Date& date, STATION_TIMESERIE& vecStation)
109
110
111
112
113
{
	iohandler.readStationData(date, vecStation);
}

#ifdef _POPC_
114
void BufferedIOHandler::writeMeteoData(std::vector< METEO_TIMESERIE >& vecMeteo,
115
                                       const std::string& name)
116
#else 
117
void BufferedIOHandler::writeMeteoData(const std::vector< METEO_TIMESERIE >& vecMeteo,
118
                                       const std::string& name)
119
120
#endif
{
121
	iohandler.writeMeteoData(vecMeteo, name);
122
123
}

124
void BufferedIOHandler::setDfltBufferProperties()
125
{
126
	double chunk_size_days = 15.; //default chunk size value
127
128
129
	chunks = 1;
	cfg.getValue("BUFF_CHUNK_SIZE", "General", chunk_size_days, Config::nothrow); //in days
	cfg.getValue("BUFF_CHUNKS", "General", chunks, Config::nothrow);
130
	chunk_size = Duration(chunk_size_days, 0);
131
132
133
134
135
136

	//get buffer centering options
	double buff_centering = -1.;
	double buff_start = -1.;
	cfg.getValue("BUFF_CENTERING", "General", buff_centering, Config::nothrow);
	cfg.getValue("BUFF_BEFORE", "General", buff_start, Config::nothrow);
137
	if ((buff_centering != -1.) && (buff_start != -1.))
138
139
		throw InvalidArgumentException("Please do NOT provide both BUFF_CENTERING and BUFF_BEFORE!!", AT);

140
	if (buff_start != -1.){
141
142
		buff_before = Duration(buff_start, 0);
	} else {
143
144
		if (buff_centering != -1.){
			if ((buff_centering < 0.) || (buff_centering > 1.))
145
				throw InvalidArgumentException("BUFF_CENTERING must be between 0 and 1", AT);
146

147
148
149
150
151
			buff_before = chunk_size * buff_centering;
		} else {
			buff_before = chunk_size * 0.1; //10% centering by default
		}
	}
152
153
}

154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
double BufferedIOHandler::getAvgSamplingRate()
{
	if (vec_buffer_meteo.size() > 0){
		unsigned int sum = 0;
		for (unsigned int ii=0; ii<vec_buffer_meteo.size(); ii++){
			//count all data
			sum += vec_buffer_meteo[ii].size();
		}
		if (sum > 0){
			double days = buffer_end.getJulianDate() - buffer_start.getJulianDate();
			return ((double)sum / days);
		}
	}

	return IOUtils::nodata;
}

171
const std::vector< METEO_TIMESERIE >& BufferedIOHandler::get_complete_buffer(Date& start, Date& end)
172
173
174
175
176
177
178
{
	start = buffer_start;
	end   = buffer_end;

	return vec_buffer_meteo; //return reference
}

179
void BufferedIOHandler::readMeteoData(const Date& date_start, const Date& date_end, 
180
                                      std::vector< METEO_TIMESERIE >& vecMeteo,
181
                                      const unsigned int& /*stationindex*/)
182
183
{
	vecMeteo.clear();
184
185
	const Date new_buffer_start(date_start-buff_before); //taking centering into account
	Date new_buffer_end(new_buffer_start + chunk_size*chunks);
186
	vector< vector<MeteoData> > tmp_meteo_buffer; //it must be here -> adresses copied in 2. are still valid
187
188
189
190
191
192

	//Read MeteoData for requested interval in chunks, furthermore buffer it
	//Try to buffer after the requested chunk for subsequent calls

	//0. initialize if not already initialized
	if (vec_buffer_meteo.size() == 0) //init
193
		bufferData(new_buffer_start, new_buffer_end, vec_buffer_meteo);
194
195
196
197

	unsigned int buffer_size = vec_buffer_meteo.size();

	//1. Check whether data is in buffer already, and buffer it if not
198
	if ((date_start < buffer_start) || (date_end > buffer_end)){
199
		//rebuffer data
200
		if ((new_buffer_end != buffer_end) || (new_buffer_start != buffer_start)){
201
			//rebuffer for real
202
			bufferData(new_buffer_start, new_buffer_end, vec_buffer_meteo);
203
204
			buffer_size = vec_buffer_meteo.size();
		}
205

206
207
		while (date_end > new_buffer_end){
			//if the requested interval is bigger than a normal buffer, we have to increase the buffer anyway...
208
			iohandler.readMeteoData(new_buffer_end, new_buffer_end+chunk_size*chunks, tmp_meteo_buffer);
209
210

			if (tmp_meteo_buffer.size() != buffer_size)
211
				throw IOException("The number of stations changed over time, this is not handled yet!", AT);
212
213
214
215
216
217
218
219
220
221
222
223
224
			
			//Loop through stations and append data
			for (unsigned int ii=0; ii<buffer_size; ii++){
				unsigned int station_size = vec_buffer_meteo[ii].size();

				if ((station_size > 0) && (tmp_meteo_buffer[ii].size() > 0)){
					//check if the last element equals the first one
					if (vec_buffer_meteo[ii][station_size-1].date >= tmp_meteo_buffer[ii][0].date)
						vec_buffer_meteo[ii].pop_back(); //delete the element with the same date
				}

				vec_buffer_meteo[ii].insert(vec_buffer_meteo[ii].end(), tmp_meteo_buffer[ii].begin(), tmp_meteo_buffer[ii].end());
			}
225
226
			new_buffer_end += chunk_size*chunks;
			buffer_end = new_buffer_end;
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
		}
	}

	//2. Copy appropriate data into vecMeteo
	for (unsigned int ii=0; ii<buffer_size; ii++){
		vecMeteo.push_back(vector<MeteoData>()); //insert one empty vector of MeteoData

		if (vec_buffer_meteo[ii].size() == 0) continue; //no data in buffer for this station

		unsigned int pos_start = IOUtils::seek(date_start, vec_buffer_meteo[ii], false);
		if (pos_start == IOUtils::npos) pos_start = 0;

		unsigned int pos_end = IOUtils::seek(date_end, vec_buffer_meteo[ii], false);//HACK:: edit IOUtils::seek to accept an offset
		if (pos_end == IOUtils::npos)	pos_end = vec_buffer_meteo[ii].size() - 1; //just copy until the end of the buffer
		//cout << "Station " << ii << ": pos_start=" << pos_start << "  pos_end=" << pos_end << endl; 
		if (vec_buffer_meteo[ii][pos_end].date > date_end){
			if (pos_end > pos_start)	pos_end--;
		} else {
			pos_end++;
		}
		//cout << "Station " << ii << ": pos_start=" << pos_start << "  pos_end=" << pos_end << endl; 
		vecMeteo[ii].insert(vecMeteo[ii].begin(), vec_buffer_meteo[ii].begin()+pos_start, vec_buffer_meteo[ii].begin()+pos_end);
	}
}

252
void BufferedIOHandler::bufferData(const Date& date_start, const Date& date_end, std::vector< METEO_TIMESERIE >& vecvecMeteo){
253
254
255
	//TODO: implement reading data by chunks. It has to be done the same way as rebuffering
	vecvecMeteo.clear(); //the plugins do it internally anyway, but this is cheap and safe...
	iohandler.readMeteoData(date_start, date_end, vecvecMeteo);
256
257
258
259
	buffer_start = date_start;
	buffer_end   = date_end;
}

260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
/**
 * @brief Push a vector of time series of MeteoData objects into the local buffer. 
 *        This overwrites the local buffer. This method is a way to bypass the internal reading
 *        of MeteoData from a certain source.
 * @param date_start Representing the beginning of the data
 * @param date_end Representing the end of the data
 * @param vecMeteo The actual data being pushed into vec_buffer_meteo
 */
void BufferedIOHandler::push_meteo_data(const Date& date_start, const Date& date_end,
                                        const std::vector< METEO_TIMESERIE >& vecMeteo)
{
	//perform check on date_start and date_end
	if (date_end < date_start)
		throw InvalidArgumentException("date_start cannot be greater than date_end", AT);

	buffer_start     = date_start;
	buffer_end       = date_end;
	vec_buffer_meteo = vecMeteo;
}

280
void BufferedIOHandler::readSpecialPoints(std::vector<Coords>& in_cpa)
281
{
282
	iohandler.readSpecialPoints(in_cpa);
283
284
}

285
void BufferedIOHandler::write2DGrid(const Grid2DObject& in_grid2Dobj, const std::string& in_name)
286
{
287
	iohandler.write2DGrid(in_grid2Dobj, in_name);
288
289
290
291
292
293
}

void BufferedIOHandler::clearBuffer(){
	mapBufferedGrids.clear();
}

294
295
296
std::ostream& operator<<(std::ostream& os, const BufferedIOHandler& data)
{
	os << "<BufferedIOHandler>\n";
297
298
	os << "Config& cfg = " << hex << &data.cfg << dec << "\n";
	os << "IOHandler &iohandler = " << hex << &data.iohandler << dec << "\n";
299

300
	os << "Buffering " << data.chunks << " chunk(s) of " <<data.chunk_size.getJulianDate() << " days\n";
301
	
302
303
304
305
306
307
308
309
310
311
	os << "Current buffer content (" << data.vec_buffer_meteo.size() << " stations, " 
	   << data.mapBufferedGrids.size() << " grids):\n";

	for(unsigned int ii=0; ii<data.vec_buffer_meteo.size(); ii++) {
		if (data.vec_buffer_meteo[ii].size() > 0){
			os << std::setw(10) << data.vec_buffer_meteo[ii][0].meta.stationID << " = "
			   << data.vec_buffer_meteo[ii][0].date.toString(Date::ISO) << " - "
			   << data.vec_buffer_meteo[ii][data.vec_buffer_meteo[ii].size()-1].date.toString(Date::ISO) << ", "
			   << data.vec_buffer_meteo[ii].size() << " timesteps" << endl;
		}
312
313
314
315
316
317
318
319
320
321
322
323
324
	}

	std::map<std::string, Grid2DObject>::const_iterator it1;
	for (it1=data.mapBufferedGrids.begin(); it1 != data.mapBufferedGrids.end(); it1++){
		os << setw(10) << "Grid" << " = " << it1->first << ", ";
		os << (it1->second).ncols << " x " << (it1->second).nrows << " @ " << (it1->second).cellsize << "m\n";
	}

	os << "</BufferedIOHandler>\n";

	return os;
}

325
} //namespace