WSL/SLF GitLab Repository

IOHandler.cmake.cc 29 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
/***********************************************************************************/
/*  Copyright 2009-2012 WSL Institute for Snow and Avalanche Research  SLF-DAVOS   */
/***********************************************************************************/
/* This file is part of MeteoIO.
    MeteoIO is free software: you can redistribute it and/or modify
    it under the terms of the GNU Lesser General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    MeteoIO is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public License
    along with MeteoIO.  If not, see <http://www.gnu.org/licenses/>.
*/

19
#include <meteoio/IOUtils.h>
20
#include <meteoio/MathOptim.h>
21
22
#include <meteoio/IOHandler.h>

23
#cmakedefine PLUGIN_ALPUG
24
25
26
27
28
29
30
31
32
33
34
35
36
37
#cmakedefine PLUGIN_ARCIO
#cmakedefine PLUGIN_A3DIO
#cmakedefine PLUGIN_ARPSIO
#cmakedefine PLUGIN_GRASSIO
#cmakedefine PLUGIN_GEOTOPIO
#cmakedefine PLUGIN_SMETIO
#cmakedefine PLUGIN_SNIO
#cmakedefine PLUGIN_PGMIO
#cmakedefine PLUGIN_IMISIO
#cmakedefine PLUGIN_GRIBIO
#cmakedefine PLUGIN_PNGIO
#cmakedefine PLUGIN_BORMAIO
#cmakedefine PLUGIN_COSMOXMLIO
#cmakedefine PLUGIN_GSNIO
38
#cmakedefine PLUGIN_NETCDFIO
Thomas Egger's avatar
Thomas Egger committed
39
#cmakedefine PLUGIN_PSQLIO
40
#cmakedefine PLUGIN_SASEIO
41

42
#include <meteoio/plugins/ALPUG.h>
43
44
45
46
47
48
49
50
51
52
53
54
55
#include <meteoio/plugins/ARCIO.h>
#include <meteoio/plugins/A3DIO.h>
#include <meteoio/plugins/ARPSIO.h>
#include <meteoio/plugins/GrassIO.h>
#include <meteoio/plugins/GeotopIO.h>
#include <meteoio/plugins/PGMIO.h>
#include <meteoio/plugins/SMETIO.h>
#include <meteoio/plugins/SNIO.h>

#ifdef PLUGIN_BORMAIO
#include <meteoio/plugins/BormaIO.h>
#endif

56
57
58
59
#ifdef PLUGIN_CNRMIO
#include <meteoio/plugins/CNRMIO.h>
#endif

60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
#ifdef PLUGIN_COSMOXMLIO
#include <meteoio/plugins/CosmoXMLIO.h>
#endif

#ifdef PLUGIN_IMISIO
#include <meteoio/plugins/ImisIO.h>
#endif

#ifdef PLUGIN_GRIBIO
#include <meteoio/plugins/GRIBIO.h>
#endif

#ifdef PLUGIN_GSNIO
#include <meteoio/plugins/GSNIO.h>
#endif

76
77
78
79
#ifdef PLUGIN_NETCDFIO
#include <meteoio/plugins/NetCDFIO.h>
#endif

80
81
82
83
#ifdef PLUGIN_PNGIO
#include <meteoio/plugins/PNGIO.h>
#endif

Thomas Egger's avatar
Thomas Egger committed
84
85
86
87
#ifdef PLUGIN_PSQLIO
#include <meteoio/plugins/PSQLIO.h>
#endif

88
89
90
91
#ifdef PLUGIN_SASEIO
#include <meteoio/plugins/SASEIO.h>
#endif

92
93
94
95
96
using namespace std;

namespace mio {
 /**
 * @page plugins Plugins overview
97
98
 * The data access is handled by a system of plugins. They all offer the same interface, meaning that a plugin can transparently be replaced by another one. Since they might rely on third party libraries for accessing the data, they have been created as plugins, that is they are  only compiled if requested when configuring the compilation with cmake.
 * A plugin can therefore fail to run if it has not been compiled.
99
100
101
102
103
104
105
 *
 * @section available_categories Data sources categories
 * Several data sources categories have been defined that can be provided by a different plugin. Each data source category is defined by a specific key in the configuration file (usually, io.ini):
 * - METEO, for meteorological time series
 * - DEM, for Digital Elevation Maps
 * - LANDUSE, for land cover information
 * - GRID2D, for generic 2D grids (they can contain meteo fields and be recognized as such or arbitrary gridded data)
106
 * - POI, for a list of Points Of Interest that can be used for providing extra information at some specific location (extracting time series at a few selected points, etc)
107
108
109
110
111
112
113
114
115
116
117
118
119
120
 *
 * A plugin is "connected" to a given data source category simply by giving its keyword as value for the data source key:
 * @code
 * METEO = SMET
 * DEM = ARC
 * @endcode
 * Each plugin might have its own specific options, meaning that it might require its own keywords. Please check in each plugin documentation the supported options and keys (see links below).
 * Moreover, a given plugin might only support a given category for read or write (for example, PNG: there is no easy and safe way to interpret a given color as a given numeric value without knowing its color scale, so reading a png has been disabled).
 * Finally, the plugins usually don't implement all these categories (for example, ArcGIS file format only describes 2D grids, so the ARC plugin will only deal with 2D grids), so please check what a given plugin implements before connecting it to a specific data source category.
 *
 * @section available_plugins Available plugins
 * So far the following plugins have been implemented (by keyword for the io.ini key/value config file). Please read the documentation for each plugin in order to know the plugin-specific keywords:
 * <center><table border="1">
 * <tr><th>Plugin keyword</th><th>Provides</th><th>Description</th><th>Extra requirements</th></tr>
121
 * <tr><td>\subpage alpug "ALPUG"</td><td>meteo</td><td>data files generated by the %ALPUG meteo stations</td><td></td></tr>
122
 * <tr><td>\subpage a3d "A3D"</td><td>meteo, poi</td><td>original Alpine3D meteo files</td><td></td></tr>
123
124
125
 * <tr><td>\subpage arc "ARC"</td><td>dem, landuse, grid2d</td><td>ESRI/ARC ascii grid files</td><td></td></tr>
 * <tr><td>\subpage arps "ARPS"</td><td>dem, grid2d</td><td>ARPS ascii formatted grids</td><td></td></tr>
 * <tr><td>\subpage borma "BORMA"</td><td>meteo</td><td>Borma xml meteo files</td><td><A HREF="http://libxmlplusplus.sourceforge.net/">libxml++</A></td></tr>
126
 * <tr><td>\subpage cnrm "CNRM"</td><td>dem, grid2d, meteo</td><td>NetCDF meteorological timeseries following the <A HREF="http://www.cnrm.meteo.fr/?lang=en">CNRM</A> schema</td><td><A HREF="http://www.unidata.ucar.edu/downloads/netcdf/index.jsp">NetCDF-C library</A></td></tr>
127
 * <tr><td>\subpage cosmoxml "COSMOXML"</td><td>meteo</td><td>MeteoSwiss COSMO's postprocessing XML format</td><td><A HREF="http://xmlsoft.org/">libxml2</A></td></tr>
128
129
130
 * <tr><td>\subpage geotop "GEOTOP"</td><td>meteo</td><td>GeoTop meteo files</td><td></td></tr>
 * <tr><td>\subpage grass "GRASS"</td><td>dem, landuse, grid2d</td><td>Grass grid files</td><td></td></tr>
 * <tr><td>\subpage gribio "GRIB"</td><td>meteo, dem, grid2d</td><td>GRIB meteo grid files</td><td><A HREF="http://www.ecmwf.int/products/data/software/grib_api.html">grib-api</A></td></tr>
131
 * <tr><td>\subpage gsn "GSN"</td><td>meteo</td><td>connects to the Global Sensor Network web service interface</td><td><A HREF="http://curl.haxx.se/libcurl/">libcurl</A></td></tr>
132
 * <tr><td>\subpage imis "IMIS"</td><td>meteo</td><td>connects to the IMIS database</td><td><A HREF="http://docs.oracle.com/cd/B12037_01/appdev.101/b10778/introduction.htm">Oracle's OCCI library</A></td></tr>
133
 * <tr><td>\subpage netcdf "NETCDF"</td><td>dem, grid2d</td><td>NetCDF grids</td><td><A HREF="http://www.unidata.ucar.edu/downloads/netcdf/index.jsp">NetCDF-C library</A></td></tr>
134
135
 * <tr><td>\subpage pgmio "PGM"</td><td>dem, grid2d</td><td>PGM grid files</td><td></td></tr>
 * <tr><td>\subpage pngio "PNG"</td><td>dem, grid2d</td><td>PNG grid files</td><td><A HREF="http://www.libpng.org/pub/png/libpng.html">libpng</A></td></tr>
136
 * <tr><td>\subpage psqlio "PSQL"</td><td>meteo</td><td>connects to PostgreSQL database</td><td><A HREF="http://www.postgresql.org/">PostgreSQL</A>'s libpq</td></tr>
137
 * <tr><td>\subpage sase "SASE"</td><td>meteo</td><td>connects to the SASE database</td><td><A HREF="https://dev.mysql.com/doc/refman/5.0/en/c-api.html">MySQL's C API</A></td></tr>
138
 * <tr><td>\subpage smetio "SMET"</td><td>meteo, poi</td><td>SMET data files</td><td></td></tr>
139
140
141
 * <tr><td>\subpage snowpack "SNOWPACK"</td><td>meteo</td><td>original SNOWPACK meteo files</td><td></td></tr>
 * </table></center>
 *
142
 * @section data_manipulations Raw data editing
143
 * @subsection data_generators Data generators
144
 * It is also possible to duplicate a meteorological parameter as another meteorological parameter. This is done by specifying a COPY key, following the syntax
145
 * new_name::COPY = existing_parameter. For example:
146
 * @code
147
 * VW_avg::COPY = VW
148
149
150
151
 * @endcode
 * This creates a new parameter VW_avg that starts as an exact copy of the raw data of VW, for each station. This newly created parameter is
 * then processed as any other meteorological parameter (thus going through filtering, generic processing, spatial interpolations). This only current
 * limitation is that the parameter providing the raw data must be defined for all stations (even if filled with nodata, this is good enough).
152
153
154
155
156
157
 *
 * @subsection data_exclusion Data exclusion
 * It is possible to exclude specific parameters from given stations (on a per station basis). This is either done by using the station ID
 * followed by "::exclude" as key with a space delimited list of \ref meteoparam "meteorological parameters" to exclude for the station as key.
 * Another possibility is to provide a file containing one station ID per line followed by a space delimited list of \ref meteoparam "meteorological parameters"
 * to exclude for the station (the path to the file can be a relative path and will be properly resolved).
158
159
160
 * 
 * The exact opposite can also be done, excluding ALL parameters except the ones declared with the "::keep" statement (or a file containing one station ID
 * per line followed by a space delimited list of \ref meteoparam "meteorological parameters" to keep for the station).
161
162
 *
 * @code
163
 * WFJ2::EXCLUDE = HS PSUM                       ;inline declaration of parameters exclusion
164
 * KLO3::KEEP = TA RH VW DW                      ;inline declaration of parameters to keep
165
166
 *
 * EXCLUDE_FILE = ../input/meteo/excludes.csv    ;parameters exclusions defined in a separate file
167
 * KEEP_FILE = ../input/meteo/keeps.csv          ;parameters to keep defined in a separate file
168
169
170
171
172
 * @endcode
 *
 * In the second example (relying on a separate file), the file "../input/meteo/excludes.csv" could look like this:
 * @code
 * WFJ2 TA RH
173
 * KLO3 HS PSUM
174
175
 * @endcode
 *
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
 * @subsection data_merging Data merging
 * It is possible to merge different data sets together, based on a common station name. This is enabled with the key \em MERGE_BY_NAME in the [Input] section.
 * If set to \em true, all stations that have the same station name will be merged together, in the order they have been declared/read by the plugin
 * (ie the first station that has a value for a given parameter has priority). This is useful, for example, to provide measurements from different 
 * stations that actually share the same measurement location or to build "composite" station from multiple real stations.
 * @code
 * STATION1 = *WFJ
 * STATION2 = WFJ2
 * STATION3 = WFJ3
 * 
 * *WFJ::KEEP = ILWR PSUM
 * WFJ2::EXCLUDE = PSUM ILWR RSWR
 * WFJ1::KEEP = ISWR VW DW
 * 
 * MERGE_BY_NAME = true
 * @endcode
 * In the above example, if the same station name would have been given to "*WFJ", "WFJ2" and "WFJ1", then a composite station with station ID
 * "*WFJ" (ie the first one of the list) would be built with ILWR and PSUM coming from the original *WFJ station, every fields of WFJ2 excepted PSUM, ILWR and ISWR
 * and only ISWR, VW, DW from WFJ1 <b>when</b> WFJ2 does not have them.
195
196
 */

197
IOInterface* IOHandler::getPlugin(const std::string& plugin_name) const
198
{
199
200
201
#ifdef PLUGIN_ALPUG
	if (plugin_name == "ALPUG") return new ALPUG(cfg);
#endif
202
#ifdef PLUGIN_ARCIO
203
	if (plugin_name == "ARC") return new ARCIO(cfg);
204
205
#endif
#ifdef PLUGIN_A3DIO
206
	if (plugin_name == "A3D") return new A3DIO(cfg);
207
208
#endif
#ifdef PLUGIN_ARPSIO
209
	if (plugin_name == "ARPS") return new ARPSIO(cfg);
210
211
#endif
#ifdef PLUGIN_GRASSIO
212
	if (plugin_name == "GRASS") return new GrassIO(cfg);
213
214
#endif
#ifdef PLUGIN_GEOTOPIO
215
	if (plugin_name == "GEOTOP") return new GeotopIO(cfg);
216
217
#endif
#ifdef PLUGIN_SMETIO
218
	if (plugin_name == "SMET") return new SMETIO(cfg);
219
220
#endif
#ifdef PLUGIN_SNIO
221
	if (plugin_name == "SNOWPACK") return new SNIO(cfg);
222
223
#endif
#ifdef PLUGIN_PGMIO
224
	if (plugin_name == "PGM") return new PGMIO(cfg);
225
226
#endif
#ifdef PLUGIN_IMISIO
227
	if (plugin_name == "IMIS") return new ImisIO(cfg);
228
229
#endif
#ifdef PLUGIN_GRIBIO
230
	if (plugin_name == "GRIB") return new GRIBIO(cfg);
231
232
#endif
#ifdef PLUGIN_PNGIO
233
	if (plugin_name == "PNG") return new PNGIO(cfg);
234
235
#endif
#ifdef PLUGIN_BORMAIO
236
	if (plugin_name == "BORMA") return new BormaIO(cfg);
237
238
#endif
#ifdef PLUGIN_COSMOXMLIO
239
	if (plugin_name == "COSMOXML") return new CosmoXMLIO(cfg);
240
#endif
241
242
243
#ifdef PLUGIN_CNRMIO
	if (plugin_name == "CNRM") return new CNRMIO(cfg);
#endif
244
#ifdef PLUGIN_GSNIO
245
	if (plugin_name == "GSN") return new GSNIO(cfg);
246
#endif
247
#ifdef PLUGIN_NETCDFIO
248
	if (plugin_name == "NETCDF") return new NetCDFIO(cfg);
249
#endif
Thomas Egger's avatar
Thomas Egger committed
250
#ifdef PLUGIN_PSQLIO
251
	if (plugin_name == "PSQL") return new PSQLIO(cfg);
Thomas Egger's avatar
Thomas Egger committed
252
#endif
253
254
255
#ifdef PLUGIN_SASEIO
	if (plugin_name == "SASE") return new SASEIO(cfg);
#endif
256
257

	return NULL; //no plugin found
258
259
}

260
261
262
263
264
265
266
//this is actually an object factory
IOInterface* IOHandler::getPlugin(const std::string& cfgkey, const std::string& cfgsection)
{
	std::string op_src;
	cfg.getValue(cfgkey, cfgsection, op_src);

	if (mapPlugins.find(op_src) == mapPlugins.end()) {
267
268
		IOInterface *ioPtr = getPlugin(op_src);
		if (ioPtr==NULL)
269
			throw IOException("Cannot find plugin " + op_src + " as requested in file " + cfg.getSourceName() + ". Has it been activated through ccmake? Is it declared in IOHandler::getPlugin?", AT);
270
271
		else
			mapPlugins[op_src] = ioPtr;
272
273
274
275
276
	}

	return mapPlugins[op_src];
}

277
278
//Copy constructor
IOHandler::IOHandler(const IOHandler& aio)
279
           : IOInterface(), cfg(aio.cfg), mapPlugins(aio.mapPlugins), excluded_params(aio.excluded_params), kept_params(aio.kept_params),
280
281
             copy_parameter(aio.copy_parameter), copy_name(aio.copy_name), enable_copying(aio.enable_copying), 
             excludes_ready(aio.excludes_ready), keeps_ready(aio.keeps_ready), mergeByName(aio.mergeByName)
Mathias Bavay's avatar
Mathias Bavay committed
282
{}
283

Mathias Bavay's avatar
Mathias Bavay committed
284
IOHandler::IOHandler(const Config& cfgreader)
285
286
           : IOInterface(), cfg(cfgreader), mapPlugins(), excluded_params(), kept_params(), copy_parameter(), copy_name(), 
           enable_copying(false), excludes_ready(false), keeps_ready(false), mergeByName(false)
287
288
{
	parse_copy_config();
289
	cfg.getValue("MERGE_BY_NAME", "Input", mergeByName, IOUtils::nothrow);
290
291
}

Mathias Bavay's avatar
Mathias Bavay committed
292
293
IOHandler::~IOHandler() throw()
{
294
	// Get rid of the objects
295
296
297
	std::map<std::string, IOInterface*>::iterator mapit;
	for (mapit = mapPlugins.begin(); mapit!=mapPlugins.end(); ++mapit) {
		delete mapit->second;
298
299
300
	}
}

301
IOHandler& IOHandler::operator=(const IOHandler& source) {
302
	if (this != &source) {
303
		mapPlugins = source.mapPlugins;
304
		excluded_params = source.excluded_params;
305
		kept_params = source.kept_params;
306
307
308
		copy_parameter = source.copy_parameter;
		copy_name = source.copy_name;
		enable_copying = source.enable_copying;
309
		excludes_ready = source.excludes_ready;
310
		keeps_ready = source.keeps_ready;
311
		mergeByName = source.mergeByName;
312
313
314
315
	}
	return *this;
}

316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
void IOHandler::read2DGrid(Grid2DObject& grid_out, const std::string& i_filename)
{
	IOInterface *plugin = getPlugin("GRID2D", "Input");
	plugin->read2DGrid(grid_out, i_filename);
}

void IOHandler::read2DGrid(Grid2DObject& grid_out, const MeteoGrids::Parameters& parameter, const Date& date)
{
	IOInterface *plugin = getPlugin("GRID2D", "Input");
	plugin->read2DGrid(grid_out, parameter, date);
}

void IOHandler::readDEM(DEMObject& dem_out)
{
	IOInterface *plugin = getPlugin("DEM", "Input");
	plugin->readDEM(dem_out);
	dem_out.update();
}

void IOHandler::readLanduse(Grid2DObject& landuse_out)
{
	IOInterface *plugin = getPlugin("LANDUSE", "Input");
	plugin->readLanduse(landuse_out);
}

341
void IOHandler::readStationData(const Date& date, STATIONS_SET& vecStation)
342
343
344
{
	IOInterface *plugin = getPlugin("METEO", "Input");
	plugin->readStationData(date, vecStation);
345
	if (mergeByName) merge_by_name(vecStation);
346
347
348
}

void IOHandler::readMeteoData(const Date& dateStart, const Date& dateEnd,
349
                              std::vector<METEO_SET>& vecMeteo,
350
351
352
353
                              const size_t& stationindex)
{
	IOInterface *plugin = getPlugin("METEO", "Input");
	plugin->readMeteoData(dateStart, dateEnd, vecMeteo, stationindex);
354
	
355
	checkTimestamps(vecMeteo);
356
	
357
358
	if (!excludes_ready) create_exclude_map();
	exclude_params(vecMeteo);
359
360
361
	
	if (!keeps_ready) create_keep_map();
	keep_params(vecMeteo);
362

363
364
	if (mergeByName) merge_by_name(vecMeteo);
	
365
366
	copy_parameters(stationindex, vecMeteo);
}
Mathias Bavay's avatar
Mathias Bavay committed
367

368
void IOHandler::writeMeteoData(const std::vector<METEO_SET>& vecMeteo,
369
370
371
372
373
374
375
376
377
378
379
380
                               const std::string& name)
{
	IOInterface *plugin = getPlugin("METEO", "Output");
	plugin->writeMeteoData(vecMeteo, name);
}

void IOHandler::readAssimilationData(const Date& date_in, Grid2DObject& da_out)
{
	IOInterface *plugin = getPlugin("DA", "Input");
	plugin->readAssimilationData(date_in, da_out);
}

381
382
383
void IOHandler::readPOI(std::vector<Coords>& pts) {
	IOInterface *plugin = getPlugin("POI", "Input");
	plugin->readPOI(pts);
384
385
386
387
388
389
390
391
392
393
394
395
396
397
}

void IOHandler::write2DGrid(const Grid2DObject& grid_in, const std::string& name)
{
	IOInterface *plugin = getPlugin("GRID2D", "Output");
	plugin->write2DGrid(grid_in, name);
}

void IOHandler::write2DGrid(const Grid2DObject& grid_in, const MeteoGrids::Parameters& parameter, const Date& date)
{
	IOInterface *plugin = getPlugin("GRID2D", "Output");
	plugin->write2DGrid(grid_in, parameter, date);
}

398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
/**
* Make sure all timestamps are unique and in increasing order
*/
void IOHandler::checkTimestamps(const std::vector<METEO_SET>& vecVecMeteo) const
{
	for (size_t stat_idx=0; stat_idx<vecVecMeteo.size(); ++stat_idx) { //for each station
		const size_t nr_timestamps = vecVecMeteo[stat_idx].size();
		if (nr_timestamps==0) continue;

		Date previous_date( vecVecMeteo[stat_idx].front().date );
		for (size_t ii=1; ii<nr_timestamps; ++ii) {
			const Date& current_date = vecVecMeteo[stat_idx][ii].date;
			if (current_date<=previous_date) {
				const StationData& station = vecVecMeteo[stat_idx][ii].meta;
				throw IOException("Error at time "+current_date.toString(Date::ISO)+" for station \""+station.stationName+"\" ("+station.stationID+") : timestamps must be in increasing order and unique!", AT);
			}
			previous_date = current_date;
		}
	}
}

419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
//merge stations that have identical names
void IOHandler::merge_by_name(STATIONS_SET& vecStation) const
{
	const size_t idxMiddle = Optim::ceil( static_cast<double>(vecStation.size()) / 2. );
	for (size_t ii=0; (ii<=idxMiddle) && (ii<vecStation.size()); ii++) {
		for (size_t jj=ii+1; jj<vecStation.size(); jj++) {
			if (vecStation[ii].stationName==vecStation[jj].stationName) {
				vecStation[ii].merge( vecStation[jj] );
				std::swap( vecStation[jj], vecStation.back() );
				vecStation.pop_back();
				jj--; //we need to re-compare the current station since it has been swapped
			}
		}
	}
}

//in this implementation, we consider that the station name does NOT change over time
void IOHandler::merge_by_name(std::vector<METEO_SET>& vecVecMeteo) const
{
	const size_t idxMiddle = Optim::ceil( static_cast<double>(vecVecMeteo.size()) / 2. );
	for (size_t ii=0; (ii<=idxMiddle) && (ii<vecVecMeteo.size()); ii++) {
		if (vecVecMeteo[ii].empty())  continue;
		
		for (size_t jj=ii+1; jj<vecVecMeteo.size(); jj++) {
			if (vecVecMeteo[jj].empty()) continue;
			
			if (vecVecMeteo[ii][0].meta.stationName==vecVecMeteo[jj][0].meta.stationName) {
				MeteoData::mergeTimeSeries(vecVecMeteo[ii], vecVecMeteo[jj]);
				std::swap( vecVecMeteo[jj], vecVecMeteo.back() );
				vecVecMeteo.pop_back();
				jj--; //we need to re-compare the current station since it has been swapped
			}
		}
	}
}

455
456
void IOHandler::create_exclude_map()
{
457
	excludes_ready = true;
458
	string exclude_file;
459
	cfg.getValue("EXCLUDE_FILE", "Input", exclude_file, IOUtils::nothrow);
460

461
462
463
464
465
	if (!exclude_file.empty()) {
		//if this is a relative path, prefix the path with the current path
		const std::string prefix = ( IOUtils::isAbsolutePath(exclude_file) )? "" : cfg.getConfigRootDir()+"/";
		const std::string path = IOUtils::getPath(prefix+exclude_file, true);  //clean & resolve path
		const std::string filename = path + "/" + IOUtils::getFilename(exclude_file);
466

467
468
		if (!IOUtils::fileExists(filename)) throw FileAccessException(filename, AT); //prevent invalid filenames
		std::ifstream fin(filename.c_str(), std::ifstream::in);
469
		if (fin.fail()) throw FileAccessException(filename, AT);
470

471
472
		try {
			const char eoln = IOUtils::getEoln(fin); //get the end of line character for the file
473

474
475
			vector<string> tmpvec;
			string line;
476

477
478
479
480
			while (!fin.eof()) { //Go through file
				getline(fin, line, eoln); //read complete line meta information
				IOUtils::stripComments(line);
				const size_t ncols = IOUtils::readLineToVec(line, tmpvec, ' ');
481

482
				if (ncols > 1) {
483
					for (vector<string>::iterator it = tmpvec.begin()+1; it != tmpvec.end(); ++it) {
484
485
						IOUtils::toUpper(*it);
					}
486

487
488
					const set<string> tmpset(tmpvec.begin()+1, tmpvec.end());
					excluded_params[ tmpvec[0] ] = tmpset;
489
490
				}
			}
491
492
493
		} catch (const std::exception&) {
			fin.close();
			throw;
494
		}
495

496
497
498
		fin.close();
	}

499
500
501
502
503
504
505
506
507
508
	vector<string> exclude_keys;
	const size_t nrOfStations = cfg.findKeys(exclude_keys, "::EXCLUDE", "Input", true);
	for (size_t ii=0; ii<nrOfStations; ++ii) {
		const size_t found = exclude_keys[ii].find_first_of(":");
		if (found==std::string::npos) continue;

		const string station( exclude_keys[ii].substr(0,found) );
		std::vector<std::string> vecString;
		cfg.getValue(exclude_keys[ii], "Input", vecString);
		if (vecString.empty()) throw InvalidArgumentException("Empty value for key \""+exclude_keys[ii]+"\"", AT);
509
		for (vector<string>::iterator it = vecString.begin(); it != vecString.end(); ++it) {
510
511
512
513
514
515
			IOUtils::toUpper(*it);
		}

		const set<string> tmpset(vecString.begin(), vecString.end());
		excluded_params[ station ] = tmpset;
	}
516
517
}

518
519
520
521
522
523
524
525
526
527
528
529
void IOHandler::create_keep_map()
{
	keeps_ready = true;
	string keep_file;
	cfg.getValue("KEEP_FILE", "Input", keep_file, IOUtils::nothrow);

	if (!keep_file.empty()) {
		//if this is a relative path, prefix the path with the current path
		const std::string prefix = ( IOUtils::isAbsolutePath(keep_file) )? "" : cfg.getConfigRootDir()+"/";
		const std::string path = IOUtils::getPath(prefix+keep_file, true);  //clean & resolve path
		const std::string filename = path + "/" + IOUtils::getFilename(keep_file);

530
531
		if (!IOUtils::fileExists(filename)) throw FileAccessException(filename, AT); //prevent invalid filenames
		std::ifstream fin(filename.c_str(), std::ifstream::in);
532
533
534
535
536
537
538
539
540
541
542
543
544
545
		if (fin.fail()) throw FileAccessException(filename, AT);

		try {
			const char eoln = IOUtils::getEoln(fin); //get the end of line character for the file

			vector<string> tmpvec;
			string line;

			while (!fin.eof()) { //Go through file
				getline(fin, line, eoln); //read complete line meta information
				IOUtils::stripComments(line);
				const size_t ncols = IOUtils::readLineToVec(line, tmpvec, ' ');

				if (ncols > 1) {
546
					for (vector<string>::iterator it = tmpvec.begin()+1; it != tmpvec.end(); ++it) {
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
						IOUtils::toUpper(*it);
					}

					const set<string> tmpset(tmpvec.begin()+1, tmpvec.end());
					kept_params[ tmpvec[0] ] = tmpset;
				}
			}
		} catch (const std::exception&) {
			fin.close();
			throw;
		}

		fin.close();
	}

	vector<string> keep_keys;
	const size_t nrOfStations = cfg.findKeys(keep_keys, "::KEEP", "Input", true);
	for (size_t ii=0; ii<nrOfStations; ++ii) {
		const size_t found = keep_keys[ii].find_first_of(":");
		if (found==std::string::npos) continue;

		const string station( keep_keys[ii].substr(0,found) );
		std::vector<std::string> vecString;
		cfg.getValue(keep_keys[ii], "Input", vecString);
		if (vecString.empty()) throw InvalidArgumentException("Empty value for key \""+keep_keys[ii]+"\"", AT);
572
		for (vector<string>::iterator it = vecString.begin(); it != vecString.end(); ++it) {
573
574
575
576
577
578
579
580
			IOUtils::toUpper(*it);
		}

		const set<string> tmpset(vecString.begin(), vecString.end());
		kept_params[ station ] = tmpset;
	}
}

581
/**
582
* @brief reset to nodata the parameters marked as EXCLUDE on a per station basis
583
*/
584
585
586
587
void IOHandler::exclude_params(std::vector<METEO_SET>& vecVecMeteo) const
{
	if (excluded_params.empty()) return;

588
	for (size_t station=0; station<vecVecMeteo.size(); ++station) { //loop over the stations
589
		if (vecVecMeteo[station].empty()) continue;
590
591
592
593
594
595
		const string stationID = vecVecMeteo[station][0].meta.stationID;
		const map< string, set<string> >::const_iterator it = excluded_params.find(stationID);
		if (it == excluded_params.end()) continue;

		const set<string> excluded = it->second;

596
		for (size_t ii=0; ii<vecVecMeteo[station].size(); ++ii) { //loop over the timesteps
597
598
599
600
601
602
603
604
605
606
			std::set<std::string>::const_iterator it_set;
			for (it_set=excluded.begin(); it_set != excluded.end(); ++it_set) {
				const string param = *it_set;
				if (vecVecMeteo[station][ii].param_exists(param))
					vecVecMeteo[station][ii](param) = IOUtils::nodata;
			}
		}
	}
}

607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
/**
* @brief only keep the parameters marked as KEEP on a per station basis
*/
void IOHandler::keep_params(std::vector<METEO_SET>& vecVecMeteo) const
{
	if (kept_params.empty()) return;

	for (size_t station=0; station<vecVecMeteo.size(); ++station) { //loop over the stations
		if (vecVecMeteo[station].empty()) continue;
		
		const string stationID = vecVecMeteo[station][0].meta.stationID;
		const map< string, set<string> >::const_iterator it = kept_params.find(stationID);
		if (it == kept_params.end()) continue;

		const set<string> kept = it->second;
		
		for (size_t ii=0; ii<vecVecMeteo[station].size(); ++ii) {
			MeteoData& md_ref = vecVecMeteo[station][ii];
			MeteoData md( md_ref );
			md.reset(); //delete all meteo fields
			
			std::set<std::string>::const_iterator it_set;
			for (it_set=kept.begin(); it_set != kept.end(); ++it_set) { //loop over the parameters to keep
				const string param = *it_set;
				if (!md.param_exists(param)) continue;
				 md(param) = md_ref(param);
			}
			
			//copy back the new object into vecVecMeteo
			md_ref = md;
		}
	}
}

641
642
643
644
/**
* Parse [Input] section for potential parameters that the user wants
* duplicated (as '%%::COPY = %%')
*/
645
646
647
void IOHandler::parse_copy_config()
{
	vector<string> copy_keys;
648
	const size_t nrOfMatches = cfg.findKeys(copy_keys, "::COPY", "Input", true); //search anywhere in key
649

650
	for (size_t ii=0; ii<nrOfMatches; ++ii) {
651
		const string name_of_copy = copy_keys[ii].substr( 0, copy_keys[ii].find_first_of(":") );
652
		string initial_name;
653
654
655
656
657
658
659
660
661
		cfg.getValue(copy_keys[ii], "Input", initial_name);
		if ((name_of_copy.length() > 0) && (initial_name.length() > 0)){
			copy_parameter.push_back(initial_name);
			copy_name.push_back(name_of_copy);
			enable_copying = true;
		}
	}
}

662
663
664
665
666
/**
* This procedure runs through the MeteoData objects in vecMeteo and according to user
* configuration copies a certain present meteo parameter to another one, named by the
* user in the [Input] section of the io.ini, e.g.
* [Input]
667
* TA2::COPY = TA
668
669
670
* means that TA2 will be the name of a new parameter in MeteoData with the copied value
* of the meteo parameter MeteoData::TA
*/
671
void IOHandler::copy_parameters(const size_t& stationindex, std::vector< METEO_SET >& vecMeteo) const
672
673
674
{
	if (!enable_copying) return; //Nothing configured

675
676
	if (stationindex != IOUtils::npos && stationindex>=vecMeteo.size())
		throw IndexOutOfBoundsException("Accessing stationindex in readMeteoData that is out of bounds", AT);
677

678
679
	const size_t station_start = (stationindex==IOUtils::npos)? 0 : stationindex;
	const size_t station_end = (stationindex==IOUtils::npos)? vecMeteo.size() : stationindex+1;
680
681
682
	const size_t nr_of_params = copy_parameter.size();
	vector<size_t> indices; //will hold the indices of the parameters to be copied

683
684
	for (size_t ii=station_start; ii<station_end; ++ii) { //for each station
		for (size_t jj=0; jj<vecMeteo[ii].size(); ++jj) { //for each MeteoData object of one station
685
686

			if (jj==0) { //buffer the index numbers
687
				for (size_t kk=0; kk<nr_of_params; ++kk) {
688
689
					const size_t param_index = vecMeteo[ii][jj].getParameterIndex(copy_parameter[kk]);
					if (param_index == IOUtils::npos) {
690
						std::ostringstream ss;
691
692
693
694
695
696
697
698
699
						ss << "At " << vecMeteo[ii][jj].date.toString(Date::ISO) << ", station " << vecMeteo[ii][jj].meta.stationID;
						ss << " has no parameter \"" << copy_parameter[kk] << "\" to copy!\n";
						throw InvalidArgumentException(ss.str(), AT);
					}

					indices.push_back(param_index);
				}
			}

700
			for (size_t kk=0; kk<nr_of_params; ++kk) {
701
702
703
704
705
706
707
708
				const size_t newparam = vecMeteo[ii][jj].addParameter(copy_name[kk]);
				vecMeteo[ii][jj](newparam) = vecMeteo[ii][jj](indices[kk]);
			}
		}
		indices.clear(); //may change for every station
	}
}

709
const std::string IOHandler::toString() const
710
{
711
	std::ostringstream os;
712
713
714
715
	os << "<IOHandler>\n";
	os << "Config& cfg = " << hex << &cfg << dec << "\n";

	os << "<mapPlugins>\n";
716
717
718
	std::map<std::string, IOInterface*>::const_iterator it1;
	for (it1=mapPlugins.begin(); it1 != mapPlugins.end(); ++it1) {
		os << setw(10) << it1->first << " = " << hex <<  it1->second << dec << "\n";
719
720
	}
	os << "</mapPlugins>\n";
721

722
723
724
725
726
727
728
729
730
731
732
	if (!excluded_params.empty()) {
		os << "<excluded_params>\n";
		std::map< std::string, std::set<std::string> >::const_iterator it_exc;
		for (it_exc=excluded_params.begin(); it_exc != excluded_params.end(); ++it_exc) {
			os << setw(10) << it_exc->first << " = ";
			std::set<std::string>::const_iterator it_set;
			for (it_set=(it_exc->second).begin(); it_set != (it_exc->second).end(); ++it_set)
				os << *it_set << " ";
			os << "\n";
		}
		os << "</excluded_params>\n";
733
	}
734
735
736
	
	if (mergeByName)
		os << "Merge stations by stationName\n";
737

738
739
740
741
742
	os << "</IOHandler>\n";
	return os.str();
}

} //end namespace