WSL/SLF GitLab Repository

IOHandler.cmake.cc 32.8 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
/***********************************************************************************/
/*  Copyright 2009-2012 WSL Institute for Snow and Avalanche Research  SLF-DAVOS   */
/***********************************************************************************/
/* This file is part of MeteoIO.
    MeteoIO is free software: you can redistribute it and/or modify
    it under the terms of the GNU Lesser General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    MeteoIO is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public License
    along with MeteoIO.  If not, see <http://www.gnu.org/licenses/>.
*/

19
#include <meteoio/IOUtils.h>
20
#include <meteoio/MathOptim.h>
21
22
#include <meteoio/IOHandler.h>

23
#cmakedefine PLUGIN_ALPUG
24
25
26
27
28
29
30
31
32
33
34
35
36
37
#cmakedefine PLUGIN_ARCIO
#cmakedefine PLUGIN_A3DIO
#cmakedefine PLUGIN_ARPSIO
#cmakedefine PLUGIN_GRASSIO
#cmakedefine PLUGIN_GEOTOPIO
#cmakedefine PLUGIN_SMETIO
#cmakedefine PLUGIN_SNIO
#cmakedefine PLUGIN_PGMIO
#cmakedefine PLUGIN_IMISIO
#cmakedefine PLUGIN_GRIBIO
#cmakedefine PLUGIN_PNGIO
#cmakedefine PLUGIN_BORMAIO
#cmakedefine PLUGIN_COSMOXMLIO
#cmakedefine PLUGIN_GSNIO
38
#cmakedefine PLUGIN_NETCDFIO
Thomas Egger's avatar
Thomas Egger committed
39
#cmakedefine PLUGIN_PSQLIO
40
#cmakedefine PLUGIN_SASEIO
41

42
#include <meteoio/plugins/ALPUG.h>
43
44
45
46
47
48
49
50
51
52
53
54
55
#include <meteoio/plugins/ARCIO.h>
#include <meteoio/plugins/A3DIO.h>
#include <meteoio/plugins/ARPSIO.h>
#include <meteoio/plugins/GrassIO.h>
#include <meteoio/plugins/GeotopIO.h>
#include <meteoio/plugins/PGMIO.h>
#include <meteoio/plugins/SMETIO.h>
#include <meteoio/plugins/SNIO.h>

#ifdef PLUGIN_BORMAIO
#include <meteoio/plugins/BormaIO.h>
#endif

56
57
58
59
#ifdef PLUGIN_CNRMIO
#include <meteoio/plugins/CNRMIO.h>
#endif

60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
#ifdef PLUGIN_COSMOXMLIO
#include <meteoio/plugins/CosmoXMLIO.h>
#endif

#ifdef PLUGIN_IMISIO
#include <meteoio/plugins/ImisIO.h>
#endif

#ifdef PLUGIN_GRIBIO
#include <meteoio/plugins/GRIBIO.h>
#endif

#ifdef PLUGIN_GSNIO
#include <meteoio/plugins/GSNIO.h>
#endif

76
77
78
79
#ifdef PLUGIN_NETCDFIO
#include <meteoio/plugins/NetCDFIO.h>
#endif

80
81
82
83
#ifdef PLUGIN_PNGIO
#include <meteoio/plugins/PNGIO.h>
#endif

Thomas Egger's avatar
Thomas Egger committed
84
85
86
87
#ifdef PLUGIN_PSQLIO
#include <meteoio/plugins/PSQLIO.h>
#endif

88
89
90
91
#ifdef PLUGIN_SASEIO
#include <meteoio/plugins/SASEIO.h>
#endif

92
93
94
95
96
using namespace std;

namespace mio {
 /**
 * @page plugins Plugins overview
97
98
 * The data access is handled by a system of plugins. They all offer the same interface, meaning that a plugin can transparently be replaced by another one. Since they might rely on third party libraries for accessing the data, they have been created as plugins, that is they are  only compiled if requested when configuring the compilation with cmake.
 * A plugin can therefore fail to run if it has not been compiled.
99
100
101
102
103
104
105
 *
 * @section available_categories Data sources categories
 * Several data sources categories have been defined that can be provided by a different plugin. Each data source category is defined by a specific key in the configuration file (usually, io.ini):
 * - METEO, for meteorological time series
 * - DEM, for Digital Elevation Maps
 * - LANDUSE, for land cover information
 * - GRID2D, for generic 2D grids (they can contain meteo fields and be recognized as such or arbitrary gridded data)
106
 * - POI, for a list of Points Of Interest that can be used for providing extra information at some specific location (extracting time series at a few selected points, etc)
107
108
109
110
111
112
113
114
115
116
117
118
119
120
 *
 * A plugin is "connected" to a given data source category simply by giving its keyword as value for the data source key:
 * @code
 * METEO = SMET
 * DEM = ARC
 * @endcode
 * Each plugin might have its own specific options, meaning that it might require its own keywords. Please check in each plugin documentation the supported options and keys (see links below).
 * Moreover, a given plugin might only support a given category for read or write (for example, PNG: there is no easy and safe way to interpret a given color as a given numeric value without knowing its color scale, so reading a png has been disabled).
 * Finally, the plugins usually don't implement all these categories (for example, ArcGIS file format only describes 2D grids, so the ARC plugin will only deal with 2D grids), so please check what a given plugin implements before connecting it to a specific data source category.
 *
 * @section available_plugins Available plugins
 * So far the following plugins have been implemented (by keyword for the io.ini key/value config file). Please read the documentation for each plugin in order to know the plugin-specific keywords:
 * <center><table border="1">
 * <tr><th>Plugin keyword</th><th>Provides</th><th>Description</th><th>Extra requirements</th></tr>
121
 * <tr><td>\subpage alpug "ALPUG"</td><td>meteo</td><td>data files generated by the %ALPUG meteo stations</td><td></td></tr>
122
 * <tr><td>\subpage a3d "A3D"</td><td>meteo, poi</td><td>original Alpine3D meteo files</td><td></td></tr>
123
124
125
 * <tr><td>\subpage arc "ARC"</td><td>dem, landuse, grid2d</td><td>ESRI/ARC ascii grid files</td><td></td></tr>
 * <tr><td>\subpage arps "ARPS"</td><td>dem, grid2d</td><td>ARPS ascii formatted grids</td><td></td></tr>
 * <tr><td>\subpage borma "BORMA"</td><td>meteo</td><td>Borma xml meteo files</td><td><A HREF="http://libxmlplusplus.sourceforge.net/">libxml++</A></td></tr>
126
 * <tr><td>\subpage cnrm "CNRM"</td><td>dem, grid2d, meteo</td><td>NetCDF meteorological timeseries following the <A HREF="http://www.cnrm.meteo.fr/?lang=en">CNRM</A> schema</td><td><A HREF="http://www.unidata.ucar.edu/downloads/netcdf/index.jsp">NetCDF-C library</A></td></tr>
127
 * <tr><td>\subpage cosmoxml "COSMOXML"</td><td>meteo</td><td>MeteoSwiss COSMO's postprocessing XML format</td><td><A HREF="http://xmlsoft.org/">libxml2</A></td></tr>
128
129
130
 * <tr><td>\subpage geotop "GEOTOP"</td><td>meteo</td><td>GeoTop meteo files</td><td></td></tr>
 * <tr><td>\subpage grass "GRASS"</td><td>dem, landuse, grid2d</td><td>Grass grid files</td><td></td></tr>
 * <tr><td>\subpage gribio "GRIB"</td><td>meteo, dem, grid2d</td><td>GRIB meteo grid files</td><td><A HREF="http://www.ecmwf.int/products/data/software/grib_api.html">grib-api</A></td></tr>
131
 * <tr><td>\subpage gsn "GSN"</td><td>meteo</td><td>connects to the Global Sensor Network web service interface</td><td><A HREF="http://curl.haxx.se/libcurl/">libcurl</A></td></tr>
132
 * <tr><td>\subpage imis "IMIS"</td><td>meteo</td><td>connects to the IMIS database</td><td><A HREF="http://docs.oracle.com/cd/B12037_01/appdev.101/b10778/introduction.htm">Oracle's OCCI library</A></td></tr>
133
 * <tr><td>\subpage netcdf "NETCDF"</td><td>dem, grid2d</td><td>NetCDF grids</td><td><A HREF="http://www.unidata.ucar.edu/downloads/netcdf/index.jsp">NetCDF-C library</A></td></tr>
134
135
 * <tr><td>\subpage pgmio "PGM"</td><td>dem, grid2d</td><td>PGM grid files</td><td></td></tr>
 * <tr><td>\subpage pngio "PNG"</td><td>dem, grid2d</td><td>PNG grid files</td><td><A HREF="http://www.libpng.org/pub/png/libpng.html">libpng</A></td></tr>
136
 * <tr><td>\subpage psqlio "PSQL"</td><td>meteo</td><td>connects to PostgreSQL database</td><td><A HREF="http://www.postgresql.org/">PostgreSQL</A>'s libpq</td></tr>
137
 * <tr><td>\subpage sase "SASE"</td><td>meteo</td><td>connects to the SASE database</td><td><A HREF="https://dev.mysql.com/doc/refman/5.0/en/c-api.html">MySQL's C API</A></td></tr>
138
 * <tr><td>\subpage smetio "SMET"</td><td>meteo, poi</td><td>SMET data files</td><td></td></tr>
139
140
141
 * <tr><td>\subpage snowpack "SNOWPACK"</td><td>meteo</td><td>original SNOWPACK meteo files</td><td></td></tr>
 * </table></center>
 *
142
 * @section data_manipulations Raw data editing
143
 * @subsection data_generators Data generators
144
 * It is also possible to duplicate a meteorological parameter as another meteorological parameter. This is done by specifying a COPY key, following the syntax
145
 * new_name::COPY = existing_parameter. For example:
146
 * @code
147
 * VW_avg::COPY = VW
148
149
150
151
 * @endcode
 * This creates a new parameter VW_avg that starts as an exact copy of the raw data of VW, for each station. This newly created parameter is
 * then processed as any other meteorological parameter (thus going through filtering, generic processing, spatial interpolations). This only current
 * limitation is that the parameter providing the raw data must be defined for all stations (even if filled with nodata, this is good enough).
152
153
154
155
156
157
 *
 * @subsection data_exclusion Data exclusion
 * It is possible to exclude specific parameters from given stations (on a per station basis). This is either done by using the station ID
 * followed by "::exclude" as key with a space delimited list of \ref meteoparam "meteorological parameters" to exclude for the station as key.
 * Another possibility is to provide a file containing one station ID per line followed by a space delimited list of \ref meteoparam "meteorological parameters"
 * to exclude for the station (the path to the file can be a relative path and will be properly resolved).
158
159
160
 * 
 * The exact opposite can also be done, excluding ALL parameters except the ones declared with the "::keep" statement (or a file containing one station ID
 * per line followed by a space delimited list of \ref meteoparam "meteorological parameters" to keep for the station).
161
162
 *
 * @code
163
 * WFJ2::EXCLUDE = HS PSUM                       ;inline declaration of parameters exclusion
164
 * KLO3::KEEP = TA RH VW DW                      ;inline declaration of parameters to keep
165
166
 *
 * EXCLUDE_FILE = ../input/meteo/excludes.csv    ;parameters exclusions defined in a separate file
167
 * KEEP_FILE = ../input/meteo/keeps.csv          ;parameters to keep defined in a separate file
168
169
170
171
172
 * @endcode
 *
 * In the second example (relying on a separate file), the file "../input/meteo/excludes.csv" could look like this:
 * @code
 * WFJ2 TA RH
173
 * KLO3 HS PSUM
174
175
 * @endcode
 *
176
 * @subsection data_merging Data merging
177
178
179
180
181
182
 * It is possible to merge different data sets together, with a syntax similar to the Exclude/Keep syntax. This merging occurs <b>after</b> any 
 * EXCLUDE/KEEP commands. This is useful, for example, to provide measurements from different stations that actually share the 
 * same measurement location or to build "composite" station from multiple real stations (in this case, using EXCLUDE and/or KEEP 
 * commands to fine tune how the composite station(s) is/are built). 
 * Please note that the order of declaration defines the priority (ie the first station that has a value for a given parameter has priority).
 * 
183
184
185
 * @code
 * STATION1 = *WFJ
 * STATION2 = WFJ2
186
187
 * STATION3 = WFJ1
 * [...]
188
189
190
191
 * 
 * *WFJ::KEEP = ILWR PSUM
 * WFJ2::EXCLUDE = PSUM ILWR RSWR
 * 
192
193
 * *WFJ::MERGE = WFJ2 WFJ1
 * DRB2::MERGE = DRB1 WFJ2
194
 * @endcode
195
196
 * In order to avoid circular dependencies, a station can NOT receive data from a station AND contribute data to another station. Otherwise, a 
 * station can be merged into multiple other stations.
197
198
 */

199
IOInterface* IOHandler::getPlugin(const std::string& plugin_name) const
200
{
201
202
203
#ifdef PLUGIN_ALPUG
	if (plugin_name == "ALPUG") return new ALPUG(cfg);
#endif
204
#ifdef PLUGIN_ARCIO
205
	if (plugin_name == "ARC") return new ARCIO(cfg);
206
207
#endif
#ifdef PLUGIN_A3DIO
208
	if (plugin_name == "A3D") return new A3DIO(cfg);
209
210
#endif
#ifdef PLUGIN_ARPSIO
211
	if (plugin_name == "ARPS") return new ARPSIO(cfg);
212
213
#endif
#ifdef PLUGIN_GRASSIO
214
	if (plugin_name == "GRASS") return new GrassIO(cfg);
215
216
#endif
#ifdef PLUGIN_GEOTOPIO
217
	if (plugin_name == "GEOTOP") return new GeotopIO(cfg);
218
219
#endif
#ifdef PLUGIN_SMETIO
220
	if (plugin_name == "SMET") return new SMETIO(cfg);
221
222
#endif
#ifdef PLUGIN_SNIO
223
	if (plugin_name == "SNOWPACK") return new SNIO(cfg);
224
225
#endif
#ifdef PLUGIN_PGMIO
226
	if (plugin_name == "PGM") return new PGMIO(cfg);
227
228
#endif
#ifdef PLUGIN_IMISIO
229
	if (plugin_name == "IMIS") return new ImisIO(cfg);
230
231
#endif
#ifdef PLUGIN_GRIBIO
232
	if (plugin_name == "GRIB") return new GRIBIO(cfg);
233
234
#endif
#ifdef PLUGIN_PNGIO
235
	if (plugin_name == "PNG") return new PNGIO(cfg);
236
237
#endif
#ifdef PLUGIN_BORMAIO
238
	if (plugin_name == "BORMA") return new BormaIO(cfg);
239
240
#endif
#ifdef PLUGIN_COSMOXMLIO
241
	if (plugin_name == "COSMOXML") return new CosmoXMLIO(cfg);
242
#endif
243
244
245
#ifdef PLUGIN_CNRMIO
	if (plugin_name == "CNRM") return new CNRMIO(cfg);
#endif
246
#ifdef PLUGIN_GSNIO
247
	if (plugin_name == "GSN") return new GSNIO(cfg);
248
#endif
249
#ifdef PLUGIN_NETCDFIO
250
	if (plugin_name == "NETCDF") return new NetCDFIO(cfg);
251
#endif
Thomas Egger's avatar
Thomas Egger committed
252
#ifdef PLUGIN_PSQLIO
253
	if (plugin_name == "PSQL") return new PSQLIO(cfg);
Thomas Egger's avatar
Thomas Egger committed
254
#endif
255
256
257
#ifdef PLUGIN_SASEIO
	if (plugin_name == "SASE") return new SASEIO(cfg);
#endif
258
259

	return NULL; //no plugin found
260
261
}

262
263
264
265
266
267
268
//this is actually an object factory
IOInterface* IOHandler::getPlugin(const std::string& cfgkey, const std::string& cfgsection)
{
	std::string op_src;
	cfg.getValue(cfgkey, cfgsection, op_src);

	if (mapPlugins.find(op_src) == mapPlugins.end()) {
269
270
		IOInterface *ioPtr = getPlugin(op_src);
		if (ioPtr==NULL)
271
			throw IOException("Cannot find plugin " + op_src + " as requested in file " + cfg.getSourceName() + ". Has it been activated through ccmake? Is it declared in IOHandler::getPlugin?", AT);
272
273
		else
			mapPlugins[op_src] = ioPtr;
274
275
276
277
278
	}

	return mapPlugins[op_src];
}

279
280
//Copy constructor
IOHandler::IOHandler(const IOHandler& aio)
281
282
           : IOInterface(), cfg(aio.cfg), mapPlugins(aio.mapPlugins), excluded_params(aio.excluded_params), kept_params(aio.kept_params), 
             merge_commands(aio.merge_commands), merged_stations(aio.merged_stations),
283
             copy_parameter(aio.copy_parameter), copy_name(aio.copy_name), enable_copying(aio.enable_copying), 
284
             excludes_ready(aio.excludes_ready), keeps_ready(aio.keeps_ready), merge_ready(aio.merge_ready)
Mathias Bavay's avatar
Mathias Bavay committed
285
{}
286

Mathias Bavay's avatar
Mathias Bavay committed
287
IOHandler::IOHandler(const Config& cfgreader)
288
289
290
           : IOInterface(), cfg(cfgreader), mapPlugins(), excluded_params(), kept_params(), 
             merge_commands(), merged_stations(), copy_parameter(), copy_name(), 
             enable_copying(false), excludes_ready(false), keeps_ready(false), merge_ready(false)
291
292
293
294
{
	parse_copy_config();
}

Mathias Bavay's avatar
Mathias Bavay committed
295
296
IOHandler::~IOHandler() throw()
{
297
	// Get rid of the objects
298
299
300
	std::map<std::string, IOInterface*>::iterator mapit;
	for (mapit = mapPlugins.begin(); mapit!=mapPlugins.end(); ++mapit) {
		delete mapit->second;
301
302
303
	}
}

304
IOHandler& IOHandler::operator=(const IOHandler& source) {
305
	if (this != &source) {
306
		mapPlugins = source.mapPlugins;
307
		excluded_params = source.excluded_params;
308
		kept_params = source.kept_params;
309
310
		merge_commands = source.merge_commands;
		merged_stations = source.merged_stations;
311
312
313
		copy_parameter = source.copy_parameter;
		copy_name = source.copy_name;
		enable_copying = source.enable_copying;
314
		excludes_ready = source.excludes_ready;
315
		keeps_ready = source.keeps_ready;
316
		merge_ready = source.merge_ready;
317
318
319
320
	}
	return *this;
}

321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
void IOHandler::read2DGrid(Grid2DObject& grid_out, const std::string& i_filename)
{
	IOInterface *plugin = getPlugin("GRID2D", "Input");
	plugin->read2DGrid(grid_out, i_filename);
}

void IOHandler::read2DGrid(Grid2DObject& grid_out, const MeteoGrids::Parameters& parameter, const Date& date)
{
	IOInterface *plugin = getPlugin("GRID2D", "Input");
	plugin->read2DGrid(grid_out, parameter, date);
}

void IOHandler::readDEM(DEMObject& dem_out)
{
	IOInterface *plugin = getPlugin("DEM", "Input");
	plugin->readDEM(dem_out);
	dem_out.update();
}

void IOHandler::readLanduse(Grid2DObject& landuse_out)
{
	IOInterface *plugin = getPlugin("LANDUSE", "Input");
	plugin->readLanduse(landuse_out);
}

346
void IOHandler::readStationData(const Date& date, STATIONS_SET& vecStation)
347
348
349
{
	IOInterface *plugin = getPlugin("METEO", "Input");
	plugin->readStationData(date, vecStation);
350
351
352
	
	if (!merge_ready) create_merge_map(); 
	merge_stations(vecStation);
353
354
355
}

void IOHandler::readMeteoData(const Date& dateStart, const Date& dateEnd,
356
                              std::vector<METEO_SET>& vecMeteo,
357
358
359
360
                              const size_t& stationindex)
{
	IOInterface *plugin = getPlugin("METEO", "Input");
	plugin->readMeteoData(dateStart, dateEnd, vecMeteo, stationindex);
361
	
362
	checkTimestamps(vecMeteo);
363
	
364
365
	if (!excludes_ready) create_exclude_map();
	exclude_params(vecMeteo);
366
367
368
	
	if (!keeps_ready) create_keep_map();
	keep_params(vecMeteo);
369

370
371
	if (!merge_ready) create_merge_map(); 
	merge_stations(vecMeteo);
372
	
373
374
	copy_parameters(stationindex, vecMeteo);
}
Mathias Bavay's avatar
Mathias Bavay committed
375

376
void IOHandler::writeMeteoData(const std::vector<METEO_SET>& vecMeteo,
377
378
379
380
381
382
383
384
385
386
387
388
                               const std::string& name)
{
	IOInterface *plugin = getPlugin("METEO", "Output");
	plugin->writeMeteoData(vecMeteo, name);
}

void IOHandler::readAssimilationData(const Date& date_in, Grid2DObject& da_out)
{
	IOInterface *plugin = getPlugin("DA", "Input");
	plugin->readAssimilationData(date_in, da_out);
}

389
390
391
void IOHandler::readPOI(std::vector<Coords>& pts) {
	IOInterface *plugin = getPlugin("POI", "Input");
	plugin->readPOI(pts);
392
393
394
395
396
397
398
399
400
401
402
403
404
405
}

void IOHandler::write2DGrid(const Grid2DObject& grid_in, const std::string& name)
{
	IOInterface *plugin = getPlugin("GRID2D", "Output");
	plugin->write2DGrid(grid_in, name);
}

void IOHandler::write2DGrid(const Grid2DObject& grid_in, const MeteoGrids::Parameters& parameter, const Date& date)
{
	IOInterface *plugin = getPlugin("GRID2D", "Output");
	plugin->write2DGrid(grid_in, parameter, date);
}

406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
/**
* Make sure all timestamps are unique and in increasing order
*/
void IOHandler::checkTimestamps(const std::vector<METEO_SET>& vecVecMeteo) const
{
	for (size_t stat_idx=0; stat_idx<vecVecMeteo.size(); ++stat_idx) { //for each station
		const size_t nr_timestamps = vecVecMeteo[stat_idx].size();
		if (nr_timestamps==0) continue;

		Date previous_date( vecVecMeteo[stat_idx].front().date );
		for (size_t ii=1; ii<nr_timestamps; ++ii) {
			const Date& current_date = vecVecMeteo[stat_idx][ii].date;
			if (current_date<=previous_date) {
				const StationData& station = vecVecMeteo[stat_idx][ii].meta;
				throw IOException("Error at time "+current_date.toString(Date::ISO)+" for station \""+station.stationName+"\" ("+station.stationID+") : timestamps must be in increasing order and unique!", AT);
			}
			previous_date = current_date;
		}
	}
}

427
428
429
430
431
432
433
434
435
436
void IOHandler::create_merge_map()
{
	merge_ready = true;
	
	vector<string> merge_keys;
	const size_t nrOfStations = cfg.findKeys(merge_keys, "::MERGE", "Input", true);
	for (size_t ii=0; ii<nrOfStations; ++ii) {
		const size_t found = merge_keys[ii].find_first_of(":");
		if (found==std::string::npos) continue;

437
		const string station( IOUtils::strToUpper(merge_keys[ii].substr(0,found)) );
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
		std::vector<std::string> vecString;
		cfg.getValue(merge_keys[ii], "Input", vecString);
		if (vecString.empty()) throw InvalidArgumentException("Empty value for key \""+merge_keys[ii]+"\"", AT);
		
		for (vector<string>::iterator it = vecString.begin(); it != vecString.end(); ++it) {
			IOUtils::toUpper(*it);
			const std::vector<std::string>::const_iterator vec_it = find (merged_stations.begin(), merged_stations.end(), *it);
			if (vec_it==merged_stations.end()) merged_stations.push_back( *it ); //this station will be merged into another one
		}

		merge_commands[ station ] = vecString;
	}
	
	//sort the merged_stations vector so searches will be faster
	std::sort(merged_stations.begin(), merged_stations.end());
	
	//make sure there is no "chain merge": station A merging station B and station C merging station A
	std::map< std::string, std::vector<std::string> >::iterator it_dest;
	for(it_dest=merge_commands.begin(); it_dest!=merge_commands.end(); ++it_dest) {
		const string &stationID = it_dest->first;
		if (std::binary_search(merged_stations.begin(), merged_stations.end(), stationID))
			throw InvalidArgumentException("\'chain merge\' detected for station \'"+stationID+"\', this is not supported (see documentation)", AT);
	}
}

463
//merge stations that have identical names
464
void IOHandler::merge_stations(STATIONS_SET& vecStation) const
465
{
466
	for (size_t ii=0; ii<vecStation.size(); ii++) {
467
		const string toStationID = IOUtils::strToUpper(vecStation[ii].stationID);
468
469
470
471
472
473
474
475
476
477
478
		//we do not support "chain merge": station A merging station B and station C merging station A
		if ( std::find(merged_stations.begin(), merged_stations.end(), toStationID)!=merged_stations.end() ) continue;
		
		const map< string, vector<string> >::const_iterator it = merge_commands.find( toStationID );
		if (it == merge_commands.end()) continue; //no merge commands for this station

		const vector<string> &merge_from( it->second );
		for (size_t idx=0; idx<merge_from.size() ; ++idx) {
			const string fromStationID( merge_from[idx] );
			
			for (size_t jj=0; jj<vecStation.size(); jj++) {
479
480
				const string curr_station = IOUtils::strToUpper(vecStation[jj].stationID);
				if (curr_station==fromStationID) vecStation[ii].merge( vecStation[jj] );
481
482
483
			}
		}
	}
484
485
486
487
488
489
490
491
492
493
494
	
	//remove the stations that have been merged into other ones
	for (size_t ii=0; ii<vecStation.size(); ii++) {
		const string toStationID = vecStation[ii].stationID;
		const vector<string>::const_iterator it = std::find(merged_stations.begin(), merged_stations.end(), toStationID);
		if ( it!=merged_stations.end() ) {
			std::swap( vecStation[ii], vecStation.back() );
			vecStation.pop_back();
			ii--; //in case we have multiple identical stations ID
		}
	}
495
496
497
}

//in this implementation, we consider that the station name does NOT change over time
498
void IOHandler::merge_stations(std::vector<METEO_SET>& vecVecMeteo) const
499
{
500
	for (size_t ii=0; ii<vecVecMeteo.size(); ii++) {
501
		if (vecVecMeteo[ii].empty())  continue;
502
503
		const string toStationID = IOUtils::strToUpper(vecVecMeteo[ii][0].meta.stationID);

504
505
506
507
508
509
510
511
512
513
514
		//we do not support "chain merge": station A merging station B and station C merging station A
		if ( std::find(merged_stations.begin(), merged_stations.end(), toStationID)!=merged_stations.end() ) continue;
		
		const map< string, vector<string> >::const_iterator it = merge_commands.find( toStationID );
		if (it == merge_commands.end()) continue; //no merge commands for this station

		const vector<string> &merge_from( it->second );
		for (size_t idx=0; idx<merge_from.size(); ++idx) {
			const string fromStationID( merge_from[idx] );
			for (size_t jj=0; jj<vecVecMeteo.size(); jj++) {
				if (vecVecMeteo[jj].empty()) continue;
515
516
				const string curr_station = IOUtils::strToUpper(vecVecMeteo[jj][0].meta.stationID);
				if (curr_station==fromStationID)
517
					MeteoData::mergeTimeSeries(vecVecMeteo[ii], vecVecMeteo[jj]);
518
519
520
			}
		}
	}
521
522
523
	
	//remove the stations that have been merged into other ones
	for (size_t ii=0; ii<vecVecMeteo.size(); ii++) {
524
		const string toStationID = IOUtils::strToUpper(vecVecMeteo[ii][0].meta.stationID);
525
526
527
528
529
530
531
		const vector<string>::const_iterator it = std::find(merged_stations.begin(), merged_stations.end(), toStationID);
		if ( it!=merged_stations.end() ) {
			std::swap( vecVecMeteo[ii], vecVecMeteo.back() );
			vecVecMeteo.pop_back();
			ii--; //in case we have multiple identical stations ID
		}
	}
532
533
}

534
535
void IOHandler::create_exclude_map()
{
536
	excludes_ready = true;
537
	string exclude_file;
538
	cfg.getValue("EXCLUDE_FILE", "Input", exclude_file, IOUtils::nothrow);
539

540
541
542
543
544
	if (!exclude_file.empty()) {
		//if this is a relative path, prefix the path with the current path
		const std::string prefix = ( IOUtils::isAbsolutePath(exclude_file) )? "" : cfg.getConfigRootDir()+"/";
		const std::string path = IOUtils::getPath(prefix+exclude_file, true);  //clean & resolve path
		const std::string filename = path + "/" + IOUtils::getFilename(exclude_file);
545

546
547
		if (!IOUtils::fileExists(filename)) throw FileAccessException(filename, AT); //prevent invalid filenames
		std::ifstream fin(filename.c_str(), std::ifstream::in);
548
		if (fin.fail()) throw FileAccessException(filename, AT);
549

550
551
		try {
			const char eoln = IOUtils::getEoln(fin); //get the end of line character for the file
552

553
554
			vector<string> tmpvec;
			string line;
555

556
557
558
559
			while (!fin.eof()) { //Go through file
				getline(fin, line, eoln); //read complete line meta information
				IOUtils::stripComments(line);
				const size_t ncols = IOUtils::readLineToVec(line, tmpvec, ' ');
560

561
				if (ncols > 1) {
562
					for (vector<string>::iterator it = tmpvec.begin()+1; it != tmpvec.end(); ++it) {
563
564
						IOUtils::toUpper(*it);
					}
565

566
567
					const set<string> tmpset(tmpvec.begin()+1, tmpvec.end());
					excluded_params[ tmpvec[0] ] = tmpset;
568
569
				}
			}
570
571
572
		} catch (const std::exception&) {
			fin.close();
			throw;
573
		}
574

575
576
577
		fin.close();
	}

578
579
580
581
582
583
	vector<string> exclude_keys;
	const size_t nrOfStations = cfg.findKeys(exclude_keys, "::EXCLUDE", "Input", true);
	for (size_t ii=0; ii<nrOfStations; ++ii) {
		const size_t found = exclude_keys[ii].find_first_of(":");
		if (found==std::string::npos) continue;

584
		const string station( IOUtils::strToUpper(exclude_keys[ii].substr(0,found)) );
585
586
587
		std::vector<std::string> vecString;
		cfg.getValue(exclude_keys[ii], "Input", vecString);
		if (vecString.empty()) throw InvalidArgumentException("Empty value for key \""+exclude_keys[ii]+"\"", AT);
588
		for (vector<string>::iterator it = vecString.begin(); it != vecString.end(); ++it) {
589
590
591
592
593
594
			IOUtils::toUpper(*it);
		}

		const set<string> tmpset(vecString.begin(), vecString.end());
		excluded_params[ station ] = tmpset;
	}
595
596
}

597
598
599
600
601
602
603
604
605
606
607
608
void IOHandler::create_keep_map()
{
	keeps_ready = true;
	string keep_file;
	cfg.getValue("KEEP_FILE", "Input", keep_file, IOUtils::nothrow);

	if (!keep_file.empty()) {
		//if this is a relative path, prefix the path with the current path
		const std::string prefix = ( IOUtils::isAbsolutePath(keep_file) )? "" : cfg.getConfigRootDir()+"/";
		const std::string path = IOUtils::getPath(prefix+keep_file, true);  //clean & resolve path
		const std::string filename = path + "/" + IOUtils::getFilename(keep_file);

609
610
		if (!IOUtils::fileExists(filename)) throw FileAccessException(filename, AT); //prevent invalid filenames
		std::ifstream fin(filename.c_str(), std::ifstream::in);
611
612
613
614
615
616
617
618
619
620
621
622
623
624
		if (fin.fail()) throw FileAccessException(filename, AT);

		try {
			const char eoln = IOUtils::getEoln(fin); //get the end of line character for the file

			vector<string> tmpvec;
			string line;

			while (!fin.eof()) { //Go through file
				getline(fin, line, eoln); //read complete line meta information
				IOUtils::stripComments(line);
				const size_t ncols = IOUtils::readLineToVec(line, tmpvec, ' ');

				if (ncols > 1) {
625
					for (vector<string>::iterator it = tmpvec.begin()+1; it != tmpvec.end(); ++it) {
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
						IOUtils::toUpper(*it);
					}

					const set<string> tmpset(tmpvec.begin()+1, tmpvec.end());
					kept_params[ tmpvec[0] ] = tmpset;
				}
			}
		} catch (const std::exception&) {
			fin.close();
			throw;
		}

		fin.close();
	}

	vector<string> keep_keys;
	const size_t nrOfStations = cfg.findKeys(keep_keys, "::KEEP", "Input", true);
	for (size_t ii=0; ii<nrOfStations; ++ii) {
		const size_t found = keep_keys[ii].find_first_of(":");
		if (found==std::string::npos) continue;

647
		const string station( IOUtils::strToUpper(keep_keys[ii].substr(0,found)) );
648
649
650
		std::vector<std::string> vecString;
		cfg.getValue(keep_keys[ii], "Input", vecString);
		if (vecString.empty()) throw InvalidArgumentException("Empty value for key \""+keep_keys[ii]+"\"", AT);
651
		for (vector<string>::iterator it = vecString.begin(); it != vecString.end(); ++it) {
652
653
654
655
656
657
658
659
			IOUtils::toUpper(*it);
		}

		const set<string> tmpset(vecString.begin(), vecString.end());
		kept_params[ station ] = tmpset;
	}
}

660
/**
661
* @brief reset to nodata the parameters marked as EXCLUDE on a per station basis
662
*/
663
664
665
666
void IOHandler::exclude_params(std::vector<METEO_SET>& vecVecMeteo) const
{
	if (excluded_params.empty()) return;

667
	for (size_t station=0; station<vecVecMeteo.size(); ++station) { //loop over the stations
668
		if (vecVecMeteo[station].empty()) continue;
669
		const string stationID = IOUtils::strToUpper(vecVecMeteo[station][0].meta.stationID);
670
671
672
673
674
		const map< string, set<string> >::const_iterator it = excluded_params.find(stationID);
		if (it == excluded_params.end()) continue;

		const set<string> excluded = it->second;

675
		for (size_t ii=0; ii<vecVecMeteo[station].size(); ++ii) { //loop over the timesteps
676
677
678
679
680
681
682
683
684
685
			std::set<std::string>::const_iterator it_set;
			for (it_set=excluded.begin(); it_set != excluded.end(); ++it_set) {
				const string param = *it_set;
				if (vecVecMeteo[station][ii].param_exists(param))
					vecVecMeteo[station][ii](param) = IOUtils::nodata;
			}
		}
	}
}

686
687
688
689
690
691
692
693
694
695
/**
* @brief only keep the parameters marked as KEEP on a per station basis
*/
void IOHandler::keep_params(std::vector<METEO_SET>& vecVecMeteo) const
{
	if (kept_params.empty()) return;

	for (size_t station=0; station<vecVecMeteo.size(); ++station) { //loop over the stations
		if (vecVecMeteo[station].empty()) continue;
		
696
		const string stationID = IOUtils::strToUpper(vecVecMeteo[station][0].meta.stationID);
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
		const map< string, set<string> >::const_iterator it = kept_params.find(stationID);
		if (it == kept_params.end()) continue;

		const set<string> kept = it->second;
		
		for (size_t ii=0; ii<vecVecMeteo[station].size(); ++ii) {
			MeteoData& md_ref = vecVecMeteo[station][ii];
			MeteoData md( md_ref );
			md.reset(); //delete all meteo fields
			
			std::set<std::string>::const_iterator it_set;
			for (it_set=kept.begin(); it_set != kept.end(); ++it_set) { //loop over the parameters to keep
				const string param = *it_set;
				if (!md.param_exists(param)) continue;
				 md(param) = md_ref(param);
			}
			
			//copy back the new object into vecVecMeteo
			md_ref = md;
		}
	}
}

720
721
722
723
/**
* Parse [Input] section for potential parameters that the user wants
* duplicated (as '%%::COPY = %%')
*/
724
725
726
void IOHandler::parse_copy_config()
{
	vector<string> copy_keys;
727
	const size_t nrOfMatches = cfg.findKeys(copy_keys, "::COPY", "Input", true); //search anywhere in key
728

729
	for (size_t ii=0; ii<nrOfMatches; ++ii) {
730
		const string name_of_copy = copy_keys[ii].substr( 0, copy_keys[ii].find_first_of(":") );
731
		string initial_name;
732
733
734
735
736
737
738
739
740
		cfg.getValue(copy_keys[ii], "Input", initial_name);
		if ((name_of_copy.length() > 0) && (initial_name.length() > 0)){
			copy_parameter.push_back(initial_name);
			copy_name.push_back(name_of_copy);
			enable_copying = true;
		}
	}
}

741
742
743
744
745
/**
* This procedure runs through the MeteoData objects in vecMeteo and according to user
* configuration copies a certain present meteo parameter to another one, named by the
* user in the [Input] section of the io.ini, e.g.
* [Input]
746
* TA2::COPY = TA
747
748
749
* means that TA2 will be the name of a new parameter in MeteoData with the copied value
* of the meteo parameter MeteoData::TA
*/
750
void IOHandler::copy_parameters(const size_t& stationindex, std::vector< METEO_SET >& vecMeteo) const
751
752
753
{
	if (!enable_copying) return; //Nothing configured

754
755
	if (stationindex != IOUtils::npos && stationindex>=vecMeteo.size())
		throw IndexOutOfBoundsException("Accessing stationindex in readMeteoData that is out of bounds", AT);
756

757
758
	const size_t station_start = (stationindex==IOUtils::npos)? 0 : stationindex;
	const size_t station_end = (stationindex==IOUtils::npos)? vecMeteo.size() : stationindex+1;
759
760
761
	const size_t nr_of_params = copy_parameter.size();
	vector<size_t> indices; //will hold the indices of the parameters to be copied

762
763
	for (size_t ii=station_start; ii<station_end; ++ii) { //for each station
		for (size_t jj=0; jj<vecMeteo[ii].size(); ++jj) { //for each MeteoData object of one station
764
765

			if (jj==0) { //buffer the index numbers
766
				for (size_t kk=0; kk<nr_of_params; ++kk) {
767
768
					const size_t param_index = vecMeteo[ii][jj].getParameterIndex(copy_parameter[kk]);
					if (param_index == IOUtils::npos) {
769
						std::ostringstream ss;
770
771
772
773
774
775
776
777
778
						ss << "At " << vecMeteo[ii][jj].date.toString(Date::ISO) << ", station " << vecMeteo[ii][jj].meta.stationID;
						ss << " has no parameter \"" << copy_parameter[kk] << "\" to copy!\n";
						throw InvalidArgumentException(ss.str(), AT);
					}

					indices.push_back(param_index);
				}
			}

779
			for (size_t kk=0; kk<nr_of_params; ++kk) {
780
781
782
783
784
785
786
787
				const size_t newparam = vecMeteo[ii][jj].addParameter(copy_name[kk]);
				vecMeteo[ii][jj](newparam) = vecMeteo[ii][jj](indices[kk]);
			}
		}
		indices.clear(); //may change for every station
	}
}

788
const std::string IOHandler::toString() const
789
{
790
	std::ostringstream os;
791
792
793
794
	os << "<IOHandler>\n";
	os << "Config& cfg = " << hex << &cfg << dec << "\n";

	os << "<mapPlugins>\n";
795
796
797
	std::map<std::string, IOInterface*>::const_iterator it1;
	for (it1=mapPlugins.begin(); it1 != mapPlugins.end(); ++it1) {
		os << setw(10) << it1->first << " = " << hex <<  it1->second << dec << "\n";
798
799
	}
	os << "</mapPlugins>\n";
800

801
802
803
804
805
806
807
808
809
810
811
	if (!excluded_params.empty()) {
		os << "<excluded_params>\n";
		std::map< std::string, std::set<std::string> >::const_iterator it_exc;
		for (it_exc=excluded_params.begin(); it_exc != excluded_params.end(); ++it_exc) {
			os << setw(10) << it_exc->first << " = ";
			std::set<std::string>::const_iterator it_set;
			for (it_set=(it_exc->second).begin(); it_set != (it_exc->second).end(); ++it_set)
				os << *it_set << " ";
			os << "\n";
		}
		os << "</excluded_params>\n";
812
	}
813
	
814
815
816
817
818
819
820
821
822
823
824
	if (!merge_commands.empty()) {
		os << "<merge_commands>\n";
		std::map< std::string, std::vector<std::string> >::const_iterator it_merge;
		for (it_merge=merge_commands.begin(); it_merge != merge_commands.end(); ++it_merge) {
			os << setw(10) << it_merge->first << " <- ";
			for (size_t ii=0; ii<it_merge->second.size(); ++ii)
				os << it_merge->second[ii] << " ";
			os << "\n";
		}
		os << "</merge_commands>\n";
	}
825

826
827
828
829
830
	os << "</IOHandler>\n";
	return os.str();
}

} //end namespace