WSL/SLF GitLab Repository

IOHandler.cmake.cc 37.3 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
/***********************************************************************************/
/*  Copyright 2009-2012 WSL Institute for Snow and Avalanche Research  SLF-DAVOS   */
/***********************************************************************************/
/* This file is part of MeteoIO.
    MeteoIO is free software: you can redistribute it and/or modify
    it under the terms of the GNU Lesser General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    MeteoIO is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public License
    along with MeteoIO.  If not, see <http://www.gnu.org/licenses/>.
*/

19
#include <meteoio/IOUtils.h>
20
#include <meteoio/MathOptim.h>
21
22
#include <meteoio/IOHandler.h>

23
#cmakedefine PLUGIN_ALPUG
24
25
26
27
28
29
30
31
32
#cmakedefine PLUGIN_ARCIO
#cmakedefine PLUGIN_A3DIO
#cmakedefine PLUGIN_ARPSIO
#cmakedefine PLUGIN_GRASSIO
#cmakedefine PLUGIN_GEOTOPIO
#cmakedefine PLUGIN_SMETIO
#cmakedefine PLUGIN_SNIO
#cmakedefine PLUGIN_PGMIO
#cmakedefine PLUGIN_IMISIO
33
#cmakedefine PLUGIN_OSHDIO
34
35
36
37
38
#cmakedefine PLUGIN_GRIBIO
#cmakedefine PLUGIN_PNGIO
#cmakedefine PLUGIN_BORMAIO
#cmakedefine PLUGIN_COSMOXMLIO
#cmakedefine PLUGIN_GSNIO
39
#cmakedefine PLUGIN_NETCDFIO
Thomas Egger's avatar
Thomas Egger committed
40
#cmakedefine PLUGIN_PSQLIO
41
#cmakedefine PLUGIN_SASEIO
42

43
#include <meteoio/plugins/ALPUG.h>
44
45
46
47
48
49
50
51
52
53
54
55
56
#include <meteoio/plugins/ARCIO.h>
#include <meteoio/plugins/A3DIO.h>
#include <meteoio/plugins/ARPSIO.h>
#include <meteoio/plugins/GrassIO.h>
#include <meteoio/plugins/GeotopIO.h>
#include <meteoio/plugins/PGMIO.h>
#include <meteoio/plugins/SMETIO.h>
#include <meteoio/plugins/SNIO.h>

#ifdef PLUGIN_BORMAIO
#include <meteoio/plugins/BormaIO.h>
#endif

57
58
59
60
#ifdef PLUGIN_CNRMIO
#include <meteoio/plugins/CNRMIO.h>
#endif

61
62
63
64
65
66
67
68
#ifdef PLUGIN_COSMOXMLIO
#include <meteoio/plugins/CosmoXMLIO.h>
#endif

#ifdef PLUGIN_IMISIO
#include <meteoio/plugins/ImisIO.h>
#endif

69
70
71
72
#ifdef PLUGIN_OSHDIO
#include <meteoio/plugins/OshdIO.h>
#endif

73
74
75
76
77
78
79
80
#ifdef PLUGIN_GRIBIO
#include <meteoio/plugins/GRIBIO.h>
#endif

#ifdef PLUGIN_GSNIO
#include <meteoio/plugins/GSNIO.h>
#endif

81
82
83
84
#ifdef PLUGIN_NETCDFIO
#include <meteoio/plugins/NetCDFIO.h>
#endif

85
86
87
88
#ifdef PLUGIN_PNGIO
#include <meteoio/plugins/PNGIO.h>
#endif

Thomas Egger's avatar
Thomas Egger committed
89
90
91
92
#ifdef PLUGIN_PSQLIO
#include <meteoio/plugins/PSQLIO.h>
#endif

93
94
95
96
#ifdef PLUGIN_SASEIO
#include <meteoio/plugins/SASEIO.h>
#endif

97
98
99
100
using namespace std;

namespace mio {
 /**
101
102
103
104
 * @page data_sources Data input overview
 * The data access is handled by a system of plugins. They all offer the same interface, meaning that a plugin can transparently be replaced by another one. Since they 
 * might rely on third party libraries for accessing the data, they have been created as plugins, that is they are  only compiled if requested when configuring the 
 * compilation with cmake. A plugin can therefore fail to run if it has not been compiled.
105
 *
106
107
 * Please have a look at the support for \subpage coords "coordinate systems".
 * 
108
109
110
111
112
113
 * @section available_categories Data sources categories
 * Several data sources categories have been defined that can be provided by a different plugin. Each data source category is defined by a specific key in the configuration file (usually, io.ini):
 * - METEO, for meteorological time series
 * - DEM, for Digital Elevation Maps
 * - LANDUSE, for land cover information
 * - GRID2D, for generic 2D grids (they can contain meteo fields and be recognized as such or arbitrary gridded data)
114
 * - POI, for a list of Points Of Interest that can be used for providing extra information at some specific location (extracting time series at a few selected points, etc)
115
116
117
118
119
120
121
122
123
124
 *
 * A plugin is "connected" to a given data source category simply by giving its keyword as value for the data source key:
 * @code
 * METEO = SMET
 * DEM = ARC
 * @endcode
 * Each plugin might have its own specific options, meaning that it might require its own keywords. Please check in each plugin documentation the supported options and keys (see links below).
 * Moreover, a given plugin might only support a given category for read or write (for example, PNG: there is no easy and safe way to interpret a given color as a given numeric value without knowing its color scale, so reading a png has been disabled).
 * Finally, the plugins usually don't implement all these categories (for example, ArcGIS file format only describes 2D grids, so the ARC plugin will only deal with 2D grids), so please check what a given plugin implements before connecting it to a specific data source category.
 *
125
 * @subsection available_plugins Available plugins
126
127
128
 * So far the following plugins have been implemented (by keyword for the io.ini key/value config file). Please read the documentation for each plugin in order to know the plugin-specific keywords:
 * <center><table border="1">
 * <tr><th>Plugin keyword</th><th>Provides</th><th>Description</th><th>Extra requirements</th></tr>
129
 * <tr><td>\subpage alpug "ALPUG"</td><td>meteo</td><td>data files generated by the %ALPUG meteo stations</td><td></td></tr>
130
 * <tr><td>\subpage a3d "A3D"</td><td>meteo, poi</td><td>original Alpine3D meteo files</td><td></td></tr>
131
132
133
 * <tr><td>\subpage arc "ARC"</td><td>dem, landuse, grid2d</td><td>ESRI/ARC ascii grid files</td><td></td></tr>
 * <tr><td>\subpage arps "ARPS"</td><td>dem, grid2d</td><td>ARPS ascii formatted grids</td><td></td></tr>
 * <tr><td>\subpage borma "BORMA"</td><td>meteo</td><td>Borma xml meteo files</td><td><A HREF="http://libxmlplusplus.sourceforge.net/">libxml++</A></td></tr>
134
 * <tr><td>\subpage cnrm "CNRM"</td><td>dem, grid2d, meteo</td><td>NetCDF meteorological timeseries following the <A HREF="http://www.cnrm.meteo.fr/?lang=en">CNRM</A> schema</td><td><A HREF="http://www.unidata.ucar.edu/downloads/netcdf/index.jsp">NetCDF-C library</A></td></tr>
135
 * <tr><td>\subpage cosmoxml "COSMOXML"</td><td>meteo</td><td>MeteoSwiss COSMO's postprocessing XML format</td><td><A HREF="http://xmlsoft.org/">libxml2</A></td></tr>
136
137
138
 * <tr><td>\subpage geotop "GEOTOP"</td><td>meteo</td><td>GeoTop meteo files</td><td></td></tr>
 * <tr><td>\subpage grass "GRASS"</td><td>dem, landuse, grid2d</td><td>Grass grid files</td><td></td></tr>
 * <tr><td>\subpage gribio "GRIB"</td><td>meteo, dem, grid2d</td><td>GRIB meteo grid files</td><td><A HREF="http://www.ecmwf.int/products/data/software/grib_api.html">grib-api</A></td></tr>
139
 * <tr><td>\subpage gsn "GSN"</td><td>meteo</td><td>connects to the Global Sensor Network web service interface</td><td><A HREF="http://curl.haxx.se/libcurl/">libcurl</A></td></tr>
140
 * <tr><td>\subpage imis "IMIS"</td><td>meteo</td><td>connects to the IMIS database</td><td><A HREF="http://docs.oracle.com/cd/B12037_01/appdev.101/b10778/introduction.htm">Oracle's OCCI library</A></td></tr>
141
 * <tr><td>\subpage netcdf "NETCDF"</td><td>dem, grid2d</td><td>NetCDF grids</td><td><A HREF="http://www.unidata.ucar.edu/downloads/netcdf/index.jsp">NetCDF-C library</A></td></tr>
142
 * <tr><td>\subpage oshd "OSHD"</td><td>meteo</td><td>OSHD generated binary Matlab files</td><td><A HREF="https://sourceforge.net/projects/matio">libmatio</A></td></tr>
143
144
 * <tr><td>\subpage pgmio "PGM"</td><td>dem, grid2d</td><td>PGM grid files</td><td></td></tr>
 * <tr><td>\subpage pngio "PNG"</td><td>dem, grid2d</td><td>PNG grid files</td><td><A HREF="http://www.libpng.org/pub/png/libpng.html">libpng</A></td></tr>
145
 * <tr><td>\subpage psqlio "PSQL"</td><td>meteo</td><td>connects to PostgreSQL database</td><td><A HREF="http://www.postgresql.org/">PostgreSQL</A>'s libpq</td></tr>
146
 * <tr><td>\subpage sase "SASE"</td><td>meteo</td><td>connects to the SASE database</td><td><A HREF="https://dev.mysql.com/doc/refman/5.0/en/c-api.html">MySQL's C API</A></td></tr>
147
 * <tr><td>\subpage smetio "SMET"</td><td>meteo, poi</td><td>SMET data files</td><td></td></tr>
148
149
150
 * <tr><td>\subpage snowpack "SNOWPACK"</td><td>meteo</td><td>original SNOWPACK meteo files</td><td></td></tr>
 * </table></center>
 *
151
 * @section data_manipulations Raw data editing
152
 * @subsection data_generators Data generators
153
 * It is also possible to duplicate a meteorological parameter as another meteorological parameter. This is done by specifying a COPY key, following the syntax
154
 * new_name::COPY = existing_parameter. For example:
155
 * @code
156
 * VW_avg::COPY = VW
157
158
159
160
 * @endcode
 * This creates a new parameter VW_avg that starts as an exact copy of the raw data of VW, for each station. This newly created parameter is
 * then processed as any other meteorological parameter (thus going through filtering, generic processing, spatial interpolations). This only current
 * limitation is that the parameter providing the raw data must be defined for all stations (even if filled with nodata, this is good enough).
161
162
 *
 * @subsection data_exclusion Data exclusion
163
 * It is possible to exclude specific parameters from given stations (on a per station basis). This is either done by using the station ID (or the '*' wildcard) 
164
165
166
 * followed by "::exclude" as key with a space delimited list of \ref meteoparam "meteorological parameters" to exclude for the station as key.
 * Another possibility is to provide a file containing one station ID per line followed by a space delimited list of \ref meteoparam "meteorological parameters"
 * to exclude for the station (the path to the file can be a relative path and will be properly resolved).
167
168
169
 * 
 * The exact opposite can also be done, excluding ALL parameters except the ones declared with the "::keep" statement (or a file containing one station ID
 * per line followed by a space delimited list of \ref meteoparam "meteorological parameters" to keep for the station).
170
171
 *
 * @code
172
 * WFJ2::EXCLUDE = HS PSUM                       ;inline declaration of parameters exclusion
173
 * KLO3::KEEP = TA RH VW DW                      ;inline declaration of parameters to keep
174
175
 *
 * EXCLUDE_FILE = ../input/meteo/excludes.csv    ;parameters exclusions defined in a separate file
176
 * KEEP_FILE = ../input/meteo/keeps.csv          ;parameters to keep defined in a separate file
177
178
179
180
181
 * @endcode
 *
 * In the second example (relying on a separate file), the file "../input/meteo/excludes.csv" could look like this:
 * @code
 * WFJ2 TA RH
182
 * KLO3 HS PSUM
183
 * @endcode
184
185
186
187
188
189
190
 * 
 * Another example relying on wildcards (the kept/excluded parameters lists are additive):
 * @code
 * *::KEEP = TA RH                               ;all stations will keep TA and RH and reject the other parameters
 * WFJ2::KEEP = HS PSUM                          ;WFJ2 will keep TA and RH as defined above but also HS and PSUM
 *
 * @endcode
191
 *
192
 * @subsection data_merging Data merging
193
194
195
196
 * It is possible to merge different data sets together, with a syntax similar to the Exclude/Keep syntax. This merging occurs <b>after</b> any 
 * EXCLUDE/KEEP commands. This is useful, for example, to provide measurements from different stations that actually share the 
 * same measurement location or to build "composite" station from multiple real stations (in this case, using EXCLUDE and/or KEEP 
 * commands to fine tune how the composite station(s) is/are built). 
197
198
 * Please note that the order of declaration defines the priority (ie the first station that has a value for a given parameter has priority). Please also
 * note that only common timestamps will be merged! (ie if the stations have different sampling rates, it might end up that no merge gets performed)
199
 * 
200
201
202
 * @code
 * STATION1 = *WFJ
 * STATION2 = WFJ2
203
204
 * STATION3 = WFJ1
 * [...]
205
206
207
208
 * 
 * *WFJ::KEEP = ILWR PSUM
 * WFJ2::EXCLUDE = PSUM ILWR RSWR
 * 
209
210
 * *WFJ::MERGE = WFJ2 WFJ1
 * DRB2::MERGE = DRB1 WFJ2
211
 * @endcode
212
213
 * In order to avoid circular dependencies, a station can NOT receive data from a station AND contribute data to another station. Otherwise, a 
 * station can be merged into multiple other stations.
214
215
216
217
218
219
 * 
 * @section virtual_stations_section Virtual stations
 * It is possible to use spatially interpolated meteorological fields or time series of 2D grids to extract meteorological time series for a set of points.
 * This is handled as "virtual stations" since the data will seem to originate from points where no station is present. This is described in the 
 * \subpage virtual_stations "virtual stations" page.
 * 
220
221
 */

222
IOInterface* IOHandler::getPlugin(const std::string& plugin_name) const
223
{
224
225
226
#ifdef PLUGIN_ALPUG
	if (plugin_name == "ALPUG") return new ALPUG(cfg);
#endif
227
#ifdef PLUGIN_ARCIO
228
	if (plugin_name == "ARC") return new ARCIO(cfg);
229
230
#endif
#ifdef PLUGIN_A3DIO
231
	if (plugin_name == "A3D") return new A3DIO(cfg);
232
233
#endif
#ifdef PLUGIN_ARPSIO
234
	if (plugin_name == "ARPS") return new ARPSIO(cfg);
235
236
#endif
#ifdef PLUGIN_GRASSIO
237
	if (plugin_name == "GRASS") return new GrassIO(cfg);
238
239
#endif
#ifdef PLUGIN_GEOTOPIO
240
	if (plugin_name == "GEOTOP") return new GeotopIO(cfg);
241
242
#endif
#ifdef PLUGIN_SMETIO
243
	if (plugin_name == "SMET") return new SMETIO(cfg);
244
245
#endif
#ifdef PLUGIN_SNIO
246
	if (plugin_name == "SNOWPACK") return new SNIO(cfg);
247
248
#endif
#ifdef PLUGIN_PGMIO
249
	if (plugin_name == "PGM") return new PGMIO(cfg);
250
251
#endif
#ifdef PLUGIN_IMISIO
252
	if (plugin_name == "IMIS") return new ImisIO(cfg);
253
#endif
254
255
256
#ifdef PLUGIN_OSHDIO
	if (plugin_name == "OSHD") return new OshdIO(cfg);
#endif
257
#ifdef PLUGIN_GRIBIO
258
	if (plugin_name == "GRIB") return new GRIBIO(cfg);
259
260
#endif
#ifdef PLUGIN_PNGIO
261
	if (plugin_name == "PNG") return new PNGIO(cfg);
262
263
#endif
#ifdef PLUGIN_BORMAIO
264
	if (plugin_name == "BORMA") return new BormaIO(cfg);
265
266
#endif
#ifdef PLUGIN_COSMOXMLIO
267
	if (plugin_name == "COSMOXML") return new CosmoXMLIO(cfg);
268
#endif
269
270
271
#ifdef PLUGIN_CNRMIO
	if (plugin_name == "CNRM") return new CNRMIO(cfg);
#endif
272
#ifdef PLUGIN_GSNIO
273
	if (plugin_name == "GSN") return new GSNIO(cfg);
274
#endif
275
#ifdef PLUGIN_NETCDFIO
276
	if (plugin_name == "NETCDF") return new NetCDFIO(cfg);
277
#endif
Thomas Egger's avatar
Thomas Egger committed
278
#ifdef PLUGIN_PSQLIO
279
	if (plugin_name == "PSQL") return new PSQLIO(cfg);
Thomas Egger's avatar
Thomas Egger committed
280
#endif
281
282
283
#ifdef PLUGIN_SASEIO
	if (plugin_name == "SASE") return new SASEIO(cfg);
#endif
284
285

	return NULL; //no plugin found
286
287
}

288
289
290
//this is actually an object factory
IOInterface* IOHandler::getPlugin(const std::string& cfgkey, const std::string& cfgsection)
{
291
	const std::string op_src = cfg.get(cfgkey, cfgsection);
292
293

	if (mapPlugins.find(op_src) == mapPlugins.end()) {
294
295
		IOInterface *ioPtr = getPlugin(op_src);
		if (ioPtr==NULL)
296
			throw IOException("Cannot find plugin " + op_src + " as requested in file " + cfg.getSourceName() + ". Has it been activated through ccmake? Is it declared in IOHandler::getPlugin?", AT);
297
298
		else
			mapPlugins[op_src] = ioPtr;
299
300
301
302
303
	}

	return mapPlugins[op_src];
}

304
305
//Copy constructor
IOHandler::IOHandler(const IOHandler& aio)
306
307
           : IOInterface(), cfg(aio.cfg), mapPlugins(aio.mapPlugins), excluded_params(aio.excluded_params), kept_params(aio.kept_params), 
             merge_commands(aio.merge_commands), merged_stations(aio.merged_stations),
308
             copy_parameter(aio.copy_parameter), copy_name(aio.copy_name), enable_copying(aio.enable_copying), 
309
             excludes_ready(aio.excludes_ready), keeps_ready(aio.keeps_ready), merge_ready(aio.merge_ready)
Mathias Bavay's avatar
Mathias Bavay committed
310
{}
311

Mathias Bavay's avatar
Mathias Bavay committed
312
IOHandler::IOHandler(const Config& cfgreader)
313
314
315
           : IOInterface(), cfg(cfgreader), mapPlugins(), excluded_params(), kept_params(), 
             merge_commands(), merged_stations(), copy_parameter(), copy_name(), 
             enable_copying(false), excludes_ready(false), keeps_ready(false), merge_ready(false)
316
317
318
319
{
	parse_copy_config();
}

Mathias Bavay's avatar
Mathias Bavay committed
320
321
IOHandler::~IOHandler() throw()
{
322
	// Get rid of the objects
323
324
325
	std::map<std::string, IOInterface*>::iterator mapit;
	for (mapit = mapPlugins.begin(); mapit!=mapPlugins.end(); ++mapit) {
		delete mapit->second;
326
327
328
	}
}

329
IOHandler& IOHandler::operator=(const IOHandler& source) {
330
	if (this != &source) {
331
		mapPlugins = source.mapPlugins;
332
		excluded_params = source.excluded_params;
333
		kept_params = source.kept_params;
334
335
		merge_commands = source.merge_commands;
		merged_stations = source.merged_stations;
336
337
338
		copy_parameter = source.copy_parameter;
		copy_name = source.copy_name;
		enable_copying = source.enable_copying;
339
		excludes_ready = source.excludes_ready;
340
		keeps_ready = source.keeps_ready;
341
		merge_ready = source.merge_ready;
342
343
344
345
	}
	return *this;
}

346
347
348
349
350
351
352
353
354
355
356
357
void IOHandler::read2DGrid(Grid2DObject& grid_out, const std::string& i_filename)
{
	IOInterface *plugin = getPlugin("GRID2D", "Input");
	plugin->read2DGrid(grid_out, i_filename);
}

void IOHandler::read2DGrid(Grid2DObject& grid_out, const MeteoGrids::Parameters& parameter, const Date& date)
{
	IOInterface *plugin = getPlugin("GRID2D", "Input");
	plugin->read2DGrid(grid_out, parameter, date);
}

358
359
360
361
362
363
364
365
366
367
368
369
void IOHandler::read3DGrid(Grid3DObject& grid_out, const std::string& i_filename)
{
	IOInterface *plugin = getPlugin("GRID3D", "Input");
	plugin->read3DGrid(grid_out, i_filename);
}

void IOHandler::read3DGrid(Grid3DObject& grid_out, const MeteoGrids::Parameters& parameter, const Date& date)
{
	IOInterface *plugin = getPlugin("GRID3D", "Input");
	plugin->read3DGrid(grid_out, parameter, date);
}

370
371
372
373
374
375
376
377
378
379
380
381
382
void IOHandler::readDEM(DEMObject& dem_out)
{
	IOInterface *plugin = getPlugin("DEM", "Input");
	plugin->readDEM(dem_out);
	dem_out.update();
}

void IOHandler::readLanduse(Grid2DObject& landuse_out)
{
	IOInterface *plugin = getPlugin("LANDUSE", "Input");
	plugin->readLanduse(landuse_out);
}

383
void IOHandler::readStationData(const Date& date, STATIONS_SET& vecStation)
384
385
386
{
	IOInterface *plugin = getPlugin("METEO", "Input");
	plugin->readStationData(date, vecStation);
387
388
389
	
	if (!merge_ready) create_merge_map(); 
	merge_stations(vecStation);
390
391
392
}

void IOHandler::readMeteoData(const Date& dateStart, const Date& dateEnd,
393
                              std::vector<METEO_SET>& vecMeteo,
394
395
396
397
                              const size_t& stationindex)
{
	IOInterface *plugin = getPlugin("METEO", "Input");
	plugin->readMeteoData(dateStart, dateEnd, vecMeteo, stationindex);
398
	
399
	checkTimestamps(vecMeteo);
400
	
401
402
	if (!excludes_ready) create_exclude_map();
	exclude_params(vecMeteo);
403
404
405
	
	if (!keeps_ready) create_keep_map();
	keep_params(vecMeteo);
406
	
407
408
	if (!merge_ready) create_merge_map(); 
	merge_stations(vecMeteo);
409
	
410
411
	copy_parameters(stationindex, vecMeteo);
}
Mathias Bavay's avatar
Mathias Bavay committed
412

413
void IOHandler::writeMeteoData(const std::vector<METEO_SET>& vecMeteo,
414
415
416
417
418
419
420
421
422
423
424
425
                               const std::string& name)
{
	IOInterface *plugin = getPlugin("METEO", "Output");
	plugin->writeMeteoData(vecMeteo, name);
}

void IOHandler::readAssimilationData(const Date& date_in, Grid2DObject& da_out)
{
	IOInterface *plugin = getPlugin("DA", "Input");
	plugin->readAssimilationData(date_in, da_out);
}

426
427
428
void IOHandler::readPOI(std::vector<Coords>& pts) {
	IOInterface *plugin = getPlugin("POI", "Input");
	plugin->readPOI(pts);
429
430
431
432
433
434
435
436
437
438
439
440
441
442
}

void IOHandler::write2DGrid(const Grid2DObject& grid_in, const std::string& name)
{
	IOInterface *plugin = getPlugin("GRID2D", "Output");
	plugin->write2DGrid(grid_in, name);
}

void IOHandler::write2DGrid(const Grid2DObject& grid_in, const MeteoGrids::Parameters& parameter, const Date& date)
{
	IOInterface *plugin = getPlugin("GRID2D", "Output");
	plugin->write2DGrid(grid_in, parameter, date);
}

443
444
445
446
447
448
449
450
451
452
453
454
void IOHandler::write3DGrid(const Grid3DObject& grid_out, const std::string& options)
{
	IOInterface *plugin = getPlugin("GRID3D", "Output");
	plugin->write3DGrid(grid_out, options);
}

void IOHandler::write3DGrid(const Grid3DObject& grid_out, const MeteoGrids::Parameters& parameter, const Date& date)
{
	IOInterface *plugin = getPlugin("GRID3D", "Output");
	plugin->write3DGrid(grid_out, parameter, date);
}

455
456
457
458
459
460
461
462
463
464
465
466
467
468
/**
* Make sure all timestamps are unique and in increasing order
*/
void IOHandler::checkTimestamps(const std::vector<METEO_SET>& vecVecMeteo) const
{
	for (size_t stat_idx=0; stat_idx<vecVecMeteo.size(); ++stat_idx) { //for each station
		const size_t nr_timestamps = vecVecMeteo[stat_idx].size();
		if (nr_timestamps==0) continue;

		Date previous_date( vecVecMeteo[stat_idx].front().date );
		for (size_t ii=1; ii<nr_timestamps; ++ii) {
			const Date& current_date = vecVecMeteo[stat_idx][ii].date;
			if (current_date<=previous_date) {
				const StationData& station = vecVecMeteo[stat_idx][ii].meta;
469
470
471
472
				if (current_date==previous_date)
					throw IOException("Error for station \""+station.stationName+"\" ("+station.stationID+") at time "+current_date.toString(Date::ISO)+": timestamps must be unique!", AT);
				else
					throw IOException("Error for station \""+station.stationName+"\" ("+station.stationID+"): jumping from "+previous_date.toString(Date::ISO)+" to "+current_date.toString(Date::ISO), AT);
473
474
475
476
477
478
			}
			previous_date = current_date;
		}
	}
}

479
480
481
482
483
484
485
486
487
488
void IOHandler::create_merge_map()
{
	merge_ready = true;
	
	vector<string> merge_keys;
	const size_t nrOfStations = cfg.findKeys(merge_keys, "::MERGE", "Input", true);
	for (size_t ii=0; ii<nrOfStations; ++ii) {
		const size_t found = merge_keys[ii].find_first_of(":");
		if (found==std::string::npos) continue;

489
		const string station( IOUtils::strToUpper(merge_keys[ii].substr(0,found)) );
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
		std::vector<std::string> vecString;
		cfg.getValue(merge_keys[ii], "Input", vecString);
		if (vecString.empty()) throw InvalidArgumentException("Empty value for key \""+merge_keys[ii]+"\"", AT);
		
		for (vector<string>::iterator it = vecString.begin(); it != vecString.end(); ++it) {
			IOUtils::toUpper(*it);
			const std::vector<std::string>::const_iterator vec_it = find (merged_stations.begin(), merged_stations.end(), *it);
			if (vec_it==merged_stations.end()) merged_stations.push_back( *it ); //this station will be merged into another one
		}
		merge_commands[ station ] = vecString;
	}
	
	//sort the merged_stations vector so searches will be faster
	std::sort(merged_stations.begin(), merged_stations.end());
	
	//make sure there is no "chain merge": station A merging station B and station C merging station A
	std::map< std::string, std::vector<std::string> >::iterator it_dest;
	for(it_dest=merge_commands.begin(); it_dest!=merge_commands.end(); ++it_dest) {
		const string &stationID = it_dest->first;
		if (std::binary_search(merged_stations.begin(), merged_stations.end(), stationID))
			throw InvalidArgumentException("\'chain merge\' detected for station \'"+stationID+"\', this is not supported (see documentation)", AT);
	}
}

514
//merge stations that have identical names
515
void IOHandler::merge_stations(STATIONS_SET& vecStation) const
516
{
517
518
	if (merge_commands.empty()) return;
	
519
	for (size_t ii=0; ii<vecStation.size(); ii++) {
520
		const string toStationID( IOUtils::strToUpper( vecStation[ii].stationID ) );
521
522
523
524
525
526
		//we do not support "chain merge": station A merging station B and station C merging station A
		if ( std::find(merged_stations.begin(), merged_stations.end(), toStationID)!=merged_stations.end() ) continue;
		
		const map< string, vector<string> >::const_iterator it = merge_commands.find( toStationID );
		if (it == merge_commands.end()) continue; //no merge commands for this station

527
528
529
		const std::vector<std::string> &merge_from( it->second );
		for (std::vector<std::string>::const_iterator it_set=merge_from.begin(); it_set != merge_from.end(); ++it_set) {
			const string fromStationID( *it_set );
530
			
531
			bool found = false;
532
			for (size_t jj=0; jj<vecStation.size(); jj++) {
533
534
535
536
537
				const string curr_station( IOUtils::strToUpper(vecStation[jj].stationID) );
				if (curr_station==fromStationID) {
					vecStation[ii].merge( vecStation[jj] );
					found = true;
				}
538
			}
539
540
			if (!found)
				throw InvalidArgumentException("Station ID '"+fromStationID+"' not found when merging toward station '"+toStationID+"'", AT);
541
542
		}
	}
543
544
545
	
	//remove the stations that have been merged into other ones
	for (size_t ii=0; ii<vecStation.size(); ii++) {
546
547
		const string stationID( IOUtils::strToUpper( vecStation[ii].stationID ) );
		const vector<string>::const_iterator it = std::find(merged_stations.begin(), merged_stations.end(), stationID);
548
549
550
551
552
553
		if ( it!=merged_stations.end() ) {
			std::swap( vecStation[ii], vecStation.back() );
			vecStation.pop_back();
			ii--; //in case we have multiple identical stations ID
		}
	}
554
555
556
}

//in this implementation, we consider that the station name does NOT change over time
557
void IOHandler::merge_stations(std::vector<METEO_SET>& vecVecMeteo) const
558
{
559
560
	if (merge_commands.empty()) return;
	
561
	for (size_t ii=0; ii<vecVecMeteo.size(); ii++) { //loop over the stations
562
		if (vecVecMeteo[ii].empty())  continue;
563
		const string toStationID( IOUtils::strToUpper(vecVecMeteo[ii][0].meta.stationID) );
564
565
566
567
568
569
		//we do not support "chain merge": station A merging station B and station C merging station A
		if ( std::find(merged_stations.begin(), merged_stations.end(), toStationID)!=merged_stations.end() ) continue;
		
		const map< string, vector<string> >::const_iterator it = merge_commands.find( toStationID );
		if (it == merge_commands.end()) continue; //no merge commands for this station

570
571
572
573
574
		const std::vector<std::string> &merge_from( it->second );
		for (std::vector<std::string>::const_iterator it_set=merge_from.begin(); it_set != merge_from.end(); ++it_set) {
			const string fromStationID( *it_set );
			
			bool found = false;
575
			for (size_t jj=0; jj<vecVecMeteo.size(); jj++) { //loop over the available stations in the current dataset
576
				if (vecVecMeteo[jj].empty()) continue;
577
578
				const string curr_station( IOUtils::strToUpper(vecVecMeteo[jj][0].meta.stationID) );
				if (curr_station==fromStationID) {
579
					MeteoData::mergeTimeSeries(vecVecMeteo[ii], vecVecMeteo[jj]);
580
581
					found = true;
				}
582
			}
583
584
			if (!found)
				throw InvalidArgumentException("Station ID '"+fromStationID+"' not found when merging toward station '"+toStationID+"'", AT);
585
586
		}
	}
587
588
589
	
	//remove the stations that have been merged into other ones
	for (size_t ii=0; ii<vecVecMeteo.size(); ii++) {
590
		if (vecVecMeteo[ii].empty())  continue;
591
592
		const string stationID( IOUtils::strToUpper(vecVecMeteo[ii][0].meta.stationID) );
		const vector<string>::const_iterator it = std::find(merged_stations.begin(), merged_stations.end(), stationID);
593
594
595
596
597
598
		if ( it!=merged_stations.end() ) {
			std::swap( vecVecMeteo[ii], vecVecMeteo.back() );
			vecVecMeteo.pop_back();
			ii--; //in case we have multiple identical stations ID
		}
	}
599
600
}

601
602
void IOHandler::create_exclude_map()
{
603
	excludes_ready = true;
604
	const string exclude_file = cfg.get("EXCLUDE_FILE", "Input", IOUtils::nothrow);
605

606
607
608
609
610
	if (!exclude_file.empty()) {
		//if this is a relative path, prefix the path with the current path
		const std::string prefix = ( IOUtils::isAbsolutePath(exclude_file) )? "" : cfg.getConfigRootDir()+"/";
		const std::string path = IOUtils::getPath(prefix+exclude_file, true);  //clean & resolve path
		const std::string filename = path + "/" + IOUtils::getFilename(exclude_file);
611

612
		if (!IOUtils::fileExists(filename)) throw AccessException(filename, AT); //prevent invalid filenames
613
		std::ifstream fin(filename.c_str(), std::ifstream::in);
614
		if (fin.fail()) throw AccessException(filename, AT);
615

616
617
		try {
			const char eoln = IOUtils::getEoln(fin); //get the end of line character for the file
618

619
620
			vector<string> tmpvec;
			string line;
621

622
623
624
625
			while (!fin.eof()) { //Go through file
				getline(fin, line, eoln); //read complete line meta information
				IOUtils::stripComments(line);
				const size_t ncols = IOUtils::readLineToVec(line, tmpvec, ' ');
626

627
				if (ncols > 1) {
628
					for (vector<string>::iterator it = tmpvec.begin()+1; it != tmpvec.end(); ++it) {
629
630
						IOUtils::toUpper(*it);
					}
631

632
					const std::set<std::string> tmpset(tmpvec.begin()+1, tmpvec.end());
633
					excluded_params[ tmpvec[0] ] = tmpset;
634
635
				}
			}
636
637
638
		} catch (const std::exception&) {
			fin.close();
			throw;
639
		}
640

641
642
643
		fin.close();
	}

644
645
646
647
648
649
	vector<string> exclude_keys;
	const size_t nrOfStations = cfg.findKeys(exclude_keys, "::EXCLUDE", "Input", true);
	for (size_t ii=0; ii<nrOfStations; ++ii) {
		const size_t found = exclude_keys[ii].find_first_of(":");
		if (found==std::string::npos) continue;

650
		const std::string station( IOUtils::strToUpper(exclude_keys[ii].substr(0,found)) );
651
652
653
		std::vector<std::string> vecString;
		cfg.getValue(exclude_keys[ii], "Input", vecString);
		if (vecString.empty()) throw InvalidArgumentException("Empty value for key \""+exclude_keys[ii]+"\"", AT);
654
		for (vector<string>::iterator it = vecString.begin(); it != vecString.end(); ++it) {
655
656
657
			IOUtils::toUpper(*it);
		}

658
		const std::set<std::string> tmpset(vecString.begin(), vecString.end());
659
660
		excluded_params[ station ] = tmpset;
	}
661
662
663
664
665
666
667
668
669
670
671
672
673
674
	
	//Handle "*" wildcard: add the params to all other declared stations
	map< string, set<string> >::const_iterator it_station = excluded_params.find("*");
	if (it_station!=excluded_params.end()) {
		const std::set<std::string> wildcard( excluded_params["*"] );
		for (it_station=excluded_params.begin(); it_station!=excluded_params.end(); ++it_station) {
			std::set<std::string> params( it_station->second );
			
			for (std::set<std::string>::iterator it=wildcard.begin(); it!=wildcard.end(); ++it)
				params.insert( *it ); //merging: keep in mind that a set can not contain duplicates
			
			excluded_params[ it_station->first ] = params;
		}
	}
675
676
}

677
678
679
void IOHandler::create_keep_map()
{
	keeps_ready = true;
680
	const string keep_file = cfg.get("KEEP_FILE", "Input", IOUtils::nothrow);
681
682
683
684
685
686
687

	if (!keep_file.empty()) {
		//if this is a relative path, prefix the path with the current path
		const std::string prefix = ( IOUtils::isAbsolutePath(keep_file) )? "" : cfg.getConfigRootDir()+"/";
		const std::string path = IOUtils::getPath(prefix+keep_file, true);  //clean & resolve path
		const std::string filename = path + "/" + IOUtils::getFilename(keep_file);

688
		if (!IOUtils::fileExists(filename)) throw AccessException(filename, AT); //prevent invalid filenames
689
		std::ifstream fin(filename.c_str(), std::ifstream::in);
690
		if (fin.fail()) throw AccessException(filename, AT);
691
692
693
694
695
696
697
698
699
700
701
702
703

		try {
			const char eoln = IOUtils::getEoln(fin); //get the end of line character for the file

			vector<string> tmpvec;
			string line;

			while (!fin.eof()) { //Go through file
				getline(fin, line, eoln); //read complete line meta information
				IOUtils::stripComments(line);
				const size_t ncols = IOUtils::readLineToVec(line, tmpvec, ' ');

				if (ncols > 1) {
704
					for (vector<string>::iterator it = tmpvec.begin()+1; it != tmpvec.end(); ++it) {
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
						IOUtils::toUpper(*it);
					}

					const set<string> tmpset(tmpvec.begin()+1, tmpvec.end());
					kept_params[ tmpvec[0] ] = tmpset;
				}
			}
		} catch (const std::exception&) {
			fin.close();
			throw;
		}

		fin.close();
	}

	vector<string> keep_keys;
	const size_t nrOfStations = cfg.findKeys(keep_keys, "::KEEP", "Input", true);
	for (size_t ii=0; ii<nrOfStations; ++ii) {
		const size_t found = keep_keys[ii].find_first_of(":");
		if (found==std::string::npos) continue;

726
		const std::string station( IOUtils::strToUpper(keep_keys[ii].substr(0,found)) );
727
728
729
		std::vector<std::string> vecString;
		cfg.getValue(keep_keys[ii], "Input", vecString);
		if (vecString.empty()) throw InvalidArgumentException("Empty value for key \""+keep_keys[ii]+"\"", AT);
730
		for (vector<string>::iterator it = vecString.begin(); it != vecString.end(); ++it) {
731
732
733
			IOUtils::toUpper(*it);
		}

734
		const std::set<std::string> tmpset(vecString.begin(), vecString.end());
735
736
		kept_params[ station ] = tmpset;
	}
737
738
739
740
741
742
743
744
745
746
747
748
749
750
	
	//Handle "*" wildcard: add the params to all other declared stations
	map< string, set<string> >::const_iterator it_station = kept_params.find("*");
	if (it_station!=kept_params.end()) {
		const std::set<std::string> wildcard( kept_params["*"] );
		for (it_station=kept_params.begin(); it_station!=kept_params.end(); ++it_station) {
			std::set<std::string> params( it_station->second );
			
			for (std::set<std::string>::iterator it=wildcard.begin(); it!=wildcard.end(); ++it)
				params.insert( *it ); //merging: keep in mind that a set can not contain duplicates
			
			kept_params[ it_station->first ] = params;
		}
	}
751
752
}

753
/**
754
* @brief reset to nodata the parameters marked as EXCLUDE on a per station basis
755
*/
756
757
758
759
void IOHandler::exclude_params(std::vector<METEO_SET>& vecVecMeteo) const
{
	if (excluded_params.empty()) return;

760
	for (size_t station=0; station<vecVecMeteo.size(); ++station) { //loop over the stations
761
		if (vecVecMeteo[station].empty()) continue;
762
763
764
765
766
767
		const std::string stationID( IOUtils::strToUpper(vecVecMeteo[station][0].meta.stationID) );
		map< string, set<string> >::const_iterator it = excluded_params.find(stationID);
		if (it == excluded_params.end()) {
			it = excluded_params.find("*"); //fallback: is there a wildcard like "*::KEEP"?
			if (it == excluded_params.end()) continue;
		}
768
769
770

		const set<string> excluded = it->second;

771
		for (size_t ii=0; ii<vecVecMeteo[station].size(); ++ii) { //loop over the timesteps
772
773
			for (std::set<std::string>::const_iterator it_set=excluded.begin(); it_set != excluded.end(); ++it_set) {
				const string param( *it_set );
774
775
776
777
778
779
780
				if (vecVecMeteo[station][ii].param_exists(param))
					vecVecMeteo[station][ii](param) = IOUtils::nodata;
			}
		}
	}
}

781
782
783
784
785
786
787
788
789
790
/**
* @brief only keep the parameters marked as KEEP on a per station basis
*/
void IOHandler::keep_params(std::vector<METEO_SET>& vecVecMeteo) const
{
	if (kept_params.empty()) return;

	for (size_t station=0; station<vecVecMeteo.size(); ++station) { //loop over the stations
		if (vecVecMeteo[station].empty()) continue;
		
791
792
793
794
795
796
		const std::string stationID( IOUtils::strToUpper(vecVecMeteo[station][0].meta.stationID) );
		map< string, set<string> >::const_iterator it = kept_params.find(stationID);
		if (it == kept_params.end()) {
			it = kept_params.find("*"); //fallback: is there a wildcard like "*::KEEP"?
			if (it == kept_params.end()) continue;
		}
797
798
799
800
801
802
803
804

		const set<string> kept = it->second;
		
		for (size_t ii=0; ii<vecVecMeteo[station].size(); ++ii) {
			MeteoData& md_ref = vecVecMeteo[station][ii];
			MeteoData md( md_ref );
			md.reset(); //delete all meteo fields
			
805
806
			for (std::set<std::string>::const_iterator it_set=kept.begin(); it_set != kept.end(); ++it_set) { //loop over the parameters to keep
				const string param( *it_set);
807
808
809
810
811
812
813
814
815
816
				if (!md.param_exists(param)) continue;
				 md(param) = md_ref(param);
			}
			
			//copy back the new object into vecVecMeteo
			md_ref = md;
		}
	}
}

817
818
819
820
/**
* Parse [Input] section for potential parameters that the user wants
* duplicated (as '%%::COPY = %%')
*/
821
822
823
void IOHandler::parse_copy_config()
{
	vector<string> copy_keys;
824
	const size_t nrOfMatches = cfg.findKeys(copy_keys, "::COPY", "Input", true); //search anywhere in key
825

826
	for (size_t ii=0; ii<nrOfMatches; ++ii) {
827
		const string name_of_copy = copy_keys[ii].substr( 0, copy_keys[ii].find_first_of(":") );
828
		const string initial_name = cfg.get(copy_keys[ii], "Input");
829
830
831
832
833
834
835
836
		if ((name_of_copy.length() > 0) && (initial_name.length() > 0)){
			copy_parameter.push_back(initial_name);
			copy_name.push_back(name_of_copy);
			enable_copying = true;
		}
	}
}

837
838
839
840
841
/**
* This procedure runs through the MeteoData objects in vecMeteo and according to user
* configuration copies a certain present meteo parameter to another one, named by the
* user in the [Input] section of the io.ini, e.g.
* [Input]
842
* TA2::COPY = TA
843
844
845
* means that TA2 will be the name of a new parameter in MeteoData with the copied value
* of the meteo parameter MeteoData::TA
*/
846
void IOHandler::copy_parameters(const size_t& stationindex, std::vector< METEO_SET >& vecMeteo) const
847
848
849
{
	if (!enable_copying) return; //Nothing configured

850
851
	if (stationindex != IOUtils::npos && stationindex>=vecMeteo.size())
		throw IndexOutOfBoundsException("Accessing stationindex in readMeteoData that is out of bounds", AT);
852

853
854
	const size_t station_start = (stationindex==IOUtils::npos)? 0 : stationindex;
	const size_t station_end = (stationindex==IOUtils::npos)? vecMeteo.size() : stationindex+1;
855
856
857
	const size_t nr_of_params = copy_parameter.size();
	vector<size_t> indices; //will hold the indices of the parameters to be copied

858
859
	for (size_t ii=station_start; ii<station_end; ++ii) { //for each station
		for (size_t jj=0; jj<vecMeteo[ii].size(); ++jj) { //for each MeteoData object of one station
860
861

			if (jj==0) { //buffer the index numbers
862
				for (size_t kk=0; kk<nr_of_params; ++kk) {
863
864
					const size_t param_index = vecMeteo[ii][jj].getParameterIndex(copy_parameter[kk]);
					if (param_index == IOUtils::npos) {
865
						std::ostringstream ss;
866
867
868
869
870
871
872
873
874
						ss << "At " << vecMeteo[ii][jj].date.toString(Date::ISO) << ", station " << vecMeteo[ii][jj].meta.stationID;
						ss << " has no parameter \"" << copy_parameter[kk] << "\" to copy!\n";
						throw InvalidArgumentException(ss.str(), AT);
					}

					indices.push_back(param_index);
				}
			}

875
			for (size_t kk=0; kk<nr_of_params; ++kk) {
876
877
878
879
880
881
882
883
				const size_t newparam = vecMeteo[ii][jj].addParameter(copy_name[kk]);
				vecMeteo[ii][jj](newparam) = vecMeteo[ii][jj](indices[kk]);
			}
		}
		indices.clear(); //may change for every station
	}
}

884
const std::string IOHandler::toString() const
885
{
886
	std::ostringstream os;
887
888
889
890
	os << "<IOHandler>\n";
	os << "Config& cfg = " << hex << &cfg << dec << "\n";

	os << "<mapPlugins>\n";
891
892
893
	std::map<std::string, IOInterface*>::const_iterator it1;
	for (it1=mapPlugins.begin(); it1 != mapPlugins.end(); ++it1) {
		os << setw(10) << it1->first << " = " << hex <<  it1->second << dec << "\n";
894
895
	}
	os << "</mapPlugins>\n";
896

897
898
899
900
901
902
903
904
905
906
907
	if (!excluded_params.empty()) {
		os << "<excluded_params>\n";
		std::map< std::string, std::set<std::string> >::const_iterator it_exc;
		for (it_exc=excluded_params.begin(); it_exc != excluded_params.end(); ++it_exc) {
			os << setw(10) << it_exc->first << " = ";
			std::set<std::string>::const_iterator it_set;
			for (it_set=(it_exc->second).begin(); it_set != (it_exc->second).end(); ++it_set)
				os << *it_set << " ";
			os << "\n";
		}
		os << "</excluded_params>\n";
908
	}
909
	
910
911
912
913
914
915
916
917
918
919
920
	if (!merge_commands.empty()) {
		os << "<merge_commands>\n";
		std::map< std::string, std::vector<std::string> >::const_iterator it_merge;
		for (it_merge=merge_commands.begin(); it_merge != merge_commands.end(); ++it_merge) {
			os << setw(10) << it_merge->first << " <- ";
			for (size_t ii=0; ii<it_merge->second.size(); ++ii)
				os << it_merge->second[ii] << " ";
			os << "\n";
		}
		os << "</merge_commands>\n";
	}
921

922
923
924
925
926
	os << "</IOHandler>\n";
	return os.str();
}

} //end namespace