WSL/SLF GitLab Repository

IOHandler.cmake.cc 16.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
/***********************************************************************************/
/*  Copyright 2009-2012 WSL Institute for Snow and Avalanche Research  SLF-DAVOS   */
/***********************************************************************************/
/* This file is part of MeteoIO.
    MeteoIO is free software: you can redistribute it and/or modify
    it under the terms of the GNU Lesser General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    MeteoIO is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public License
    along with MeteoIO.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <meteoio/IOHandler.h>

#cmakedefine PLUGIN_ARCIO
#cmakedefine PLUGIN_A3DIO
#cmakedefine PLUGIN_ARPSIO
#cmakedefine PLUGIN_GRASSIO
#cmakedefine PLUGIN_GEOTOPIO
#cmakedefine PLUGIN_SMETIO
#cmakedefine PLUGIN_SNIO
#cmakedefine PLUGIN_PGMIO
#cmakedefine PLUGIN_IMISIO
#cmakedefine PLUGIN_GRIBIO
#cmakedefine PLUGIN_PNGIO
#cmakedefine PLUGIN_BORMAIO
#cmakedefine PLUGIN_COSMOXMLIO
#cmakedefine PLUGIN_GSNIO
35
#cmakedefine PLUGIN_NETCDFIO
Thomas Egger's avatar
Thomas Egger committed
36
#cmakedefine PLUGIN_PSQLIO
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

#include <meteoio/plugins/ARCIO.h>
#include <meteoio/plugins/A3DIO.h>
#include <meteoio/plugins/ARPSIO.h>
#include <meteoio/plugins/GrassIO.h>
#include <meteoio/plugins/GeotopIO.h>
#include <meteoio/plugins/PGMIO.h>
#include <meteoio/plugins/SMETIO.h>
#include <meteoio/plugins/SNIO.h>

#ifdef PLUGIN_BORMAIO
#include <meteoio/plugins/BormaIO.h>
#endif

#ifdef PLUGIN_COSMOXMLIO
#include <meteoio/plugins/CosmoXMLIO.h>
#endif

#ifdef PLUGIN_IMISIO
#include <meteoio/plugins/ImisIO.h>
#endif

#ifdef PLUGIN_GRIBIO
#include <meteoio/plugins/GRIBIO.h>
#endif

#ifdef PLUGIN_GSNIO
#include <meteoio/plugins/GSNIO.h>
#endif

67
68
69
70
#ifdef PLUGIN_NETCDFIO
#include <meteoio/plugins/NetCDFIO.h>
#endif

71
72
73
74
#ifdef PLUGIN_PNGIO
#include <meteoio/plugins/PNGIO.h>
#endif

Thomas Egger's avatar
Thomas Egger committed
75
76
77
78
#ifdef PLUGIN_PSQLIO
#include <meteoio/plugins/PSQLIO.h>
#endif

79
80
81
82
83
84
85
86
87
88
89
90
91
using namespace std;

namespace mio {
 /**
 * @page plugins Plugins overview
 * The data access is handled by a system of plugins. They all offer the same interface, meaning that a plugin can transparently be replaced by another one. Since they might rely on third party libraries for accessing the data, they have been created as plugins, that is they are loaded on demand (and also compiled only if requested at compile time). A plugin can therefore fail to load (for example if it does not exist) at run time.
 *
 * @section available_categories Data sources categories
 * Several data sources categories have been defined that can be provided by a different plugin. Each data source category is defined by a specific key in the configuration file (usually, io.ini):
 * - METEO, for meteorological time series
 * - DEM, for Digital Elevation Maps
 * - LANDUSE, for land cover information
 * - GRID2D, for generic 2D grids (they can contain meteo fields and be recognized as such or arbitrary gridded data)
92
 * - POI, for a list of Points Of Interest that can be used for providing extra information at some specific location (extracting time series at a few selected points, etc)
93
94
95
96
97
98
99
100
101
102
103
104
105
106
 *
 * A plugin is "connected" to a given data source category simply by giving its keyword as value for the data source key:
 * @code
 * METEO = SMET
 * DEM = ARC
 * @endcode
 * Each plugin might have its own specific options, meaning that it might require its own keywords. Please check in each plugin documentation the supported options and keys (see links below).
 * Moreover, a given plugin might only support a given category for read or write (for example, PNG: there is no easy and safe way to interpret a given color as a given numeric value without knowing its color scale, so reading a png has been disabled).
 * Finally, the plugins usually don't implement all these categories (for example, ArcGIS file format only describes 2D grids, so the ARC plugin will only deal with 2D grids), so please check what a given plugin implements before connecting it to a specific data source category.
 *
 * @section available_plugins Available plugins
 * So far the following plugins have been implemented (by keyword for the io.ini key/value config file). Please read the documentation for each plugin in order to know the plugin-specific keywords:
 * <center><table border="1">
 * <tr><th>Plugin keyword</th><th>Provides</th><th>Description</th><th>Extra requirements</th></tr>
107
 * <tr><td>\subpage a3d "A3D"</td><td>meteo, poi</td><td>original Alpine3D meteo files</td><td></td></tr>
108
109
110
 * <tr><td>\subpage arc "ARC"</td><td>dem, landuse, grid2d</td><td>ESRI/ARC ascii grid files</td><td></td></tr>
 * <tr><td>\subpage arps "ARPS"</td><td>dem, grid2d</td><td>ARPS ascii formatted grids</td><td></td></tr>
 * <tr><td>\subpage borma "BORMA"</td><td>meteo</td><td>Borma xml meteo files</td><td><A HREF="http://libxmlplusplus.sourceforge.net/">libxml++</A></td></tr>
111
 * <tr><td>\subpage cosmoxml "COSMOXML"</td><td>meteo</td><td>MeteoSwiss COSMO's postprocessing XML format</td><td><A HREF="http://xmlsoft.org/">libxml2</A></td></tr>
112
113
114
 * <tr><td>\subpage geotop "GEOTOP"</td><td>meteo</td><td>GeoTop meteo files</td><td></td></tr>
 * <tr><td>\subpage grass "GRASS"</td><td>dem, landuse, grid2d</td><td>Grass grid files</td><td></td></tr>
 * <tr><td>\subpage gribio "GRIB"</td><td>meteo, dem, grid2d</td><td>GRIB meteo grid files</td><td><A HREF="http://www.ecmwf.int/products/data/software/grib_api.html">grib-api</A></td></tr>
115
 * <tr><td>\subpage gsn "GSN"</td><td>meteo</td><td>connects to the Global Sensor Network web service interface</td><td><A HREF="http://curl.haxx.se/libcurl/">libcurl</A></td></tr>
116
 * <tr><td>\subpage imis "IMIS"</td><td>meteo</td><td>connects to the IMIS database</td><td><A HREF="http://docs.oracle.com/cd/B12037_01/appdev.101/b10778/introduction.htm">Oracle's OCCI library</A></td></tr>
117
 * <tr><td>\subpage netcdf "NETCDF"</td><td>meteo, dem, grid2d</td><td>NetCDF grids and meteorological timeseries</td><td><A HREF="http://www.unidata.ucar.edu/downloads/netcdf/index.jsp">NetCDF-C library</A></td></tr>
118
119
 * <tr><td>\subpage pgmio "PGM"</td><td>dem, grid2d</td><td>PGM grid files</td><td></td></tr>
 * <tr><td>\subpage pngio "PNG"</td><td>dem, grid2d</td><td>PNG grid files</td><td><A HREF="http://www.libpng.org/pub/png/libpng.html">libpng</A></td></tr>
120
 * <tr><td>\subpage psqlio "PSQL"</td><td>meteo</td><td>connects to PostgreSQL database</td><td><A HREF="http://www.postgresql.org/">PostgreSQL</A>'s libpq</td></tr>
121
 * <tr><td>\subpage smetio "SMET"</td><td>meteo, poi</td><td>SMET data files</td><td></td></tr>
122
123
124
125
126
 * <tr><td>\subpage snowpack "SNOWPACK"</td><td>meteo</td><td>original SNOWPACK meteo files</td><td></td></tr>
 * </table></center>
 *
 * @section data_generators Data generators
 * It is also possible to duplicate a meteorological parameter as another meteorological parameter. This is done by specifying a COPY key, following the syntax
127
 * new_name::COPY = existing_parameter. For example:
128
 * @code
129
 * VW_avg::COPY = VW
130
131
132
133
134
135
 * @endcode
 * This creates a new parameter VW_avg that starts as an exact copy of the raw data of VW, for each station. This newly created parameter is
 * then processed as any other meteorological parameter (thus going through filtering, generic processing, spatial interpolations). This only current
 * limitation is that the parameter providing the raw data must be defined for all stations (even if filled with nodata, this is good enough).
 */

136
IOInterface* IOHandler::getPlugin(const std::string& plugin_name) const
137
138
{
#ifdef PLUGIN_ARCIO
139
	if (plugin_name == "ARC") return new ARCIO(cfg);
140
141
#endif
#ifdef PLUGIN_A3DIO
142
	if (plugin_name == "A3D") return new A3DIO(cfg);
143
144
#endif
#ifdef PLUGIN_ARPSIO
145
	if (plugin_name == "ARPS") return new ARPSIO(cfg);
146
147
#endif
#ifdef PLUGIN_GRASSIO
148
	if (plugin_name == "GRASS") return new GrassIO(cfg);
149
150
#endif
#ifdef PLUGIN_GEOTOPIO
151
	if (plugin_name == "GEOTOP") return new GeotopIO(cfg);
152
153
#endif
#ifdef PLUGIN_SMETIO
154
	if (plugin_name == "SMET") return new SMETIO(cfg);
155
156
#endif
#ifdef PLUGIN_SNIO
157
	if (plugin_name == "SNOWPACK") return new SNIO(cfg);
158
159
#endif
#ifdef PLUGIN_PGMIO
160
	if (plugin_name == "PGM") return new PGMIO(cfg);
161
162
#endif
#ifdef PLUGIN_IMISIO
163
	if (plugin_name == "IMIS") return new ImisIO(cfg);
164
165
#endif
#ifdef PLUGIN_GRIBIO
166
	if (plugin_name == "GRIB") return new GRIBIO(cfg);
167
168
#endif
#ifdef PLUGIN_PNGIO
169
	if (plugin_name == "PNG") return new PNGIO(cfg);
170
171
#endif
#ifdef PLUGIN_BORMAIO
172
	if (plugin_name == "BORMA") return new BormaIO(cfg);
173
174
#endif
#ifdef PLUGIN_COSMOXMLIO
175
	if (plugin_name == "COSMOXML") return new CosmoXMLIO(cfg);
176
177
#endif
#ifdef PLUGIN_GSNIO
178
	if (plugin_name == "GSN") return new GSNIO(cfg);
179
#endif
180
#ifdef PLUGIN_NETCDFIO
181
	if (plugin_name == "NETCDF") return new NetCDFIO(cfg);
182
#endif
Thomas Egger's avatar
Thomas Egger committed
183
#ifdef PLUGIN_PSQLIO
184
	if (plugin_name == "PSQL") return new PSQLIO(cfg);
Thomas Egger's avatar
Thomas Egger committed
185
#endif
186
187

	return NULL; //no plugin found
188
189
190
191
}

//Copy constructor
IOHandler::IOHandler(const IOHandler& aio)
192
193
           : IOInterface(), cfg(aio.cfg), mapPlugins(aio.mapPlugins), copy_parameter(aio.copy_parameter),
             copy_name(aio.copy_name), enable_copying(aio.enable_copying)
Mathias Bavay's avatar
Mathias Bavay committed
194
{}
195

Mathias Bavay's avatar
Mathias Bavay committed
196
197
IOHandler::IOHandler(const Config& cfgreader)
           : IOInterface(), cfg(cfgreader), mapPlugins(), copy_parameter(), copy_name(), enable_copying(false)
198
199
200
201
{
	parse_copy_config();
}

Mathias Bavay's avatar
Mathias Bavay committed
202
203
IOHandler::~IOHandler() throw()
{
204
	// Get rid of the objects
205
206
207
	std::map<std::string, IOInterface*>::iterator mapit;
	for (mapit = mapPlugins.begin(); mapit!=mapPlugins.end(); ++mapit) {
		delete mapit->second;
208
209
210
	}
}

211
212
213
214
215
216
217
218
219
220
IOHandler& IOHandler::operator=(const IOHandler& source) {
	if(this != &source) {
		mapPlugins = source.mapPlugins;
		copy_parameter = source.copy_parameter;
		copy_name = source.copy_name;
		enable_copying = source.enable_copying;
	}
	return *this;
}

221
222
IOInterface* IOHandler::getPlugin(const std::string& cfgkey, const std::string& cfgsection)
{
223
	std::string op_src;
224
225
	cfg.getValue(cfgkey, cfgsection, op_src);

226
227
228
229
	if (mapPlugins.find(op_src) == mapPlugins.end()) {
		mapPlugins[op_src] = getPlugin(op_src);
		if (mapPlugins[op_src]==NULL)
			throw IOException("Cannot find plugin " + op_src + " as requested in file " + cfg.getSourceName() + ". Has it been activated through ccmake? Is it registered in IOHandler::registerPlugins?", AT);
230
231
	}

232
	return mapPlugins[op_src];
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
}

void IOHandler::read2DGrid(Grid2DObject& grid_out, const std::string& i_filename)
{
	IOInterface *plugin = getPlugin("GRID2D", "Input");
	plugin->read2DGrid(grid_out, i_filename);
}

void IOHandler::read2DGrid(Grid2DObject& grid_out, const MeteoGrids::Parameters& parameter, const Date& date)
{
	IOInterface *plugin = getPlugin("GRID2D", "Input");
	plugin->read2DGrid(grid_out, parameter, date);
}

void IOHandler::readDEM(DEMObject& dem_out)
{
	IOInterface *plugin = getPlugin("DEM", "Input");
	plugin->readDEM(dem_out);
	dem_out.update();
}

void IOHandler::readLanduse(Grid2DObject& landuse_out)
{
	IOInterface *plugin = getPlugin("LANDUSE", "Input");
	plugin->readLanduse(landuse_out);
}

260
void IOHandler::readStationData(const Date& date, STATIONS_SET& vecStation)
261
262
263
264
265
{
	IOInterface *plugin = getPlugin("METEO", "Input");
	plugin->readStationData(date, vecStation);
}

266
void IOHandler::readMeteoData(const Date& date, METEO_SET& vecMeteo)
267
268
269
270
271
{
	std::vector< std::vector<MeteoData> > meteoTmpBuffer;
	readMeteoData(date, date, meteoTmpBuffer);

	vecMeteo.clear();
272
273
	vecMeteo.reserve( meteoTmpBuffer.size() );

274
	for (size_t ii=0; ii<meteoTmpBuffer.size(); ++ii) {//stations
275
276
		if (!meteoTmpBuffer[ii].empty())
			vecMeteo.push_back( meteoTmpBuffer[ii].front() );
277
	}
278
}
279

280
281
void IOHandler::checkTimestamps(const std::vector<METEO_SET>& vecVecMeteo) const
{
282
	for (size_t stat_idx=0; stat_idx<vecVecMeteo.size(); ++stat_idx) { //for each station
283
		const size_t nr_timestamps = vecVecMeteo[stat_idx].size();
284
		if (nr_timestamps==0) continue;
285

286
287
		Date previous_date( vecVecMeteo[stat_idx].front().date );
		for (size_t ii=1; ii<nr_timestamps; ++ii) {
288
			const Date& current_date = vecVecMeteo[stat_idx][ii].date;
289
290
			if (current_date<=previous_date) {
				const StationData& station = vecVecMeteo[stat_idx][ii].meta;
291
292
293
294
				throw IOException("Error at time "+current_date.toString(Date::ISO)+" for station \""+station.stationName+"\" ("+station.stationID+") : timestamps must be in increasing order and unique!", AT);
			}
			previous_date = current_date;
		}
295
296
297
298
	}
}

void IOHandler::readMeteoData(const Date& dateStart, const Date& dateEnd,
299
                              std::vector<METEO_SET>& vecMeteo,
300
301
302
303
                              const size_t& stationindex)
{
	IOInterface *plugin = getPlugin("METEO", "Input");
	plugin->readMeteoData(dateStart, dateEnd, vecMeteo, stationindex);
304
	checkTimestamps(vecMeteo);
305
306
307

	copy_parameters(stationindex, vecMeteo);
}
Mathias Bavay's avatar
Mathias Bavay committed
308

309
void IOHandler::writeMeteoData(const std::vector<METEO_SET>& vecMeteo,
310
311
312
313
314
315
316
317
318
319
320
321
                               const std::string& name)
{
	IOInterface *plugin = getPlugin("METEO", "Output");
	plugin->writeMeteoData(vecMeteo, name);
}

void IOHandler::readAssimilationData(const Date& date_in, Grid2DObject& da_out)
{
	IOInterface *plugin = getPlugin("DA", "Input");
	plugin->readAssimilationData(date_in, da_out);
}

322
323
324
void IOHandler::readPOI(std::vector<Coords>& pts) {
	IOInterface *plugin = getPlugin("POI", "Input");
	plugin->readPOI(pts);
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
}

void IOHandler::write2DGrid(const Grid2DObject& grid_in, const std::string& name)
{
	IOInterface *plugin = getPlugin("GRID2D", "Output");
	plugin->write2DGrid(grid_in, name);
}

void IOHandler::write2DGrid(const Grid2DObject& grid_in, const MeteoGrids::Parameters& parameter, const Date& date)
{
	IOInterface *plugin = getPlugin("GRID2D", "Output");
	plugin->write2DGrid(grid_in, parameter, date);
}

void IOHandler::parse_copy_config()
{
	/**
	 * Parse [Input] section for potential parameters that the user wants
343
	 * duplicated (as '%%::COPY = %%')
344
345
	 */
	vector<string> copy_keys;
346
	const size_t nrOfMatches = cfg.findKeys(copy_keys, "::COPY", "Input", true); //search anywhere in key
347

348
	for (size_t ii=0; ii<nrOfMatches; ++ii) {
349
		const string name_of_copy = copy_keys[ii].substr( 0, copy_keys[ii].find_first_of(":") );
350
		string initial_name;
351
352
353
354
355
356
357
358
359
		cfg.getValue(copy_keys[ii], "Input", initial_name);
		if ((name_of_copy.length() > 0) && (initial_name.length() > 0)){
			copy_parameter.push_back(initial_name);
			copy_name.push_back(name_of_copy);
			enable_copying = true;
		}
	}
}

360
361
362
363
364
/**
* This procedure runs through the MeteoData objects in vecMeteo and according to user
* configuration copies a certain present meteo parameter to another one, named by the
* user in the [Input] section of the io.ini, e.g.
* [Input]
365
* TA2::COPY = TA
366
367
368
* means that TA2 will be the name of a new parameter in MeteoData with the copied value
* of the meteo parameter MeteoData::TA
*/
369
void IOHandler::copy_parameters(const size_t& stationindex, std::vector< METEO_SET >& vecMeteo) const
370
371
372
{
	if (!enable_copying) return; //Nothing configured

373
374
	if (stationindex != IOUtils::npos && stationindex>=vecMeteo.size())
		throw IndexOutOfBoundsException("Accessing stationindex in readMeteoData that is out of bounds", AT);
375

376
377
	const size_t station_start = (stationindex==IOUtils::npos)? 0 : stationindex;
	const size_t station_end = (stationindex==IOUtils::npos)? vecMeteo.size() : stationindex+1;
378
379
380
	const size_t nr_of_params = copy_parameter.size();
	vector<size_t> indices; //will hold the indices of the parameters to be copied

381
382
	for (size_t ii=station_start; ii<station_end; ++ii) { //for each station
		for (size_t jj=0; jj<vecMeteo[ii].size(); ++jj) { //for each MeteoData object of one station
383
384

			if (jj==0) { //buffer the index numbers
385
				for (size_t kk=0; kk<nr_of_params; ++kk) {
386
387
					const size_t param_index = vecMeteo[ii][jj].getParameterIndex(copy_parameter[kk]);
					if (param_index == IOUtils::npos) {
388
						std::ostringstream ss;
389
390
391
392
393
394
395
396
397
						ss << "At " << vecMeteo[ii][jj].date.toString(Date::ISO) << ", station " << vecMeteo[ii][jj].meta.stationID;
						ss << " has no parameter \"" << copy_parameter[kk] << "\" to copy!\n";
						throw InvalidArgumentException(ss.str(), AT);
					}

					indices.push_back(param_index);
				}
			}

398
			for (size_t kk=0; kk<nr_of_params; ++kk) {
399
400
401
402
403
404
405
406
				const size_t newparam = vecMeteo[ii][jj].addParameter(copy_name[kk]);
				vecMeteo[ii][jj](newparam) = vecMeteo[ii][jj](indices[kk]);
			}
		}
		indices.clear(); //may change for every station
	}
}

407
const std::string IOHandler::toString() const
408
{
409
	std::ostringstream os;
410
411
412
413
	os << "<IOHandler>\n";
	os << "Config& cfg = " << hex << &cfg << dec << "\n";

	os << "<mapPlugins>\n";
414
415
416
	std::map<std::string, IOInterface*>::const_iterator it1;
	for (it1=mapPlugins.begin(); it1 != mapPlugins.end(); ++it1) {
		os << setw(10) << it1->first << " = " << hex <<  it1->second << dec << "\n";
417
418
419
420
421
422
423
	}
	os << "</mapPlugins>\n";
	os << "</IOHandler>\n";
	return os.str();
}

} //end namespace