WSL/SLF GitLab Repository

CosmoXMLIO.cc 22.8 KB
Newer Older
1
/***********************************************************************************/
2
/*  Copyright 2014 WSL Institute for Snow and Avalanche Research    SLF-DAVOS      */
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
/***********************************************************************************/
/* This file is part of MeteoIO.
    MeteoIO is free software: you can redistribute it and/or modify
    it under the terms of the GNU Lesser General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    MeteoIO is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public License
    along with MeteoIO.  If not, see <http://www.gnu.org/licenses/>.
*/
18
#include "CosmoXMLIO.h"
19
#include <meteoio/meteoLaws/Atmosphere.h>
20

21
#include <sstream>
22

23
#include <libxml/parserInternals.h>
24
25
26
27
28
#include <libxml/xpathInternals.h>
#if !defined(LIBXML_XPATH_ENABLED) || !defined(LIBXML_SAX1_ENABLED)
	#error Please enable XPATH and SAX1 in your version of libxml!
#endif

29
30
31
32
33
34
using namespace std;

namespace mio {
/**
 * @page cosmoxml COSMOXML
 * @section cosmoxml_format Format
35
36
 * This plugin reads the XML files as generated by <A HREF="http://www.cosmo-model.org/">Cosmo</A>'s <A HREF="http://www.cosmo-model.org/content/support/software/default.htm#fieldextra">FieldExtra</A>.
 * The files are written out by COSMO in Grib format and preprocessed by FieldExtra (MeteoSwiss) to get XML files.
37
38
 * It requires <A HREF="http://xmlsoft.org/">libxml2</A> to compile and run. It also assumes that the station IDs are unique
 * (ie two data sets with the same station ID are considered to belong to the same station).
39
 *
40
 * @section cosmoxml_cosmo_partners COSMO Group
41
42
43
44
45
46
47
48
49
50
51
52
53
 * This plugin has been developed primarily for reading XML files produced by COSMO (http://www.cosmo-model.org/) at MeteoSwiss.
 * COSMO (COnsortium for Small scale MOdelling) represents a non-hydrostatic limited-area atmospheric model, to be used both for operational and for research applications by the members of the consortium. The Consortium has the following members:
 *  - Germany, DWD, Deutscher Wetterdienst
 *  - Switzerland, MCH, MeteoSchweiz
 *  - Italy, USAM, Ufficio Generale Spazio Aereo e Meteorologia
 *  - Greece, HNMS, Hellenic National Meteorological Service
 *  - Poland, IMGW, Institute of Meteorology and Water Management
 *  - Romania, NMA, National Meteorological Administration
 *  - Russia, RHM, Federal Service for Hydrometeorology and Environmental Monitoring
 *  - Germany, AGeoBw, Amt für GeoInformationswesen der Bundeswehr
 *  - Italy, CIRA, Centro Italiano Ricerche Aerospaziali
 *  - Italy, ARPA-SIMC, ARPA Emilia Romagna Servizio Idro Meteo Clima
 *  - Italy, ARPA Piemonte, Agenzia Regionale per la Protezione Ambientale Piemonte
54
55
56
 *
 * @section cosmoxml_units Units
 * The units are assumed to be the following:
57
58
 * - temperatures in K
 * - relative humidity in %
59
60
61
 * - wind speed in m/s
 * - precipitations in mm/h
 * - radiation in W/m²
62
63
 * - snow height in cm
 * - maximal wind speed in m/s
64
65
66
 *
 * @section cosmoxml_keywords Keywords
 * This plugin uses the following keywords:
67
 * - COORDSYS:  input coordinate system (see Coords) specified in the [Input] section
68
 * - METEO:     specify COSMOXML for [Input] section
69
 * - METEOPATH: string containing the path to the xml files to be read, specified in the [Input] section
70
 * - METEOFILE: specify the xml file to read the data from (optional)
71
72
 * - METEO_PREFIX: file name prefix appearing before the date (optional)
 * - METEO_EXT: file extension (default: ".xml", give "none" to get an empty string)
73
 * - STATION#: ID of the station to read
74
 * - IMIS_STATIONS: if set to true, all station IDs provided above will be stripped of their number (to match MeteoCH naming scheme)
75
 * - USE_MODEL_LOC: if set to false, the true station location (lat, lon, altitude) is used. Otherwise, it uses the model location (default)
76
 * - XML_ENCODING: force the input file encoding, overriding the file's own encoding declaration (optional, see \ref cosmoxml_encoding "XML encoding" below)
77
 *
78
79
 * If no METEOFILE is provided, all ".xml" files in the METEOPATH directory will be read, if they match the METEO_PREFIX and METEO_EXT.
 * They <i>must</i> contain the date of the first data formatted as ISO8601 numerical UTC date in their file name. For example, a file containing simulated
80
 * meteorological fields from 2014-03-03T12:00 until 2014-03-05T00:00 could be named such as "cosmo_201403031200.xml"
81
82
 * If some numbers appear <i>before</i> the numerical date, they must be provided as part of METEO_PREFIX so the plugin can
 * properly extract the date (for MeteoSwiss, this must be set to "VNMH49").
83
 *
84
85
86
87
 * Example:
 * @code
 * [Input]
 * COORDSYS	= CH1903
88
 * METEO	= COSMOXML
89
 * METEOPATH	= ./input/meteoXMLdata
90
91
92
 * METEOFILE	= cosmo2.xml
 * STATION1	= ATT
 * STATION2	= EGH
93
 * @endcode
94
95
96
97
98
99
100
101
102
103
104
 *
 * @subsection cosmoxml_encoding XML encoding
 * Each XML document should specify its encoding. However this information might sometimes be missing or even worst, be false. This makes the XML document non-compliant.
 * Normally, COSMOXML reads the file encoding in the file itself. If this does not work (one of the two cases given above), it is possible to force the
 * encoding of the input file by using the "XML_ENCODING" option. This option takes one of the following values
 * ("LE" stands for "Little Endian" and "BE" for "Big Endian"):
 *  - for UTF/UCS: UTF-8, UTF-16-LE, UTF-16-BE, UCS-4-LE, UCS-4-BE, UCS-4-2143, UCS-4-3412, UCS-2, EBCDIC
 *  - for ISO-8859: ISO-8859-1, ISO-8859-2, ISO-8859-3, ISO-8859-4, ISO-8859-5, ISO-8859-6, ISO-8859-7, ISO-8859-8, ISO-8859-9
 *  - for Japanses: ISO-2022-JP, SHIFT-JIS, EUC-JP
 *  - for ascii: ASCII
 *
105
106
 */

107
const double CosmoXMLIO::in_tz = 0.; //Plugin specific timezone
108
109
const xmlChar* CosmoXMLIO::xml_attribute = (const xmlChar *)"id";
const xmlChar* CosmoXMLIO::xml_namespace = (const xmlChar *)"http://www.meteoswiss.ch/xmlns/modeltemplate/2";
110
const xmlChar* CosmoXMLIO::xml_namespace_abrev = (const xmlChar*)"ns";
111
112
const std::string CosmoXMLIO::StationData_xpath = "//ns:datainformation/ns:data-tables/ns:data/ns:row/ns:col";
const std::string CosmoXMLIO::MeteoData_xpath = "//ns:valueinformation/ns:values-tables/ns:data/ns:row/ns:col";
113

114
CosmoXMLIO::CosmoXMLIO(const std::string& configfile)
115
           : cache_meteo_files(), xml_stations_id(), input_id(),
116
             meteo_prefix(), meteo_ext(".xml"), plugin_nodata(-999.), imis_stations(false), use_model_loc(true), in_doc(NULL), in_xpathCtx(NULL),
117
             in_encoding(XML_CHAR_ENCODING_NONE), coordin(), coordinparam()
118
{
119
	Config cfg(configfile);
120
	init(cfg);
121
122
}

123
CosmoXMLIO::CosmoXMLIO(const Config& cfg)
124
           : cache_meteo_files(), xml_stations_id(), input_id(),
125
             meteo_prefix(), meteo_ext(".xml"), plugin_nodata(-999.), imis_stations(false), use_model_loc(true), in_doc(NULL), in_xpathCtx(NULL),
126
             in_encoding(XML_CHAR_ENCODING_NONE), coordin(), coordinparam()
127
128
129
130
131
{
	init(cfg);
}

void CosmoXMLIO::init(const Config& cfg)
132
{
133
	LIBXML_TEST_VERSION //check lib versions and call xmlInitParser()
134

135
	string coordout, coordoutparam;
136
	IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
137

138
	cfg.getValues("STATION", "INPUT", input_id);
139
	cfg.getValue("IMIS_STATIONS", "INPUT", imis_stations, IOUtils::nothrow);
140
	cfg.getValue("USE_MODEL_LOC", "INPUT", use_model_loc, IOUtils::nothrow);
141
142
143

	const std::string meteopath = cfg.get("METEOPATH", "INPUT");
	const std::string meteofile = cfg.get("METEOFILE", "INPUT", IOUtils::nothrow);
144
145
146
	cfg.getValue("METEO_PREFIX", "INPUT", meteo_prefix, IOUtils::nothrow);
	cfg.getValue("METEO_EXT", "INPUT", meteo_ext, IOUtils::nothrow);
	if( IOUtils::strToUpper(meteo_ext)=="NONE" ) meteo_ext="";
147

148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
	//input encoding forcing
	string tmp;
	cfg.getValue("XML_ENCODING", "INPUT", tmp, IOUtils::nothrow);
	if(!tmp.empty()) {
		if(tmp=="UTF-8") in_encoding=XML_CHAR_ENCODING_UTF8;
		else if(tmp=="UTF-16-LE") in_encoding=XML_CHAR_ENCODING_UTF16LE;
		else if(tmp=="UTF-16-BE") in_encoding=XML_CHAR_ENCODING_UTF16BE;
		else if(tmp=="UCS-4-LE") in_encoding=XML_CHAR_ENCODING_UCS4LE;
		else if(tmp=="UCS-4-BE") in_encoding=XML_CHAR_ENCODING_UCS4BE;
		else if(tmp=="EBCDIC") in_encoding=XML_CHAR_ENCODING_EBCDIC;
		else if(tmp=="UCS-4-2143") in_encoding=XML_CHAR_ENCODING_UCS4_2143;
		else if(tmp=="UCS-4-3412") in_encoding=XML_CHAR_ENCODING_UCS4_3412;
		else if(tmp=="UCS-2") in_encoding=XML_CHAR_ENCODING_UCS2;
		else if(tmp=="ISO-8859-1") in_encoding=XML_CHAR_ENCODING_8859_1;
		else if(tmp=="ISO-8859-2") in_encoding=XML_CHAR_ENCODING_8859_2;
		else if(tmp=="ISO-8859-3") in_encoding=XML_CHAR_ENCODING_8859_3;
		else if(tmp=="ISO-8859-4") in_encoding=XML_CHAR_ENCODING_8859_4;
		else if(tmp=="ISO-8859-5") in_encoding=XML_CHAR_ENCODING_8859_5;
		else if(tmp=="ISO-8859-6") in_encoding=XML_CHAR_ENCODING_8859_6;
		else if(tmp=="ISO-8859-7") in_encoding=XML_CHAR_ENCODING_8859_7;
		else if(tmp=="ISO-8859-8") in_encoding=XML_CHAR_ENCODING_8859_8;
		else if(tmp=="ISO-8859-9") in_encoding=XML_CHAR_ENCODING_8859_9;
		else if(tmp=="ISO-2022-JP") in_encoding=XML_CHAR_ENCODING_2022_JP;
		else if(tmp=="SHIFT-JIS") in_encoding=XML_CHAR_ENCODING_SHIFT_JIS;
		else if(tmp=="EUC-JP") in_encoding=XML_CHAR_ENCODING_EUC_JP;
		else if(tmp=="ASCII") in_encoding=XML_CHAR_ENCODING_ASCII;
		else
			throw InvalidArgumentException("Encoding \""+tmp+"\" is not supported!", AT);
	}

178
	if(!meteofile.empty()) {
179
		const string file_and_path = meteopath + "/" + meteofile;
180
181
		const std::pair<Date,std::string> tmp_pair(Date(), file_and_path);
		cache_meteo_files.push_back( tmp_pair );
182
183
184
	} else {
		scanMeteoPath(meteopath, cache_meteo_files);
	}
185
186
187
188
}

CosmoXMLIO& CosmoXMLIO::operator=(const CosmoXMLIO& source) {
	if(this != &source) {
189
		cache_meteo_files = source.cache_meteo_files;
190
191
192
193
194
195
196
197
198
		xml_stations_id = source.xml_stations_id;
		input_id = source.input_id;
		plugin_nodata = source.plugin_nodata;
		in_doc = NULL;
		in_xpathCtx = NULL;
		coordin = source.coordin;
		coordinparam = source.coordinparam;
	}
	return *this;
199
200
201
202
}

CosmoXMLIO::~CosmoXMLIO() throw()
{
203
204
205
	closeIn_XML();
}

206
void CosmoXMLIO::scanMeteoPath(const std::string& meteopath_in,  std::vector< std::pair<mio::Date,std::string> > &meteo_files) const
207
208
{
	meteo_files.clear();
209

210
211
212
213
214
	std::list<std::string> dirlist;
	IOUtils::readDirectory(meteopath_in, dirlist, meteo_ext);
	dirlist.sort();

	//Check date in every filename and cache it
215
	const size_t prefix_len = meteo_prefix.size();
216
217
218
	std::list<std::string>::const_iterator it = dirlist.begin();
	while ((it != dirlist.end())) {
		const std::string& filename = *it;
219
220
221
		const std::string::size_type prefix_pos = (prefix_len==0)? 0 : filename.find_first_of(meteo_prefix);
		if(prefix_pos==string::npos) continue;

222
		const size_t start_pos = prefix_pos+prefix_len;
223
224
225
		if(start_pos>=filename.size()) continue;

		const std::string::size_type date_pos = filename.find_first_of("0123456789", start_pos);
226
		Date date;
227
		IOUtils::convertString(date, filename.substr(date_pos,10), in_tz);
228
		const std::pair<Date,std::string> tmp(date, meteopath_in+"/"+filename);
229
230
231
232
233
234

		meteo_files.push_back(tmp);
		it++;
	}
}

235
236
237
238
239
240
void CosmoXMLIO::openIn_XML(const std::string& in_meteofile)
{
	if(in_doc!=NULL) return; //the file has already been read

	xmlInitParser();
	xmlKeepBlanksDefault(0);
241

242
243
244
245
246
247
248
249
250
	if(in_encoding==XML_CHAR_ENCODING_NONE) {
		in_doc = xmlParseFile(in_meteofile.c_str());
	} else {
		xmlParserCtxtPtr ctxt = xmlCreateFileParserCtxt( in_meteofile.c_str() );
		xmlSwitchEncoding( ctxt, in_encoding);
		xmlParseDocument( ctxt);
		in_doc = ctxt->myDoc;
	}

251
252
253
254
	if (in_doc == NULL) {
		throw FileNotFoundException("Could not open/parse file \""+in_meteofile+"\"", AT);
	}

255
	if(in_xpathCtx != NULL) xmlXPathFreeContext(in_xpathCtx); //free variable if this was not freed before
256
257
258
259
260
261
	in_xpathCtx = xmlXPathNewContext(in_doc);
	if(in_xpathCtx == NULL) {
		closeIn_XML();
		throw IOException("Unable to create new XPath context", AT);
	}

262
	if(xmlXPathRegisterNs(in_xpathCtx,  xml_namespace_abrev, xml_namespace) != 0) {
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
		throw IOException("Unable to register namespace with prefix", AT);
	}
}

void CosmoXMLIO::closeIn_XML() throw()
{
	if(in_xpathCtx!=NULL) {
		xmlXPathFreeContext(in_xpathCtx);
		in_xpathCtx = NULL;
	}
	if(in_doc!=NULL) {
		xmlFreeDoc(in_doc);
		in_doc = NULL;
	}
	xmlCleanupParser();
278
279
280
281
282
283
284
285
}

void CosmoXMLIO::read2DGrid(Grid2DObject& /*grid_out*/, const std::string& /*_name*/)
{
	//Nothing so far
	throw IOException("Nothing implemented here", AT);
}

286
287
288
289
290
291
void CosmoXMLIO::read2DGrid(Grid2DObject&, const MeteoGrids::Parameters&, const Date&)
{
	//Nothing so far
	throw IOException("Nothing implemented here", AT);
}

292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
void CosmoXMLIO::readDEM(DEMObject& /*dem_out*/)
{
	//Nothing so far
	throw IOException("Nothing implemented here", AT);
}

void CosmoXMLIO::readLanduse(Grid2DObject& /*landuse_out*/)
{
	//Nothing so far
	throw IOException("Nothing implemented here", AT);
}

void CosmoXMLIO::readAssimilationData(const Date& /*date_in*/, Grid2DObject& /*da_out*/)
{
	//Nothing so far
	throw IOException("Nothing implemented here", AT);
}

310
bool CosmoXMLIO::parseStationData(const std::string& station_id, const xmlXPathContextPtr& xpathCtx, StationData &sd)
311
{
312
313
	//match something like "//ns:valueinformation/ns:values-tables/ns:data/ns:row/ns:col[@id='station_abbreviation' and text()='ATT']/.."
	//the namespace "ns" has been previously defined
314
	const std::string xpath_id = (imis_stations)? station_id.substr(0, station_id.find_first_of("0123456789")) : station_id;
315
	const std::string xpath = StationData_xpath+"[@id='station_abbreviation' and text()='"+xpath_id+"']/.."; //and we take the parent node <row>
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333

	xmlXPathObjectPtr xpathObj = xmlXPathEvalExpression((const xmlChar*)xpath.c_str(), xpathCtx);
	if(xpathObj == NULL) return false;

	//check the number of matches
	const xmlNodeSetPtr &metadata = xpathObj->nodesetval;
	const int nr_metadata = (metadata) ? metadata->nodeNr : 0;
	if(nr_metadata==0)
		throw NoAvailableDataException("No metadata found for station \""+station_id+"\"", AT);
	if(nr_metadata>1)
		throw InvalidFormatException("Multiple definition of metadata for station \""+station_id+"\"", AT);

	//collect all the data fields
	std::string xml_id;
	double altitude = IOUtils::nodata, latitude = IOUtils::nodata, longitude = IOUtils::nodata;
	//start from the first child until the last one
	for (xmlNode *cur_node = metadata->nodeTab[0]->children; cur_node; cur_node = cur_node->next) {
		if (cur_node->type == XML_ELEMENT_NODE) {
334
335
			xmlChar *att = xmlGetProp(cur_node, xml_attribute);
			const std::string field( (const char*)(att) );
336
337
338
			xmlFree(att);

			if (cur_node->children->type == XML_TEXT_NODE) {
339
				const std::string value( (const char*)(cur_node->children->content) );
340
341

				if(field=="identifier") xml_id = value;
342
				//else if(field=="station_abbreviation") sd.stationID = value;
343
344
				else if(field=="station_name") sd.stationName = value;
				else if(field=="missing_value_code") IOUtils::convertString(plugin_nodata, value);
345
346
347
348
349
350
351
352
353
354

				if(use_model_loc) {
					if(field=="station_height") IOUtils::convertString(altitude, value);
					else if(field=="station_latitude") IOUtils::convertString(latitude, value);
					else if(field=="station_longitude") IOUtils::convertString(longitude, value);
				} else {
					if(field=="model_station_height") IOUtils::convertString(altitude, value);
					else if(field=="model_station_latitude") IOUtils::convertString(latitude, value);
					else if(field=="model_station_longitude") IOUtils::convertString(longitude, value);
				}
355
356
357
			}
		}
	}
358

359
360
	sd.stationID = station_id;

361
362
363
364
	if(latitude==IOUtils::nodata || longitude==IOUtils::nodata || altitude==IOUtils::nodata)
		throw NoAvailableDataException("Some station location information is missing for station \""+station_id+"\"", AT);
	sd.position.setProj(coordin, coordinparam);
	sd.position.setLatLon(latitude, longitude, altitude);
365

366
367
368
	if(xml_id.empty())
		throw NoAvailableDataException("XML station id missing for station \""+station_id+"\"", AT);
	xml_stations_id[station_id] = xml_id;
369

370
371
	xmlXPathFreeObject(xpathObj);
	return true;
372
373
}

374
CosmoXMLIO::MeteoReadStatus CosmoXMLIO::parseMeteoDataPoint(const Date& dateStart, const Date& dateEnd, const xmlNodePtr &element, MeteoData &md) const
375
{
376
377
378
379
380
	double iswr_dir = IOUtils::nodata, iswr_diff = IOUtils::nodata;

	//collect all the data fields
	for (xmlNode *cur_node = element; cur_node; cur_node = cur_node->next) {
		if (cur_node->type == XML_ELEMENT_NODE) {
381
382
			xmlChar *att = xmlGetProp(cur_node, xml_attribute);
			const std::string field( (const char*)(att) );
383
384
385
			xmlFree(att);

			if (cur_node->children->type == XML_TEXT_NODE) {
386
				const std::string value( (const char*)(cur_node->children->content) );
387
388
				if(field=="reference_ts") {
					IOUtils::convertString(md.date, value, in_tz);
389
390
					if(md.date<dateStart) return read_continue;
					if(md.date>dateEnd) return read_stop;
391
392
393
394
395
				} else {
					double tmp;
					IOUtils::convertString(tmp, value);
					tmp = IOUtils::standardizeNodata(tmp, plugin_nodata);

396
					//for now, we hard-code the fields mapping
397
398
399
400
401
402
403
404
405
406
407
408
					if(field=="108005") md(MeteoData::TA) = tmp;
					else if(field=="108014") md(MeteoData::RH) = tmp/100.;
					else if(field=="108015") md(MeteoData::VW) = tmp;
					else if(field=="108017") md(MeteoData::DW) = tmp;
					else if(field=="108018") md(MeteoData::VW_MAX) = tmp;
					else if(field=="108023") md(MeteoData::HNW) = tmp;
					else if(field=="108060") md(MeteoData::HS) = tmp/100.;
					else if(field=="108062") md(MeteoData::TSS) = tmp;
					else if(field=="108064") iswr_diff = tmp;
					else if(field=="108065") iswr_dir = tmp;
					else if(field=="108066") md(MeteoData::RSWR) = tmp;
					else if(field=="108067") md(MeteoData::ILWR) = tmp; //108068=olwr
409
410
411
412
				}
			}
		}
	}
413

414
415
	if(iswr_diff!=IOUtils::nodata && iswr_dir!=IOUtils::nodata)
		md(MeteoData::ISWR) = iswr_diff+iswr_dir;
416

417
418
419
420
	//because of the Kalman filter applied on VW, sometimes VW_MAX<VW
	if(md(MeteoData::VW)!=IOUtils::nodata && md(MeteoData::VW_MAX)!=IOUtils::nodata && md(MeteoData::VW_MAX)<md(MeteoData::VW))
		md(MeteoData::VW_MAX) = md(MeteoData::VW);

421
	return read_ok;
422
423
}

424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
size_t CosmoXMLIO::getFileIdx(const Date& start_date) const
{
	if(cache_meteo_files.empty())
		throw InvalidArgumentException("No input files found or configured!", AT);

	//find which file we should open
	if(cache_meteo_files.size()==1) {
		return 0;
	} else {
		for(size_t idx=1; idx<cache_meteo_files.size(); idx++) {
			if(start_date>=cache_meteo_files[idx-1].first && start_date<cache_meteo_files[idx].first) {
				return idx--;
			}
		}

		//not found, we take the closest timestamp we have
		if(start_date<cache_meteo_files.front().first)
			return 0;
		else
			return cache_meteo_files.size()-1;
	}
}

void CosmoXMLIO::readStationData(const Date& station_date, std::vector<StationData>& vecStation)
{
	vecStation.clear();

	const std::string meteofile( cache_meteo_files[ getFileIdx(station_date) ].second );
	openIn_XML(meteofile);

	//read all the stations' metadata
	for(size_t ii=0; ii<input_id.size(); ii++) {
		StationData sd;
		if(!parseStationData(input_id[ii], in_xpathCtx, sd)) {
			closeIn_XML();
			throw IOException("Unable to evaluate xpath expression for station \""+input_id[ii]+"\"", AT);
		}
		vecStation.push_back(sd);
	}

	closeIn_XML();
}

467
bool CosmoXMLIO::parseMeteoData(const Date& dateStart, const Date& dateEnd, const std::string& station_id, const StationData& sd, const xmlXPathContextPtr& xpathCtx, std::vector<MeteoData> &vecMeteo) const
468
{
469
	const std::string xpath = MeteoData_xpath+"[@id='identifier' and text()='"+station_id+"']";
470

471
472
473
474
475
476
477
478
	xmlXPathObjectPtr xpathObj = xmlXPathEvalExpression((const xmlChar*)xpath.c_str(), xpathCtx);
	if(xpathObj == NULL) return false;

	//check the number of matches
	const xmlNodeSetPtr &data = xpathObj->nodesetval;
	const int nr_data = (data) ? data->nodeNr : 0;
	if(nr_data==0)
		throw NoAvailableDataException("No data found for station \""+station_id+"\"", AT);
479

480
481
482
483
	//loop over all data for this station_id
	for(int ii=0; ii<nr_data; ii++) {
		MeteoData md( Date(), sd);

484
485
		const MeteoReadStatus status = parseMeteoDataPoint(dateStart, dateEnd, data->nodeTab[ii], md);
		if(status==read_stop) break;
486
		if(status==read_ok) vecMeteo.push_back( md );
487
	}
488

489
490
	xmlXPathFreeObject(xpathObj);
	return true;
491
492
}

493
494
495
void CosmoXMLIO::readMeteoData(const Date& dateStart, const Date& dateEnd,
                               std::vector< std::vector<MeteoData> >& vecMeteo,
                               const size_t&)
496
{
497
	vecMeteo.clear();
498
499
500
501
502
503
504
	const size_t nr_files = cache_meteo_files.size();
	size_t file_idx = getFileIdx(dateStart);
	Date nextDate;

	do {
		//since files contain overlapping data, we will only read the non-overlapping part
		//ie from start to the start date of the next file
505
		nextDate = ((file_idx+1)<nr_files)? cache_meteo_files[file_idx+1].first - 1./3600. : dateEnd;
506
507
508
509
510
511
512
513
514
515

		const std::string meteofile( cache_meteo_files[file_idx].second );
		openIn_XML(meteofile);

		//read all the stations' metadata
		std::vector<StationData> vecStation;
		for(size_t ii=0; ii<input_id.size(); ii++) {
			StationData sd;
			if(!parseStationData(input_id[ii], in_xpathCtx, sd)) {
				closeIn_XML();
516
				throw IOException("Unable to evaluate xpath expression for station \""+input_id[ii]+"\"", AT);
517
518
519
			}
			vecStation.push_back(sd);
		}
520

521
522
523
		//read all the stations' data
		for(size_t ii=0; ii<input_id.size(); ii++) {
			const string station_id = xml_stations_id[ input_id[ii] ];
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545

			//do we already have a vector with this station meteo?
			size_t found_id = IOUtils::npos;
			for(size_t jj=0; jj<vecMeteo.size(); jj++) {
				if(vecMeteo[jj].front().meta.stationID==input_id[ii]) {
					found_id = jj;
					break;
				}
			}

			if(found_id==IOUtils::npos) { //creating the station
				vector<MeteoData> vecTmp;
				if(!parseMeteoData(dateStart, nextDate, station_id, vecStation[ii], in_xpathCtx, vecTmp)) {
					closeIn_XML();
					throw IOException("Unable to evaluate xpath expression for station \""+input_id[ii]+"\"", AT);
				}
				vecMeteo.push_back(vecTmp);
			} else { //appending to the station
				if(!parseMeteoData(dateStart, nextDate, station_id, vecStation[ii], in_xpathCtx, vecMeteo[found_id])) {
					closeIn_XML();
					throw IOException("Unable to evaluate xpath expression for station \""+input_id[ii]+"\"", AT);
				}
546
			}
547
		}
548

549
550
551
552
		closeIn_XML();

		file_idx++;
	} while (file_idx<nr_files && nextDate<=dateEnd);
553
}
554

555
556
557
558
559
void CosmoXMLIO::writeMeteoData(const std::vector< std::vector<MeteoData> >& /*vecMeteo*/,
                                const std::string&)
{
	//Nothing so far
	throw IOException("Nothing implemented here", AT);
560
561
}

562
void CosmoXMLIO::readPOI(std::vector<Coords>&)
563
564
565
566
567
568
569
570
571
572
573
{
	//Nothing so far
	throw IOException("Nothing implemented here", AT);
}

void CosmoXMLIO::write2DGrid(const Grid2DObject& /*grid_in*/, const std::string& /*name*/)
{
	//Nothing so far
	throw IOException("Nothing implemented here", AT);
}

574
575
576
577
578
579
void CosmoXMLIO::write2DGrid(const Grid2DObject&, const MeteoGrids::Parameters&, const Date&)
{
	//Nothing so far
	throw IOException("Nothing implemented here", AT);
}

580
} //namespace
581