WSL/SLF GitLab Repository

BormaIO.cc 16.5 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
/***********************************************************************************/
/*  Copyright 2009 WSL Institute for Snow and Avalanche Research    SLF-DAVOS      */
/***********************************************************************************/
/* This file is part of MeteoIO.
    MeteoIO is free software: you can redistribute it and/or modify
    it under the terms of the GNU Lesser General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    MeteoIO is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public License
    along with MeteoIO.  If not, see <http://www.gnu.org/licenses/>.
*/
#include "BormaIO.h"

using namespace std;

namespace mio {
/**
 * @page borma BORMA
 * @section borma_format Format
26
27
28
 * This plugin reads the XML files as generated by the Borma system. It requires libxml to compile and run. Since
 * years are expressed on two digits, all years greater than 80 are considered to belong to the XXth century while
 * years smaller belong to the XXIst century.
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
 *
 * @section borma_units Units
 * The units are assumed to be the following:
 * - temperatures in celsius
 * - relative humidity in %
 * - wind speed in m/s
 * - precipitations in mm/h
 * - radiation in W/m²
 *
 * @section borma_keywords Keywords
 * This plugin uses the following keywords:
 * - COORDSYS: input coordinate system (see Coords) specified in the [Input] section
 * - COORDPARAM: extra input coordinates parameters (see Coords) specified in the [Input] section
 * - COORDSYS: output coordinate system (see Coords) specified in the [Output] section
 * - COORDPARAM: extra output coordinates parameters (see Coords) specified in the [Output] section
44
 * - METEOPATH: string containing the path to the xml files
45
46
47
48
49
 * - NROFSTATIONS: total number of stations listed for use
 * - STATION#: station id for the given number #
 */

const double BormaIO::plugin_nodata = -999.0; //plugin specific nodata value
50
51
const double BormaIO::default_tz = +1.; //default timezone
const double BormaIO::pivot_year = 80.; //pivot year for Y2K suppport
52

53
BormaIO::BormaIO(void (*delObj)(void*), const Config& i_cfg) : IOInterface(delObj), cfg(i_cfg)
54
55
{
	IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
56
	in_tz = default_tz;
57
	cfg.getValue("TIME_ZONE","Input",in_tz,Config::nothrow);
58
59
60
61
62
}

BormaIO::BormaIO(const std::string& configfile) : IOInterface(NULL), cfg(configfile)
{
	IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
63
	in_tz = default_tz;
64
	cfg.getValue("TIME_ZONE","Input",in_tz,Config::nothrow);
65
66
}

67
BormaIO::BormaIO(const Config& cfgreader) : IOInterface(NULL), cfg(cfgreader)
68
69
{
	IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
70
	in_tz = default_tz;
71
	cfg.getValue("TIME_ZONE","Input",in_tz,Config::nothrow);
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
}

BormaIO::~BormaIO() throw()
{
	cleanup();
}

void BormaIO::cleanup() throw()
{
	if (fin.is_open()) {//close fin if open
		fin.close();
	}
}

void BormaIO::read2DGrid(Grid2DObject&, const std::string& filename)
{
	//Nothing so far
	(void)filename;
	throw IOException("Nothing implemented here", AT);
}

void BormaIO::readDEM(DEMObject&)
{
	//Nothing so far
	throw IOException("Nothing implemented here", AT);
}

void BormaIO::readLanduse(Grid2DObject&)
{
	//Nothing so far
	throw IOException("Nothing implemented here", AT);
}

void BormaIO::writeMeteoData(const std::vector< std::vector<MeteoData> >&,
106
                             const std::string&)
107
108
109
110
111
112
113
{
	//Nothing so far
	throw IOException("Nothing implemented here", AT);
}

void BormaIO::readStationData(const Date&, std::vector<StationData>&)
{
114
	//HACK: this method MUST be implemented for BufferedIOHandler to properly work
115
116
117
118
119
	//Nothing so far
	throw IOException("Nothing implemented here", AT);
}

void BormaIO::readMeteoData(const Date& dateStart, const Date& dateEnd,
120
121
                            std::vector< std::vector<MeteoData> >& vecMeteo,
                            const unsigned int& stationindex)
122
123
124
125
126
127
128
129
130
131
132
{
	if (vecStationName.size() == 0)
		readStationNames(); //reads station names into vector<string> vecStationName

	unsigned int indexStart=0, indexEnd=vecStationName.size();

	//The following part decides whether all the stations are rebuffered or just one station
	if (stationindex == IOUtils::npos){
		vecMeteo.clear();
		vecMeteo.insert(vecMeteo.begin(), vecStationName.size(), std::vector<MeteoData>());
	} else {
133
		if (stationindex < vecMeteo.size()){
134
135
136
137
138
139
140
141
			indexStart = stationindex;
			indexEnd   = stationindex+1;
		} else {
			throw IndexOutOfBoundsException("", AT);
		}
	}

	for (unsigned int ii=indexStart; ii<indexEnd; ii++){ //loop through stations
142
		bufferData(dateStart, dateEnd, vecMeteo, ii);
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
	}
}

void BormaIO::readStationNames()
{
	vecStationName.clear();

	//Read in the StationNames
	std::string xmlpath="", str_stations="";
	int stations=0;

	cfg.getValue("NROFSTATIONS", "Input", str_stations);

	if (!IOUtils::convertString(stations, str_stations, std::dec)) {
		throw ConversionFailedException("Error while reading value for NROFSTATIONS", AT);
	}

	for (int ii=0; ii<stations; ii++) {
		std::stringstream tmp_stream;
		std::string stationname="", tmp_file="";

		tmp_stream << (ii+1); //needed to construct key name
		cfg.getValue(std::string("STATION"+tmp_stream.str()), "Input", stationname);

		vecStationName.push_back(stationname);
	}
}

void BormaIO::getFiles(const std::string& stationname, const Date& start_date, const Date& end_date,
172
                       std::vector<std::string>& vecFiles, std::vector<Date>& vecDate)
173
174
175
176
{
	std::list<std::string> dirlist = std::list<std::string>();
	std::string xmlpath="";

177
	cfg.getValue("METEOPATH", "Input", xmlpath);
178
179
180
181
182
183
184
	vecFiles.clear();
	IOUtils::readDirectory(xmlpath, dirlist, "_" + stationname + ".xml");
	dirlist.sort();

	//Check date in every filename
	std::list<std::string>::iterator it = dirlist.begin();

185
	Date tmp_date(0., 0.);
186
187
188
189
	if (start_date > end_date){ //Special case return first data >= dateStart
		while ((it != dirlist.end())) {
			//check validity of filename
			if (validFilename(*it)) {
190
191
				const std::string filename_out = *it;
				tmp_date = stringToDate(filename_out);
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207

				if (tmp_date > start_date) {
					vecFiles.push_back(xmlpath + "/" + filename_out);
					vecDate.push_back(tmp_date);
					return;
				}
			}

			it++;
		}
		return;
	}

	while ((it != dirlist.end()) && (tmp_date < end_date)) {
		//check validity of filename
		if (validFilename(*it)) {
208
209
			const std::string filename_out = *it;
			tmp_date = stringToDate(filename_out);
210
211
212
213
214
215
216
217
218
219
220
221

			if ((tmp_date >= start_date) && (tmp_date <= end_date)) {
				vecFiles.push_back(xmlpath + "/" + filename_out);
				vecDate.push_back(tmp_date);
			}
		}

		it++;
	}
}

bool BormaIO::bufferData(const Date& dateStart, const Date& dateEnd,
222
223
                         std::vector< std::vector<MeteoData> >& vecMeteo,
                         const unsigned int& stationnr)
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
{
	std::vector<std::string> vecFiles;
	std::vector<Date> vecDate;

	if (stationnr >= vecMeteo.size()) {
		throw IndexOutOfBoundsException("", AT);
	}

	vecMeteo[stationnr].clear();

	getFiles(vecStationName[stationnr], dateStart, dateEnd, vecFiles, vecDate);
	//cout << "[i] Buffering station number: " << vecStationName[stationnr] << "  " << vecFiles.size() << " files" << endl;

	if (vecFiles.size()==0) { //No files in range between dateStart and dateEnd
		return false;
	}

	for (unsigned int ii=0; ii<vecFiles.size(); ii++) {
		MeteoData meteoData;
		StationData stationData;
		xmlExtractData(vecFiles[ii], vecDate[ii], meteoData, stationData);
245
		meteoData.meta = stationData;
246
247
248
249
250
251
252
253
		vecMeteo[stationnr].push_back(meteoData);
	}

	return true;
}


void BormaIO::checkForMeteoFiles(const std::string& xmlpath, const std::string& stationname, const Date& date_in,
254
                                 std::string& filename_out, Date& date_out)
255
256
257
258
259
260
261
262
263
264
265
266
{
	std::list<std::string> dirlist = std::list<std::string>();
	IOUtils::readDirectory(xmlpath, dirlist, "_" + stationname + ".xml");
	dirlist.sort();

	//Check date in every filename
	std::list<std::string>::iterator it = dirlist.begin();

	while ((it != dirlist.end()) && (date_out < date_in)) {
		//check validity of filename
		if (validFilename(*it)) {
			filename_out = *it;
267
			date_out = stringToDate(filename_out);
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
		}

		it++;
	}
}

void BormaIO::xmlExtractData(const std::string& filename, const Date& date_in, MeteoData& md, StationData& sd)
{
	double ta=IOUtils::nodata, iswr=IOUtils::nodata, vw=IOUtils::nodata, dw=IOUtils::nodata;
	double rh=IOUtils::nodata, ilwr=IOUtils::nodata, hnw=IOUtils::nodata, tsg=IOUtils::nodata;
	double tss=IOUtils::nodata, hs=IOUtils::nodata, rswr=IOUtils::nodata;
	double longitude=IOUtils::nodata, latitude=IOUtils::nodata, altitude=IOUtils::nodata;

	//Try to read xml file
	xmlpp::DomParser parser;
283
	//parser.set_validate(); //provide DTD to check syntax
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
	parser.set_substitute_entities(); //We just want the text to be resolved/unescaped automatically.
	parser.parse_file(filename);
	if(parser) {
		//Walk the tree: ROOT NODE
		xmlpp::Node* pNode = parser.get_document()->get_root_node(); //deleted by DomParser.

		//Read in StationData
		const std::string stationName = xmlGetNodeContent(pNode, "stationsId");
		const std::string str_long = xmlGetNodeContent(pNode, "stationsLon");
		const std::string str_lati = xmlGetNodeContent(pNode, "stationsLat");
		const std::string str_alti = xmlGetNodeContent(pNode, "stationsAlt");

		xmlParseStringToDouble(str_long, longitude, "stationsLon");
		xmlParseStringToDouble(str_lati, latitude, "stationsLat");
		xmlParseStringToDouble(str_alti, altitude, "stationsAlt");

		//HACK!! would it be possible for getValueForKey() to do this transparently? (with a user flag)
		latitude = IOUtils::standardizeNodata(latitude, plugin_nodata);
		longitude = IOUtils::standardizeNodata(longitude, plugin_nodata);
		altitude = IOUtils::standardizeNodata(altitude, plugin_nodata);

		//compute/check WGS coordinates (considered as the true reference) according to the projection as defined in cfg
		Coords location(coordin, coordinparam);
		location.setLatLon(latitude, longitude, altitude);
308
		sd.setStationData(location, stationName, stationName);
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413

		//lt = ta
		const std::string str_lt = xmlGetNodeContent(pNode, "lt");
		xmlParseStringToDouble(str_lt, ta, "lt");

		//gs = iswr
		const std::string str_gs = xmlGetNodeContent(pNode, "gs");
		xmlParseStringToDouble(str_gs, iswr, "gs");

		//Wind velocity
		const std::string str_vw = xmlGetNodeContent(pNode, "wgm");
		xmlParseStringToDouble(str_vw, vw, "wgm");

		//rlf = rh
		const std::string str_rh = xmlGetNodeContent(pNode, "rlf");
		xmlParseStringToDouble(str_rh, rh, "rlf");

		//ni = hnw //TODO: not sure that this field really contains what we want...
		const std::string str_ns = xmlGetNodeContent(pNode, "ni");
		xmlParseStringToDouble(str_ns, hnw, "ni");

		//sb = ilwr
		const std::string str_sb = xmlGetNodeContent(pNode, "sb");
		xmlParseStringToDouble(str_sb, ilwr, "sb");

		//fbt = tss
		const std::string str_fbt = xmlGetNodeContent(pNode, "fbt");
		xmlParseStringToDouble(str_fbt, tss, "fbt");

		//sh = hs
		const std::string str_sh = xmlGetNodeContent(pNode, "sh");
		xmlParseStringToDouble(str_sh, hs, "sh");

		md.setDate(date_in);
		md.setData(MeteoData::TA, ta);
		md.setData(MeteoData::ISWR, iswr);
		md.setData(MeteoData::VW, vw);
		md.setData(MeteoData::DW, dw);
		md.setData(MeteoData::RH, rh);
		md.setData(MeteoData::ILWR, ilwr);
		md.setData(MeteoData::HNW, hnw);
		md.setData(MeteoData::TSG, tsg);
		md.setData(MeteoData::TSS, tss);
		md.setData(MeteoData::HS, hs);
		md.setData(MeteoData::RSWR, rswr);

		convertUnits(md);

	} else {
		throw IOException("Error parsing XML", AT);
	}
}

void BormaIO::xmlParseStringToDouble(const std::string& str_in, double& d_out, const std::string& parname)
{
	if (str_in!="") {//do nothing if empty content for a node was read
		if (!IOUtils::convertString(d_out, str_in, std::dec)) {//but if content of node is not empty, try conversion
			throw ConversionFailedException("Error while reading value for " + parname, AT);
		}
	}
}

std::string BormaIO::xmlGetNodeContent(xmlpp::Node* pNode, const std::string& nodename)
{
	xmlpp::Node* tmpNode= xmlGetNode(pNode, nodename);

	if (tmpNode!=NULL) {
		xmlpp::Node* tmpNode2= xmlGetNode(tmpNode, "text"); //Try to retrieve text content
		if (tmpNode2!=NULL) {
			const xmlpp::TextNode* nodeText = dynamic_cast<const xmlpp::TextNode*>(tmpNode2);
			return std::string(nodeText->get_content());
		}
	}

	return std::string("");
}


xmlpp::Node* BormaIO::xmlGetNode(xmlpp::Node* parentNode, const std::string& nodename)
{
	if (xmlGetNodeName(parentNode)==nodename) {
		return parentNode;
	}

	xmlpp::Node::NodeList list = parentNode->get_children();

	for(xmlpp::Node::NodeList::iterator iter = list.begin(); iter != list.end(); ++iter) {
		//xmlpp::Node* tmpNode= *iter;
		//cout << tmpNode->get_name() << endl;

		xmlpp::Node* tmpNode2= (xmlGetNode(*iter, nodename));
		if (tmpNode2!=NULL) {
			return tmpNode2;
		}
	}

	return NULL;
}

std::string BormaIO::xmlGetNodeName(xmlpp::Node* pNode)
{
	std::string nodename = pNode->get_name();
	return nodename;
}

414
Date BormaIO::stringToDate(const std::string& instr) const
415
{
416
417
418
419
420
421
422
423
424
425
426
	double year;
	IOUtils::convertString(year, instr.substr(0,2));

	Date date_out;
	//some Y2K support
	if(year<pivot_year)
		IOUtils::convertString(date_out, "20"+instr.substr(0,10) , in_tz);
	else
		IOUtils::convertString(date_out, "19"+instr.substr(0,10) , in_tz);

	return date_out;
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
}

bool BormaIO::validFilename(const std::string& tmp) const
{
	size_t pos = tmp.find_first_not_of("0123456789");//Filename must start with 10 numbers
	if (pos!=10) {
		return false;
	}

	return true;
}

void BormaIO::read2DMeteo(const Date& date_in, std::vector<MeteoData>& meteo_out)
{
	std::vector<StationData> vecStation;
	read2DMeteo(date_in, meteo_out, vecStation);
}

void BormaIO::read2DMeteo(const Date& date_in, std::vector<MeteoData>& vecMeteo, std::vector<StationData>& vecStation)
{
	vecMeteo.clear();
	vecStation.clear();

	//Read in the StationNames
	std::string xmlpath="", str_stations="";
	int stations=0;

	cfg.getValue("NROFSTATIONS", "Input", str_stations);
455
	cfg.getValue("METEOPATH", "Input", xmlpath);
456
457
458
459
460
461
462
463

	if (!IOUtils::convertString(stations, str_stations, std::dec)) {
		throw ConversionFailedException("Error while reading value for NROFSTATIONS", AT);
	}

	for (int ii=0; ii<stations; ii++) {
		std::stringstream tmp_stream;
		std::string stationname="", tmp_file="";
464
		Date tmp_date(0.0, 0.); //not a problem, the date will be overwritten
465
466
467
468
469
470
471
472
473
		MeteoData md;
		StationData sd;

		tmp_stream << (ii+1); //needed to construct key name
		cfg.getValue(std::string("STATION"+tmp_stream.str()), "Input", stationname);

		checkForMeteoFiles(xmlpath, stationname, date_in, tmp_file, tmp_date);
		//Check whether file was found
		if (tmp_date<date_in) {
474
475
476
477
			stringstream ss;
			ss << "No XML file in path '" << xmlpath << "' found for date ";
			ss << date_in.toString(Date::ISO) << " for station " << stationname;
			throw FileNotFoundException(ss.str(), AT);
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
		}

		//Read in data from XML File
		xmlExtractData(xmlpath+"/"+tmp_file, tmp_date, md, sd);

		vecMeteo.push_back(md);
		vecStation.push_back(sd);
	}
}

void BormaIO::readAssimilationData(const Date&, Grid2DObject&)
{
	//Nothing so far
	throw IOException("Nothing implemented here", AT);
}

void BormaIO::readSpecialPoints(std::vector<Coords>&)
{
	//Nothing so far
	throw IOException("Nothing implemented here", AT);
}

void BormaIO::write2DGrid(const Grid2DObject&, const std::string& name)
{
	//Nothing so far
	(void)name;
	throw IOException("Nothing implemented here", AT);
}

void BormaIO::convertUnits(MeteoData& meteo)
{
	meteo.standardizeNodata(plugin_nodata);

	//converts C to Kelvin, converts ilwr to ea, converts RH to [0,1]
	if(meteo.ta!=IOUtils::nodata) {
		meteo.ta=C_TO_K(meteo.ta);
	}

	if(meteo.tsg!=IOUtils::nodata) {
		meteo.tsg=C_TO_K(meteo.tss);
	}

	if(meteo.tss!=IOUtils::nodata) {
		meteo.tss=C_TO_K(meteo.tss);
	}

	if(meteo.rh!=IOUtils::nodata) {
		meteo.rh /= 100.;
	}
}

#ifndef _METEOIO_JNI
extern "C"
{
532
533
534
#define COMPILE_PLUGIN
#include "exports.h"

535
	METEOIO_EXPORT void deleteObject(void* obj) {
536
537
538
		delete reinterpret_cast<PluginObject*>(obj);
	}

539
	METEOIO_EXPORT void* loadObject(const string& classname, const Config& cfg) {
540
541
		if(classname == "BormaIO") {
			//cerr << "Creating dynamic handle for " << classname << endl;
542
			return new BormaIO(deleteObject, cfg);
543
544
545
546
547
548
549
550
		}
		//cerr << "Could not load " << classname << endl;
		return NULL;
	}
}
#endif

} //namespace