WSL/SLF GitLab Repository

DBO.cc 21.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
/***********************************************************************************/
/*  Copyright 2017 SLF                                                                                                                                */
/***********************************************************************************/
/* This file is part of MeteoIO.
    MeteoIO is free software: you can redistribute it and/or modify
    it under the terms of the GNU Lesser General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    MeteoIO is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public License
    along with MeteoIO.  If not, see <http://www.gnu.org/licenses/>.
*/
#include <meteoio/plugins/DBO.h>

#include <meteoio/dataClasses/Coords.h>
#include <meteoio/IOExceptions.h>
#include <meteoio/meteoLaws/Meteoconst.h>

#include <algorithm>
#include <sstream>
#include <iostream>

#include <curl/curl.h>
29
#include <meteoio/plugins/picojson.h>
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

using namespace std;

namespace mio {
/**
 * @page dbo DBO
 * @section dbo_format Format
 * This plugin reads meteorological data from DBO
 * via the RESTful web service. To compile the plugin you need to have the <a href="http://curl.haxx.se/">CURL library</a> with its headers present.
 *
 * @section dbo_keywords Keywords
 * This plugin uses the following keywords:
 * - DBO_URL: The URL of the RESTful web service e.g.http://developwis.wsl.ch:8730/osper-api
 * - DBO_USER: The username to access the service (optional)
 * - DBO_PASS: The password to authenticate the USER (optional)
45
 * - STATION#: station code for the given station, prefixed by the network it belongs ot (for example: IMIS::SLF2)
46
47
48
49
50
51
52
53
54
55
56
 * - DBO_TIMEOUT: timeout (in seconds) for the connection to the server (default: 60s)
 * - DBO_DEBUG: print the full requests/answers from the server when something does not work as expected
 *
 * @code
 * METEO	= DBO
 * DBO_URL	= http://developwis.wsl.ch:8730/osper-api
 * DBO_USER	= mylogin
 * DBO_PASS	= mypasswd
 * STATION1	= wind_tunnel_meteo
 * @endcode
 *
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
 * @section dbo_dependencies Picojson
 * This plugin relies on <A HREF="https://github.com/kazuho/picojson/">picojson</A> for reading and parsing
 * <A HREF="https://en.wikipedia.org/wiki/JSON">JSON</A> data. Picojson is released under a
 * <A HREF="https://opensource.org/licenses/BSD-2-Clause">2-Clause BSD License</A>. Please find here below
 * the full license agreement for picojson:
 *
 * @code
 * Copyright 2009-2010 Cybozu Labs, Inc.
 * Copyright 2011-2014 Kazuho Oku
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 * @endcode
90
91
 */

92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
//we keep it as a simple function in order to avoid exposing picojson stuff in the header
void printJSON(const picojson::value& v, const unsigned int& depth)
{
	if (v.is<picojson::null>()) {
		for (unsigned int jj=0; jj<depth; jj++) std::cout << "\t";
		std::cout << "NULL\n";
		return;
	}

	if (v.is<picojson::object>()) {
		const picojson::value::object& obj = v.get<picojson::object>();
		for (picojson::value::object::const_iterator ii = obj.begin(); ii != obj.end(); ++ii) {
			for (unsigned int jj=0; jj<depth; jj++) std::cout << "\t";
			std::cout << ii->first << "\n";
			printJSON(ii->second, depth+1);
		}
	} else if (v.is<std::string>()){
		for (unsigned int jj=0; jj<depth; jj++) std::cout << "\t";
		std::cout << v.get<std::string>() << "\n";
	} else if (v.is<double>()){
		for (unsigned int jj=0; jj<depth; jj++) std::cout << "\t";
		std::cout << v.get<double>() << "\n";
	} else if (v.is<bool>()){
		for (unsigned int jj=0; jj<depth; jj++) std::cout << "\t";
		std::cout << std::boolalpha << v.get<bool>() << "\n";
	} else if (v.is<picojson::array>()){ //ie vector<picojson::value>
		for (unsigned int jj=0; jj<depth; jj++) std::cout << "\t";
		const picojson::array& array = v.get<picojson::array>();
		std::cout << "array " << array.size() << "\n";
		for (size_t jj=0; jj<array.size(); jj++)
			printJSON(array[jj], depth+1);
	}
}

126
picojson::value goToJSONPath(const std::string& path, picojson::value& v)
127
128
129
130
131
132
133
134
135
136
137
{
	size_t start_pos = 0;
	if (path[0]=='$') start_pos++;
	if (path[1]=='.') start_pos++;

	const size_t end_pos = path.find(".", start_pos);
	const std::string local_path = (end_pos!=std::string::npos)? path.substr(start_pos, end_pos-start_pos) : path.substr(start_pos);
	const std::string remaining_path = (end_pos!=std::string::npos)? path.substr(end_pos+1) : "";

	if (v.is<picojson::object>()) {
		picojson::value::object& obj = v.get<picojson::object>();
138
		for (std::map<std::string,picojson::value>::iterator it = obj.begin(); it != obj.end(); ++it) {
139
140
			if (it->first==local_path) {
				if (!remaining_path.empty())
141
					goToJSONPath(remaining_path, it->second);
142
				else
143
					return it->second;
144
145
146
			}
		}
	}
147
148

	return picojson::value();
149
150
}

151
void JSONQuery(const std::string& path, picojson::value& v, std::vector<picojson::value>& results)
152
153
154
155
156
157
158
159
160
161
162
163
164
{
	if (v.is<picojson::null>()) return;

	size_t start_pos = 0;
	if (path[0]=='$') start_pos++;
	if (path[1]=='.') start_pos++;

	const size_t end_pos = path.find(".", start_pos);
	const std::string local_path = (end_pos!=std::string::npos)? path.substr(start_pos, end_pos-start_pos) : path.substr(start_pos);
	const std::string remaining_path = (end_pos!=std::string::npos)? path.substr(end_pos+1) : "";

	if (v.is<picojson::object>()) {
		picojson::value::object& obj = v.get<picojson::object>();
165
		for (std::map<std::string,picojson::value>::iterator it = obj.begin(); it != obj.end(); ++it) {
166
			if (it->first==local_path) {
167
168
169
170
				if (!remaining_path.empty()) {
					 if (it->second.is<picojson::array>()){ //ie vector<picojson::value>
						picojson::array& array = it->second.get<picojson::array>();
						for (size_t jj=0; jj<array.size(); jj++)
171
							JSONQuery(remaining_path, array[jj], results);
172
					} else
173
						JSONQuery(remaining_path, it->second, results);
174
				} else {
175
					results.push_back( it->second );
176
				}
177
178
179
180
181
182
183
184
			}
		}
	}
}

std::string getString(const std::string& path, picojson::value& v)
{
	std::vector<picojson::value> results;
185
	JSONQuery(path, v, results);
186
187
188
189
190
191
192
	if (!results.empty()) {
		if (! results.front().is<picojson::null>() &&  results.front().is<std::string>()) return  results.front().get<std::string>();
	}

	return std::string();
}

193
194
195
std::vector<std::string> getStrings(const std::string& path, picojson::value& v)
{
	std::vector<picojson::value> results;
196
	JSONQuery(path, v, results);
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211

	std::vector<std::string> vecString;
	for (size_t ii=0; ii<results.size(); ii++) {
		 if (results[ii].is<picojson::array>()){ //ie vector<picojson::value>
			const picojson::array& array = results[ii].get<picojson::array>();
			for (size_t jj=0; jj<array.size(); jj++) {
				if (! array[jj].is<picojson::null>() &&  array[jj].is<std::string>()) vecString.push_back( array[jj].get<std::string>() );
			}
		} else
			if (! results[ii].is<picojson::null>() &&  results[ii].is<std::string>()) vecString.push_back( results[ii].get<std::string>() );
	}

	return vecString;
}

212
213
214
double getDouble(const std::string& path, picojson::value& v)
{
	std::vector<picojson::value> results;
215
	JSONQuery(path, v, results);
216
217
218
219
220
221
222
223
224
225
	if (!results.empty()) {
		if (! results.front().is<picojson::null>() &&  results.front().is<double>()) return  results.front().get<double>();
	}

	return IOUtils::nodata;
}

std::vector<double> getDoubles(const std::string& path, picojson::value& v)
{
	std::vector<picojson::value> results;
226
	JSONQuery(path, v, results);
227
228
229
230
231
232
233
234

	std::vector<double> vecDouble;
	for (size_t ii=0; ii<results.size(); ii++) {
		 if (results[ii].is<picojson::array>()){ //ie vector<picojson::value>
			const picojson::array& array = results[ii].get<picojson::array>();
			for (size_t jj=0; jj<array.size(); jj++) {
				if (! array[jj].is<picojson::null>() &&  array[jj].is<double>()) vecDouble.push_back( array[jj].get<double>() );
			}
235
236
		} else
			if (! results[ii].is<picojson::null>() &&  results[ii].is<double>()) vecDouble.push_back( results[ii].get<double>() );
237
238
239
240
241
	}

	return vecDouble;
}

242
243
//converts C to Kelvin, converts RH to [0,1], HS to m
double convertUnits(const MeteoData::Parameters& param, const double& value)
Mathias Bavay's avatar
Mathias Bavay committed
244
{
245
246
	if (value==IOUtils::nodata) return value;

247
248
249
250
251
252
253
254
	switch (param) {
		case MeteoData::TA: case MeteoData::TSG: case MeteoData::TSS:
			return IOUtils::C_TO_K(value);
		case MeteoData::RH: case MeteoData::HS:
			return value/100.;
		default:
			return value;
	}
Mathias Bavay's avatar
Mathias Bavay committed
255
256
}

257
bool parseTsPoint(const picojson::value& v, Date& datum, double& value)
Mathias Bavay's avatar
Mathias Bavay committed
258
{
259
260
261
262
263
264
265
266
267
268
269
270
	if (!v.is<picojson::array>()) return false;

	const picojson::array& array = v.get<picojson::array>();
	if (array.size()!=2) return false;

	if (array[0].is<std::string>())
		IOUtils::convertString(datum, array[0].get<std::string>(), 0.); //HACK: hard-coding GMT
	else
		return false;

	if (array[1].is<double>())
		value = array[1].get<double>();
271
272
273
274
275
	else {
		if (!array[1].is<picojson::null>()) return false;
		value = IOUtils::nodata;
		return true;
	}
Mathias Bavay's avatar
Mathias Bavay committed
276
277
278
279

	return true;
}

280
void parseTimeSerie(const std::string& tsID, const MeteoData::Parameters& param, const StationData& sd, picojson::value& v, std::vector<MeteoData>& vecMeteo)
Mathias Bavay's avatar
Mathias Bavay committed
281
{
282
283
284
285
286
287
288
289
290
291
292
	picojson::value ts( goToJSONPath("$.measurements", v) );
	if (!ts.is<picojson::array>())
		throw InvalidFormatException("Could not parse timeserie "+tsID, AT);
	const picojson::array& vecRaw = ts.get<picojson::array>();

	if (vecMeteo.empty()) {
		vecMeteo.reserve(vecRaw.size());
		for (size_t ii=0; ii<vecRaw.size(); ii++) {
			Date datum;
			double value;
			if (!parseTsPoint(vecRaw[ii], datum, value)) {
293
				printJSON(vecRaw[ii], 1);
294
295
296
297
298
299
300
301
302
303
304
305
306
				std::ostringstream ss; ss << "Error parsing element " << ii << " of timeserie " << tsID;
				throw InvalidFormatException(ss.str(), AT);
			}

			MeteoData md(datum, sd);
			md(param) = convertUnits(param, value);
			vecMeteo.push_back( md );
		}
	} else {
		for (size_t ii=0; ii<vecRaw.size(); ii++) {
			Date datum;
			double value;
			if (!parseTsPoint(vecRaw[ii], datum, value)) {
307
				printJSON(vecRaw[ii], 1);
308
309
310
311
				std::ostringstream ss; ss << "Error parsing element " << ii << " of timeserie " << tsID;
				throw InvalidFormatException(ss.str(), AT);
			}

312
313
314
315
316
317
318
319
320
321
			//easy case: all TS have the same indices
			if (ii<vecMeteo.size()) {
				if (datum==vecMeteo[ii].date) {
					vecMeteo[ii](param) = convertUnits(param, value);
					continue;
				}
			}

			//hard case: the TS don't match HACK

322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
		}
	}
}

unsigned int parseInterval(const std::string& interval_str)
{
	unsigned int hour, minute, second;
	if (sscanf(interval_str.c_str(), "%u:%u:%u", &hour, &minute, &second) == 3) {
		return (hour*3600 + minute*60 + second);
	} else
		throw ConversionFailedException("Could not read aggregation interval '"+interval_str+"'", AT);
}

std::map<std::string, std::vector<DBO::tsMeta> > getTsProperties(picojson::value& v)
{
	std::map<std::string, std::vector<DBO::tsMeta> > tsMap;
Mathias Bavay's avatar
Mathias Bavay committed
338
	std::vector<picojson::value> results;
339
	JSONQuery("$.properties.timeseries", v, results);
Mathias Bavay's avatar
Mathias Bavay committed
340
341
342
343
344
345

	for (size_t ii=0; ii<results.size(); ii++) {
		 if (results[ii].is<picojson::array>()){
			const picojson::array& array = results[ii].get<picojson::array>();
			for (size_t jj=0; jj<array.size(); jj++) {
				if (! array[jj].is<picojson::null>()) {
346
					std::string code, device_code, agg_type;
347
					double id;
Mathias Bavay's avatar
Mathias Bavay committed
348
349
350
351
352
353
					unsigned int interval;
					Date since, until;

					const picojson::value::object& obj = array[jj].get<picojson::object>();
					for (picojson::value::object::const_iterator it = obj.begin(); it != obj.end(); ++it) {
						if (it->first=="code" && it->second.is<std::string>()) code = it->second.get<std::string>();
354
						if (it->first=="deviceCode" && it->second.is<std::string>()) device_code = it->second.get<std::string>();
355
						if (it->first=="id" && it->second.is<double>()) id = it->second.get<double>();
Mathias Bavay's avatar
Mathias Bavay committed
356
357
358
359
360
361
						if (it->first=="since" && it->second.is<std::string>()) IOUtils::convertString(since, it->second.get<std::string>(), 0.);
						if (it->first=="until" && it->second.is<std::string>()) IOUtils::convertString(until, it->second.get<std::string>(), 0.);
						if (it->first=="aggregationType" && it->second.is<std::string>()) agg_type = it->second.get<std::string>();
						if (it->first=="aggregationInterval" && it->second.is<std::string>()) interval = parseInterval(it->second.get<std::string>());
					}

362
363
364
					if (device_code=="BATTERY" || device_code=="LOGGER") break;
					if (agg_type=="SD") break; //we don't care about standard deviation anyway

365
366
					const std::string param_str( IOUtils::strToUpper( code.substr(0, code.find('_')) ) );
					tsMap[param_str].push_back( DBO::tsMeta(since, until, agg_type, id, interval) );
Mathias Bavay's avatar
Mathias Bavay committed
367
368
369
370
371
372
373
374
				}
			}
		}
	}

	return tsMap;
}

375
/*************************************************************************************************/
376
const int DBO::http_timeout_dflt = 60; // seconds until connect time out for libcurl
377
378
const std::string DBO::metadata_endpoint = "/osper-api/osper/stations/";
const std::string DBO::data_endpoint = "/osper-api/osper/timeseries/";
379
380
381
const std::string DBO::null_string = "null";

DBO::DBO(const std::string& configfile)
Mathias Bavay's avatar
Mathias Bavay committed
382
      : cfg(configfile), vecStationName(), vecMeta(), vecTsMeta(),
383
        coordin(), coordinparam(), coordout(), coordoutparam(),
384
        endpoint(), default_timezone(1.),
385
386
387
        http_timeout(http_timeout_dflt), dbo_debug(false)
{
	initDBOConnection();
388
	IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
389
390
391
392
	cfg.getValues("STATION", "INPUT", vecStationName); //reads station names into vector<string> vecStationName
}

DBO::DBO(const Config& cfgreader)
Mathias Bavay's avatar
Mathias Bavay committed
393
      : cfg(cfgreader), vecStationName(), vecMeta(), vecTsMeta(),
394
        coordin(), coordinparam(), coordout(), coordoutparam(),
395
        endpoint(), default_timezone(1.),
396
397
398
        http_timeout(http_timeout_dflt), dbo_debug(false)
{
	initDBOConnection();
399
	IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
	cfg.getValues("STATION", "INPUT", vecStationName); //reads station names into vector<string> vecStationName
}

void DBO::initDBOConnection() {
	curl_global_init(CURL_GLOBAL_ALL);

	cfg.getValue("DBO_TIMEOUT", "Input", http_timeout, IOUtils::nothrow);
	cfg.getValue("TIME_ZONE", "Input", default_timezone, IOUtils::nothrow);

	cfg.getValue("DBO_URL", "Input", endpoint);
	if (*endpoint.rbegin() != '/') endpoint += "/";
	cerr << "[i] Using DBO URL: " << endpoint << endl;

	cfg.getValue("DBO_DEBUG", "INPUT", dbo_debug, IOUtils::nothrow);
}

void DBO::readStationData(const Date& /*date*/, std::vector<StationData>& vecStation)
{
	vecStation.clear();
419
420
421
422
423
424
425
426
	if (vecMeta.empty()) fillStationMeta();
	vecStation = vecMeta;
}

void DBO::readMeteoData(const Date& dateStart, const Date& dateEnd,
                          std::vector< std::vector<MeteoData> >& vecMeteo)
{
	vecMeteo.clear();
427
	if (vecMeta.empty()) fillStationMeta();
428

429
430
	vecMeteo.resize(vecMeta.size());
	for(size_t ii=0; ii<vecMeta.size(); ii++)
431
432
433
434
435
436
		readData(dateStart, dateEnd, vecMeteo[ii], ii);
}

void DBO::fillStationMeta()
{
	vecMeta.clear();
Mathias Bavay's avatar
Mathias Bavay committed
437
	vecTsMeta.resize( vecStationName.size() );
438

439
	for(size_t ii=0; ii<vecStationName.size(); ii++) {
440
441
442
		std::string station_id( vecStationName[ii] );
		if (station_id.find(':')==std::string::npos) station_id = "IMIS::" + station_id;
		const std::string request( metadata_endpoint + "name=" + IOUtils::strToLower( station_id ) );
443

444
		std::stringstream ss;
445
		if (curl_read(request, ss)) {
446
			if (ss.str().empty()) throw UnknownValueException("Station not found: '"+station_id+"'", AT);
447

448
449
450
451
452
			picojson::value v;
			const std::string err( picojson::parse(v, ss.str()) );
			if (!err.empty()) throw IOException("Error while parsing JSON: "+err, AT);

			const std::vector<double> coordinates( getDoubles("$.geometry.coordinates", v) );
453
			if (coordinates.size()!=3) throw InvalidFormatException("Wrong coordinates specification!", AT);
454
455
456
457

			Coords position(coordin, coordinparam);
			position.setLatLon(coordinates[1], coordinates[0], coordinates[2]);
			const StationData sd(position, getString("$.properties.name", v), getString("$.properties.locationName", v));
458
			vecMeta.push_back( sd );
459
460

			//select proper time series
Mathias Bavay's avatar
Mathias Bavay committed
461
			vecTsMeta[ii] = getTsProperties(v);
462
463
464
465
466
467
		} else {
			if (dbo_debug)
				std::cout << "****\nRequest: " << request << "\n****\n";
			throw IOException("Could not retrieve data for station " + station_id, AT);
		}
	}
468
}
Mathias Bavay's avatar
Mathias Bavay committed
469

470
471
472
//read all data for the given station
void DBO::readData(const Date& dateStart, const Date& dateEnd, std::vector<MeteoData>& vecMeteo, const size_t& stationindex)
{
Mathias Bavay's avatar
Mathias Bavay committed
473
	//debug info
474
475
	/*for(size_t ii=0; ii<vecStationName.size(); ii++) {
		for (std::map<std::string, std::vector<DBO::tsMeta> >::iterator it = vecTsMeta[ii].begin(); it != vecTsMeta[ii].end(); ++it) {
Mathias Bavay's avatar
Mathias Bavay committed
476
			for(size_t jj=0; jj<it->second.size(); jj++)
477
				std::cout << it->first << " " << it->second[jj].toString() << "\n";
Mathias Bavay's avatar
Mathias Bavay committed
478
		}
479
	}*/
480

481
482
483
	//TODO: for station stationindex, loop over the timeseries that cover [dateStart, dateEnd] for the current station
	//vecTsMeta[ stationindex ]

484
485
486
487
488
489
490
491
492
493
494
495
496
497
	/*for (std::map<std::string, std::vector<DBO::tsMeta> >::iterator it = vecTsMeta[stationindex].begin(); it != vecTsMeta[stationindex].end(); ++it) {
		for(size_t jj=0; jj<it->second.size(); jj++)
			std::cout << it->first << " " << it->second[jj].toString() << "\n";
	}*/

	readTimeSerie(15, MeteoData::TA, dateStart, dateEnd, vecMeta[stationindex], vecMeteo);
	readTimeSerie(23, MeteoData::RH, dateStart, dateEnd, vecMeta[stationindex], vecMeteo);
	readTimeSerie(93, MeteoData::RSWR, dateStart, dateEnd, vecMeta[stationindex], vecMeteo);
	readTimeSerie(31, MeteoData::HS, dateStart, dateEnd, vecMeta[stationindex], vecMeteo);
	readTimeSerie(46, MeteoData::TSG, dateStart, dateEnd, vecMeta[stationindex], vecMeteo);

	/*for (size_t param=MeteoData::firstparam; param<=MeteoData::lastparam; param++) {

	}*/
498
499
}

500
void DBO::readTimeSerie(const unsigned int& ts_id, const MeteoData::Parameters& param, const Date& dateStart, const Date& dateEnd, const StationData& sd, std::vector<MeteoData>& vecMeteo)
501
502
{
	std::ostringstream ss_ID; ss_ID << ts_id;
503
	const std::string base_url( data_endpoint + ss_ID.str() );
504
505
506
	//const std::string period( "?from=" + dateStart.toString(Date::ISO_TZ) + "&until=" + dateEnd.toString(Date::ISO_TZ) );
	const std::string period( string("?from=2016-11-17T13:00Z") + "&until=" + "2017-01-05T13:00Z" );
	const std::string request( base_url + period );
507

508
	std::stringstream ss;
509
	if (curl_read(request, ss)) {
510
		if (ss.str().empty()) throw UnknownValueException("Timeseries not found: '"+ss_ID.str()+"'", AT);
511
512
		picojson::value v;
		const std::string err( picojson::parse(v, ss.str()) );
513
		if (!err.empty()) throw IOException("Error while parsing JSON: "+err, AT);
514

515
		parseTimeSerie(ss_ID.str(), param, sd, v, vecMeteo);
516
517
518
	} else {
		if (dbo_debug)
			std::cout << "****\nRequest: " << request << "\n****\n";
519
		throw IOException("Could not retrieve data for timeseries " + ss_ID.str(), AT);
520
521
522
523
524
525
526
	}
}

size_t DBO::data_write(void* buf, size_t size, size_t nmemb, void* userp)
{
	if (userp) {
		ostream& os = *static_cast<ostream*>(userp);
527
		const std::streamsize len = size * nmemb;
528
529
530
531
532
533
534
535
536
537
538
539

		if (os.write(static_cast<char*>(buf), len)) return len;
	}

	return 0;
}

bool DBO::curl_read(const std::string& url_query, std::ostream& os)
{
	CURLcode code(CURLE_FAILED_INIT);
	CURL* curl = curl_easy_init();

540
	const std::string url( endpoint + url_query );
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564

	if (curl) {
		if (CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &data_write))
		   && CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L))
		   && CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L))
		   && CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_FILE, &os))
		   && CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_TIMEOUT, DBO::http_timeout))
		   && CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_URL, url.c_str())))
		{
			code = curl_easy_perform(curl);
		}
		curl_easy_cleanup(curl);
	}

	if (code!=CURLE_OK) {
		if (dbo_debug)
			std::cout << "****\nRequest: " << url_query << "\n****\n";
		std::cout << "[E] " << curl_easy_strerror(code) << "\t";
	}

	return (code==CURLE_OK);
}

} //namespace