Added the structure for the upcoming DBO plugin, fixed an invalid copyright in...

Added the structure for the upcoming DBO plugin, fixed an invalid copyright in GSNIO (the old plugin was ©epfl, the new one is ©slf), added a note to the oshd plugin.

CMakeLists.txt

@@ -91,6 +91,7 @@ SET(PLUGIN_ARPSIO ON CACHE BOOL "Compilation ARPSIO ON or OFF")
 SET(PLUGIN_BORMAIO OFF CACHE BOOL "Compilation BORMAIO ON or OFF")
 SET(PLUGIN_CNRMIO OFF CACHE BOOL "Compilation CNRMIO ON or OFF")
 SET(PLUGIN_COSMOXMLIO OFF CACHE BOOL "Compilation COSMOXMLIO ON or OFF")
+SET(PLUGIN_DBO OFF CACHE BOOL "Compilation DBO ON or OFF")
 SET(PLUGIN_GEOTOPIO ON CACHE BOOL "Compilation GEOTOPIO ON or OFF")
 SET(PLUGIN_GRASSIO ON CACHE BOOL "Compilation GRASSIO ON or OFF")
 SET(PLUGIN_GRIBIO OFF CACHE BOOL "Compilation GRIBIO ON or OFF")

meteoio/IOHandler.cmake.cc

@@ -29,6 +29,7 @@
 #cmakedefine PLUGIN_A3DIO
 #cmakedefine PLUGIN_ARPSIO
 #cmakedefine PLUGIN_CNRMIO
+#cmakedefine PLUGIN_DBO
 #cmakedefine PLUGIN_GRASSIO
 #cmakedefine PLUGIN_GEOTOPIO
 #cmakedefine PLUGIN_SMETIO
@@ -67,6 +68,10 @@
 #include <meteoio/plugins/CosmoXMLIO.h>
 #endif
 
+#ifdef PLUGIN_DBO
+#include <meteoio/plugins/DBO.h>
+#endif
+
 #ifdef PLUGIN_IMISIO
 #include <meteoio/plugins/ImisIO.h>
 #endif
@@ -138,6 +143,7 @@ namespace mio {
  * <tr><td>\subpage borma "BORMA"</td><td>meteo</td><td>Borma xml meteo files</td><td><A HREF="http://libxmlplusplus.sourceforge.net/">libxml++</A></td></tr>
  * <tr><td>\subpage cnrm "CNRM"</td><td>dem, grid2d, meteo</td><td>NetCDF meteorological timeseries following the <A HREF="http://www.cnrm.meteo.fr/?lang=en">CNRM</A> schema</td><td><A HREF="http://www.unidata.ucar.edu/downloads/netcdf/index.jsp">NetCDF-C library</A></td></tr>
  * <tr><td>\subpage cosmoxml "COSMOXML"</td><td>meteo</td><td>MeteoSwiss COSMO's postprocessing XML format</td><td><A HREF="http://xmlsoft.org/">libxml2</A></td></tr>
+ * <tr><td>\subpage dbo "DBO"</td><td>meteo</td><td>connects to SLF's DBO web service interface</td><td><A HREF="http://curl.haxx.se/libcurl/">libcurl</A></td></tr>
  * <tr><td>\subpage geotop "GEOTOP"</td><td>meteo</td><td>GeoTop meteo files</td><td></td></tr>
  * <tr><td>\subpage grass "GRASS"</td><td>dem, landuse, grid2d</td><td>Grass grid files</td><td></td></tr>
  * <tr><td>\subpage gribio "GRIB"</td><td>meteo, dem, grid2d</td><td>GRIB meteo grid files</td><td><A HREF="http://www.ecmwf.int/products/data/software/grib_api.html">grib-api</A></td></tr>
@@ -311,6 +317,9 @@ IOInterface* IOHandler::getPlugin(const std::string& plugin_name) const
 #ifdef PLUGIN_COSMOXMLIO
 	if (plugin_name == "COSMOXML") return new CosmoXMLIO(cfg);
 #endif
+#ifdef PLUGIN_DBO
+	if (plugin_name == "DBO") return new DBO(cfg);
+#endif
 #ifdef PLUGIN_CNRMIO
 	if (plugin_name == "CNRM") return new CNRMIO(cfg);
 #endif

meteoio/plugins/CMakeLists.txt

@@ -33,6 +33,13 @@ IF(PLUGIN_COSMOXMLIO)
 	SET(plugins_sources ${plugins_sources} plugins/CosmoXMLIO.cc)
 ENDIF(PLUGIN_COSMOXMLIO)
 
+IF(PLUGIN_DBO)
+	FIND_PACKAGE(CURL REQUIRED)
+	INCLUDE_DIRECTORIES(SYSTEM ${CURL_INCLUDE_DIRS})
+	SET(plugin_libs ${plugin_libs} ${CURL_LIBRARIES})
+	SET(plugins_sources ${plugins_sources} plugins/DBO.cc)
+ENDIF(PLUGIN_DBO)
+
 IF(PLUGIN_GRASSIO)
 	SET(plugins_sources ${plugins_sources} plugins/GrassIO.cc)
 ENDIF(PLUGIN_GRASSIO)

meteoio/plugins/DBO.cc

+/***********************************************************************************/
+/*  Copyright 2017 SLF                                                                                                                                */
+/***********************************************************************************/
+/* This file is part of MeteoIO.
+    MeteoIO is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    MeteoIO is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with MeteoIO.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#include <meteoio/plugins/DBO.h>
+
+#include <meteoio/dataClasses/Coords.h>
+#include <meteoio/IOExceptions.h>
+#include <meteoio/meteoLaws/Meteoconst.h>
+
+#include <algorithm>
+#include <sstream>
+#include <iostream>
+
+#include <curl/curl.h>
+
+using namespace std;
+
+namespace mio {
+/**
+ * @page dbo DBO
+ * @section dbo_format Format
+ * This plugin reads meteorological data from DBO
+ * via the RESTful web service. To compile the plugin you need to have the <a href="http://curl.haxx.se/">CURL library</a> with its headers present.
+ *
+ * @section dbo_keywords Keywords
+ * This plugin uses the following keywords:
+ * - DBO_URL: The URL of the RESTful web service e.g.http://developwis.wsl.ch:8730/osper-api
+ * - DBO_USER: The username to access the service (optional)
+ * - DBO_PASS: The password to authenticate the USER (optional)
+ * - STATION#: station code for the given station, e. g. la_fouly_1034 (case sensitive!)
+ * - DBO_TIMEOUT: timeout (in seconds) for the connection to the server (default: 60s)
+ * - DBO_DEBUG: print the full requests/answers from the server when something does not work as expected
+ *
+ * @code
+ * METEO	= DBO
+ * DBO_URL	= http://developwis.wsl.ch:8730/osper-api
+ * DBO_USER	= mylogin
+ * DBO_PASS	= mypasswd
+ * STATION1	= wind_tunnel_meteo
+ * @endcode
+ *
+ */
+
+const int DBO::http_timeout_dflt = 60; // seconds until connect time out for libcurl
+const std::string DBO::sensors_endpoint = "sensors";
+const std::string DBO::sensors_format = "format=csv";
+const std::string DBO::null_string = "null";
+
+DBO::DBO(const std::string& configfile)
+      : cfg(configfile), vecStationName(),
+        endpoint(), userid(), passwd(), default_timezone(1.),
+        http_timeout(http_timeout_dflt), dbo_debug(false)
+{
+	initDBOConnection();
+	cfg.getValues("STATION", "INPUT", vecStationName); //reads station names into vector<string> vecStationName
+}
+
+DBO::DBO(const Config& cfgreader)
+      : cfg(cfgreader), vecStationName(),
+        endpoint(), userid(), passwd(), default_timezone(1.),
+        http_timeout(http_timeout_dflt), dbo_debug(false)
+{
+	initDBOConnection();
+	cfg.getValues("STATION", "INPUT", vecStationName); //reads station names into vector<string> vecStationName
+}
+
+void DBO::initDBOConnection() {
+	curl_global_init(CURL_GLOBAL_ALL);
+
+	cfg.getValue("DBO_TIMEOUT", "Input", http_timeout, IOUtils::nothrow);
+	cfg.getValue("TIME_ZONE", "Input", default_timezone, IOUtils::nothrow);
+
+	cfg.getValue("DBO_URL", "Input", endpoint);
+	if (*endpoint.rbegin() != '/') endpoint += "/";
+	cerr << "[i] Using DBO URL: " << endpoint << endl;
+
+	cfg.getValue("DBO_USER", "Input", userid, IOUtils::nothrow);
+	cfg.getValue("DBO_PASS", "Input", passwd, IOUtils::nothrow);
+	cfg.getValue("DBO_DEBUG", "INPUT", dbo_debug, IOUtils::nothrow);
+}
+
+void DBO::readStationData(const Date& /*date*/, std::vector<StationData>& vecStation)
+{
+	vecStation.clear();
+
+}
+
+void DBO::readMeteoData(const Date& /*dateStart*/, const Date& /*dateEnd*/,
+                          std::vector< std::vector<MeteoData> >& vecMeteo)
+{
+	vecMeteo.clear();
+}
+
+//this method is called on each station in order to parse the header and set the metadata
+bool DBO::parseMetadata(std::stringstream& /*ss*/, StationData &/*sd*/, std::string &/*fields*/, std::string &/*units*/) const
+{
+
+	return true;
+}
+
+void DBO::readData(const Date& dateStart, const Date& dateEnd, std::vector<MeteoData>& vecMeteo, const size_t& stationindex)
+{
+	const std::string station_id = vecStationName[stationindex];
+	const string anon_request = sensors_endpoint + "/" + IOUtils::strToLower( station_id ) + "?" + sensors_format + "&from=" + dateStart.toString(Date::ISO)
+	                            + "&to=" + dateEnd.toString(Date::ISO);
+	const string auth = "&username=" + userid + "&password=" + passwd;
+	const string request = (!userid.empty())? anon_request+auth : anon_request;
+
+	stringstream ss;
+	if (curl_read(request, ss)) {
+		vector<size_t> index;
+		string line, fields, units;
+
+		MeteoData tmpmeteo;
+		const bool meta_status = parseMetadata(ss, tmpmeteo.meta, fields, units); //read just one station
+
+		if (units.empty() || fields.empty() || meta_status==false) {
+			//when printing out a DBO error message, the # and ' ' have to be stripped from the begining -> substr(2)
+			if (ss.str().find("doesn't exist in DBO!") != std::string::npos)
+				throw NotFoundException(ss.str().substr(2), AT);
+			if (ss.str().find("doesn't have access to the sensor") != std::string::npos)
+				throw AccessException(ss.str().substr(2), AT);
+			if (ss.str().find("There is no user with the provided") != std::string::npos)
+				throw AccessException(ss.str().substr(2), AT);
+			if (ss.str().find("The query consumed too many server resources!") != std::string::npos)
+				throw IOException(ss.str().substr(2), AT);
+			if (dbo_debug) {
+				std::cout << "****\nRequest: " << request << "\n";
+				std::cout << "Reply: " << ss.str() << "\n****\nPlease check the station name!\n";
+			}
+			throw InvalidFormatException("Invalid header for station " + station_id, AT);
+		}
+		//map_parameters(fields, units, tmpmeteo, index);
+
+		do { //parse data section, the first line should already be buffered
+			if (line.empty() || (line[0] == '#') || !isdigit(line[0])) continue; //skip empty lines
+			//parse_streamElement(line, index, tmpmeteo);
+			vecMeteo.push_back( tmpmeteo );
+		} while (getline(ss, line));
+		if (vecMeteo.empty()) //ie the data section was empty
+			vecMeteo.push_back( tmpmeteo );
+	} else {
+		if (dbo_debug)
+			std::cout << "****\nRequest: " << request << "\n****\n";
+		throw IOException("Could not retrieve data for station " + station_id, AT);
+	}
+}
+
+void DBO::convertUnits(MeteoData& meteo) const
+{
+	//converts C to Kelvin, converts RH to [0,1]
+	double& ta = meteo(MeteoData::TA);
+	ta = IOUtils::C_TO_K(ta);
+
+	double& tsg = meteo(MeteoData::TSG);
+	tsg = IOUtils::C_TO_K(tsg);
+
+	double& tss = meteo(MeteoData::TSS);
+	tss = IOUtils::C_TO_K(tss);
+
+	double& rh = meteo(MeteoData::RH);
+	if (rh != IOUtils::nodata)
+		rh /= 100.;
+
+	double& hs = meteo(MeteoData::HS);
+	if (hs != IOUtils::nodata)
+		hs /= 100.;
+}
+
+size_t DBO::data_write(void* buf, size_t size, size_t nmemb, void* userp)
+{
+	if (userp) {
+		ostream& os = *static_cast<ostream*>(userp);
+		const streamsize len = size * nmemb;
+
+		if (os.write(static_cast<char*>(buf), len)) return len;
+	}
+
+	return 0;
+}
+
+bool DBO::curl_read(const std::string& url_query, std::ostream& os)
+{
+	CURLcode code(CURLE_FAILED_INIT);
+	CURL* curl = curl_easy_init();
+
+	const string url = endpoint + url_query;
+
+	if (curl) {
+		if (CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &data_write))
+		   && CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L))
+		   && CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L))
+		   && CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_FILE, &os))
+		   && CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_TIMEOUT, DBO::http_timeout))
+		   && CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_URL, url.c_str())))
+		{
+			code = curl_easy_perform(curl);
+		}
+		curl_easy_cleanup(curl);
+	}
+
+	if (code!=CURLE_OK) {
+		if (dbo_debug)
+			std::cout << "****\nRequest: " << url_query << "\n****\n";
+		std::cout << "[E] " << curl_easy_strerror(code) << "\t";
+	}
+
+	return (code==CURLE_OK);
+}
+
+} //namespace

meteoio/plugins/DBO.h

+/***********************************************************************************/
+/*  Copyright 2017 SLF                                                                                                                                */
+/***********************************************************************************/
+/* This file is part of MeteoIO.
+    MeteoIO is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    MeteoIO is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with MeteoIO.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#ifndef DBO_H
+#define DBO_H
+
+#include <meteoio/IOInterface.h>
+
+#include <string>
+#include <vector>
+
+#ifdef _MSC_VER
+	#pragma warning(disable:4512) //we don't need any = operator!
+#endif
+
+namespace mio {
+
+/**
+ * @class DBO
+ * @brief This class enables the access to the DBO RESTful web service
+ *
+ * @ingroup plugins
+ * @date   2017-01-26
+ */
+
+class DBO : public IOInterface {
+	public:
+		DBO(const std::string& configfile);
+		DBO(const DBO&);
+		DBO(const Config&);
+
+		virtual void readStationData(const Date& date, std::vector<StationData>& vecStation);
+		virtual void readMeteoData(const Date& dateStart, const Date& dateEnd,
+		                           std::vector< std::vector<MeteoData> >& vecMeteo);
+
+	private:
+		bool parseMetadata(std::stringstream& ss, StationData &sd, std::string &fields, std::string &units) const;
+		void readData(const Date& dateStart, const Date& dateEnd, std::vector<MeteoData>& vecMeteo, const size_t& stationindex);
+		void convertUnits(MeteoData& meteo) const;
+
+		void initDBOConnection();
+		static size_t data_write(void* buf, size_t size, size_t nmemb, void* userp);
+		bool curl_read(const std::string& url, std::ostream& os);
+
+		const Config cfg;
+		std::vector<std::string> vecStationName;
+		std::string endpoint, userid, passwd; ///< Variables for endpoint configuration
+		double default_timezone;
+		int http_timeout; //time out for http connections
+		bool dbo_debug;
+
+		static const int http_timeout_dflt;
+		static const std::string sensors_endpoint, sensors_format, null_string;
+};
+
+} //end namespace mio
+
+#endif

meteoio/plugins/GSNIO.cc

 /***********************************************************************************/
-/*  Copyright 2009 EPFL                                                            */
+/*  Copyright 2009 SLF                                                                                                                                */
 /***********************************************************************************/
 /* This file is part of MeteoIO.
     MeteoIO is free software: you can redistribute it and/or modify

meteoio/plugins/GSNIO.h

 /***********************************************************************************/
-/*  Copyright 2009 EPFL                                                            */
+/*  Copyright 2009 SLF                                                                                                                                */
 /***********************************************************************************/
 /* This file is part of MeteoIO.
     MeteoIO is free software: you can redistribute it and/or modify

meteoio/plugins/OshdIO.cc

@@ -279,7 +279,7 @@ void OshdIO::parseInputOutputSection()
 	//fill the params mapping vector
 	params_map.push_back( std::make_pair(MeteoData::ILWR, "ilwr") );
 	params_map.push_back( std::make_pair(MeteoData::P, "pair") );
-	params_map.push_back( std::make_pair(MeteoData::PSUM, "prec") );
+	params_map.push_back( std::make_pair(MeteoData::PSUM, "prec") ); //in mm/ts
 	params_map.push_back( std::make_pair(MeteoData::RH, "rhum") );
 	params_map.push_back( std::make_pair(MeteoData::TA, "tcor") ); //old:tair
 	params_map.push_back( std::make_pair(MeteoData::VW, "wcor") );