A new plugin has been created to read the data from the ALPUG stations (ascii...

A new plugin has been created to read the data from the ALPUG stations (ascii format). This is still a work in progress...

meteoio/IOHandler.cmake.cc

@@ -18,6 +18,7 @@
 
 #include <meteoio/IOHandler.h>
 
+#cmakedefine PLUGIN_ALPUG
 #cmakedefine PLUGIN_ARCIO
 #cmakedefine PLUGIN_A3DIO
 #cmakedefine PLUGIN_ARPSIO
@@ -36,6 +37,7 @@
 #cmakedefine PLUGIN_PSQLIO
 #cmakedefine PLUGIN_SASEIO
 
+#include <meteoio/plugins/ALPUG.h>
 #include <meteoio/plugins/ARCIO.h>
 #include <meteoio/plugins/A3DIO.h>
 #include <meteoio/plugins/ARPSIO.h>
@@ -110,6 +112,7 @@ namespace mio {
  * So far the following plugins have been implemented (by keyword for the io.ini key/value config file). Please read the documentation for each plugin in order to know the plugin-specific keywords:
  * <center><table border="1">
  * <tr><th>Plugin keyword</th><th>Provides</th><th>Description</th><th>Extra requirements</th></tr>
+ * <tr><td>\subpage alpug "ALPUG"</td><td>meteo</td><td>data files generated by the %ALPUG meteo stations</td><td></td></tr>
  * <tr><td>\subpage a3d "A3D"</td><td>meteo, poi</td><td>original Alpine3D meteo files</td><td></td></tr>
  * <tr><td>\subpage arc "ARC"</td><td>dem, landuse, grid2d</td><td>ESRI/ARC ascii grid files</td><td></td></tr>
  * <tr><td>\subpage arps "ARPS"</td><td>dem, grid2d</td><td>ARPS ascii formatted grids</td><td></td></tr>
@@ -167,6 +170,9 @@ namespace mio {
 
 IOInterface* IOHandler::getPlugin(const std::string& plugin_name) const
 {
+#ifdef PLUGIN_ALPUG
+	if (plugin_name == "ALPUG") return new ALPUG(cfg);
+#endif
 #ifdef PLUGIN_ARCIO
 	if (plugin_name == "ARC") return new ARCIO(cfg);
 #endif

meteoio/plugins/ALPUG.cc

+/***********************************************************************************/
+/*  Copyright 2015 WSL Institute for Snow and Avalanche Research    SLF-DAVOS      */
+/***********************************************************************************/
+/* This file is part of MeteoIO.
+    MeteoIO is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    MeteoIO is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with MeteoIO.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#include "ALPUG.h"
+#include "libsmet.h"
+#include <meteoio/meteoLaws/Meteoconst.h>
+
+#include <errno.h>
+#include <string.h>
+#include <sstream>
+
+using namespace std;
+
+namespace mio {
+/**
+ * @page alpug Alpug
+ * This plugin reads the ASCII data produced by the automatic weather stations by <A href="http://www.alpug.ch/">ALPUG</A>. The metadata
+ * for the stations must be provided in an additional file as well as the "description" of the fields that must be provided through the configuration
+ * key ALPUG_FIELDS.
+ * 
+ * @section alpug_format Format
+ * The files are named with the following schema: {YY}{ID}.met where {YY} represents the last two digits of the year and {ID} is the station ID.
+ * 
+ * The files contain all the measurements but no metadata and no header. The following fields can be present:
+ * @code
+ * cod area,cod,id_AWS,date hour,Mean Wind,MaxWind,WD,AT (C),HR %, SWOR,HS (cm),empty,HTS0 (cm),empty,6697,empty,GST (C),empty,????,empty,TSS (C),ISWR,P(hpa)
+ * @endcode
+ *
+ * @section alpug_units Units
+ * Temperatures are in Celsius, relative humidity between 0 and 100%, snow heights in cm.
+ *
+ * @section alpug_keywords Keywords
+ * This plugin uses the following keywords:
+ * - COORDSYS: coordinate system (see Coords); [Input] and [Output] section
+ * - COORDPARAM: extra coordinates parameters (see Coords); [Input] and [Output] section
+ * - METEOPATH: where to find/write the meteo data; [Input] and [Output] section
+ * - STATION#: input filename (in METEOPATH). As many meteofiles as needed may be specified
+ * - ALPUG_FIELDS: comma delimited list of fields. The fields <b>MUST</b> use the \ref meteoparam "MeteoData" naming scheme. Unknown or ignored fields are replaced by "%".
+ * 
+ * @code
+ * METEO        = ALPUG
+ * METEOPATH    = ./Met_files
+ * STATION1     = CAND5
+ * ALPUG_FIELDS = %,%,ID,timestamp,VW,VW_MAX,DW,TA,RH, RSWR,HS,%,%,%,%,%,TSG,%,%,%,TSS,ISWR,P
+ * @endcode
+ */
+
+const std::string ALPUG::dflt_extension = ".met";
+const double ALPUG::plugin_nodata = -999.; //plugin specific nodata value. It can also be read by the plugin (depending on what is appropriate)
+
+ALPUG::ALPUG(const std::string& configfile) : cfg(configfile)
+{
+	parseInputOutputSection();
+}
+
+ALPUG::ALPUG(const Config& cfgreader) : cfg(cfgreader)
+{
+	parseInputOutputSection();
+}
+
+ALPUG::~ALPUG() throw()
+{}
+
+void ALPUG::parseInputOutputSection()
+{
+	//default timezones
+	in_dflt_TZ = out_dflt_TZ = IOUtils::nodata;
+	cfg.getValue("TIME_ZONE","Input",in_dflt_TZ,IOUtils::nothrow);
+	cfg.getValue("TIME_ZONE","Output",out_dflt_TZ,IOUtils::nothrow);
+
+	// Parse the [Input] and [Output] sections within Config object cfg
+	IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
+
+	//Parse input section: extract number of files to read and store filenames in vecFiles
+	std::string in_meteo;
+	cfg.getValue("METEO", "Input", in_meteo, IOUtils::nothrow);
+	if (in_meteo == "ALPUG") { //keep it synchronized with IOHandler.cc for plugin mapping!!
+		cfg.getValue("METEOPATH", "Input", inpath);
+		vecIDs.clear();
+		cfg.getValues("STATION", "INPUT", vecIDs);
+		
+		string fields;
+		cfg.getValue("ALPUG_FIELDS", "Input", fields);
+		IOUtils::readLineToVec(fields, vecFields, ',');
+		if (vecFields.empty())
+			throw InvalidArgumentException("Please provide a comma delimited list of fields!", AT);
+		for (size_t ii=0; ii<vecFields.size(); ++ii) {
+			IOUtils::toUpper( vecFields[ii] );
+			IOUtils::trim( vecFields[ii] );
+		}
+	}
+
+	cfg.getValue("METEOPATH", "Output", outpath, IOUtils::nothrow);
+}
+
+void ALPUG::read2DGrid(Grid2DObject& /*grid_out*/, const std::string& /*name_in*/)
+{
+	//Nothing so far
+	throw IOException("Nothing implemented here", AT);
+}
+
+void ALPUG::read2DGrid(Grid2DObject& /*grid_out*/, const MeteoGrids::Parameters& /*parameter*/, const Date& /*date*/)
+{
+	//Nothing so far
+	throw IOException("Nothing implemented here", AT);
+}
+
+void ALPUG::readDEM(DEMObject& /*dem_out*/)
+{
+	//Nothing so far
+	throw IOException("Nothing implemented here", AT);
+}
+
+void ALPUG::readLanduse(Grid2DObject& /*landuse_out*/)
+{
+	//Nothing so far
+	throw IOException("Nothing implemented here", AT);
+}
+
+void ALPUG::readAssimilationData(const Date& /*date_in*/, Grid2DObject& /*da_out*/)
+{
+	//Nothing so far
+	throw IOException("Nothing implemented here", AT);
+}
+
+void ALPUG::readStationData(const Date&, std::vector<StationData>& /*vecStation*/)
+{
+	//Nothing so far
+	throw IOException("Nothing implemented here", AT);
+}
+
+Date ALPUG::parseDINDate(const std::string& datum) const
+{
+	int year;
+	unsigned int month, day, hour, minute;
+	double second;
+	char rest[32] = "";
+	
+	if (sscanf(datum.c_str(), "%u.%u.%d %u:%u:%lg%31s", &day, &month, &year, &hour, &minute, &second, rest) >= 6) {
+		return Date(year, month, day, hour, minute, in_dflt_TZ);
+	} else if (sscanf(datum.c_str(), "%u.%u.%d %u:%u%31s", &day, &month, &year, &hour, &minute, rest) >= 5) {
+		return Date(year, month, day, hour, minute, in_dflt_TZ);
+	}
+	
+	return Date();
+}
+
+//return TRUE if we should keep reading lines
+//if isValid==false, don't store the MeteoData object
+bool ALPUG::parseLine(const std::string& filename, const char& eoln, const size_t& nr_of_data_fields, const Date& dateStart, const Date& dateEnd, const std::string& line, MeteoData &md, bool &isValid) const
+{
+	md.reset();
+	
+	isValid = false;
+	vector<string> tmp_vec;
+	if (IOUtils::readLineToVec(line, tmp_vec, ',') == nr_of_data_fields){
+		for (size_t ii=0; ii<nr_of_data_fields; ++ii) {
+			const string field = vecFields[ii];
+			if (field=="%" || field=="ID") continue;
+			if (field=="TIMESTAMP") {
+				Date date = parseDINDate(tmp_vec[ii]);
+				if (date.isUndef())
+					throw InvalidFormatException("Invalid date \'"+tmp_vec[ii]+"\' in file \'"+filename+"\'", AT);
+				
+				if (date<dateStart) return true;
+				if (date>dateEnd) {
+					std::cout << date.toString(Date::ISO) << " > " << dateEnd.toString(Date::ISO) << "\n";
+					return false;
+				}
+				md.setDate(date);
+				continue;
+			}
+			
+			double val;
+			IOUtils::convertString(val, tmp_vec[ii]);
+			
+			if (field=="TA" || field=="TSG" || field=="TSS")
+				val += Cst::t_water_freezing_pt;
+			else if (field=="RH" || field=="HS")
+				val *= 0.01;
+			else if (field=="P")
+				val *= 100.;
+			
+			md(field) = val;
+		}
+		isValid = true;
+	} else {
+		std::ostringstream ss;
+		ss << "File \'" << filename << "\' declares " << nr_of_data_fields << " columns ";
+		ss << "but this does not match the following line:\n" << line << "\n";
+		throw InvalidFormatException(ss.str(), AT);
+	}
+	return true;
+}
+
+void ALPUG::readMetoFile(const std::string& station_id, const Date& dateStart, const Date& dateEnd, 
+                                              std::vector<MeteoData>& vecM)
+{
+	vecM.clear();
+	
+	//TODO: read accross multiple years
+	// start in the previous file, keep going in the next file (since files are restarted on Oct. 01)
+	
+	stringstream ss;
+	ss << dateStart.getYear();
+	const string extension = IOUtils::getExtension(station_id);
+	const string filename = (extension!="")?  inpath + "/" + ss.str().substr(2,2) + station_id : inpath + "/" + ss.str().substr(2,2) + station_id + dflt_extension;
+	
+	std::ifstream fin; //Input file streams
+	fin.clear();
+	fin.open (filename.c_str(), ios::in|ios::binary); //ascii does end of line translation, which messes up the pointer code
+	if (fin.fail()) {
+		ostringstream ss;
+		ss << "Error opening file \"" << filename << "\" for reading, possible reason: " << strerror(errno);
+		ss << " Please check file existence and permissions!";
+		throw FileAccessException(ss.str(), AT);
+	}
+	
+	const char eoln = smet::SMETCommon::getEoln(fin); //get the end of line character for the file
+	
+	const size_t nr_of_data_fields = vecFields.size();
+	string line;
+	size_t linenr = 0;
+	while (!fin.eof()){
+		string line;
+		getline(fin, line, eoln);
+		linenr++;
+
+		if (line.empty())
+			continue; //Pure comment lines and empty lines are ignored
+		
+		Coords pos;
+		StationData sd(pos, station_id, station_id);
+		MeteoData md;
+		md.meta = sd;
+		bool isValid;
+		if (!parseLine(filename, eoln, nr_of_data_fields, dateStart, dateEnd, line, md, isValid))
+			break;
+		
+		if(isValid)
+			vecM.push_back( md );
+	}
+	
+	fin.close();
+}
+
+void ALPUG::readMeteoData(const Date& dateStart, const Date& dateEnd,
+                             std::vector< std::vector<MeteoData> >& vecMeteo,
+                             const size_t&)
+{
+	vecMeteo.clear();
+	for (size_t ii=0; ii<vecIDs.size(); ++ii) {
+		std::vector<MeteoData> vecM;
+		readMetoFile(vecIDs[ii], dateStart, dateEnd, vecM);
+		vecMeteo.push_back( vecM );
+	}
+}
+
+void ALPUG::writeMeteoData(const std::vector< std::vector<MeteoData> >& /*vecMeteo*/,
+                              const std::string&)
+{
+	//Nothing so far
+	throw IOException("Nothing implemented here", AT);
+}
+
+void ALPUG::readPOI(std::vector<Coords>&)
+{
+	//Nothing so far
+	throw IOException("Nothing implemented here", AT);
+}
+
+void ALPUG::write2DGrid(const Grid2DObject& /*grid_in*/, const std::string& /*name*/)
+{
+	//Nothing so far
+	throw IOException("Nothing implemented here", AT);
+}
+
+void ALPUG::write2DGrid(const Grid2DObject& /*grid_in*/, const MeteoGrids::Parameters& /*parameter*/, const Date& /*date*/)
+{
+	//Nothing so far
+	throw IOException("Nothing implemented here", AT);
+}
+
+void ALPUG::cleanup() throw()
+{
+
+}
+
+} //namespace

meteoio/plugins/ALPUG.h

+/***********************************************************************************/
+/*  Copyright 2015 WSL Institute for Snow and Avalanche Research    SLF-DAVOS      */
+/***********************************************************************************/
+/* This file is part of MeteoIO.
+    MeteoIO is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    MeteoIO is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with MeteoIO.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#ifndef __ALPUG_H__
+#define __ALPUG_H__
+
+#include <meteoio/IOInterface.h>
+#include <meteoio/Config.h>
+
+#include <string>
+
+namespace mio {
+
+/**
+ * @class ALPUG
+ * @brief This plugin reads data as they come out of ALPUG stations, in ASCII
+ *
+ * @ingroup plugins
+ * @author Mathias Bavay
+ * @date   2015-03-31
+ */
+class ALPUG : public IOInterface {
+	public:
+		ALPUG(const std::string& configfile);
+		ALPUG(const ALPUG&);
+		ALPUG(const Config& cfgreader);
+		~ALPUG() throw();
+
+		virtual void read2DGrid(Grid2DObject& grid_out, const std::string& parameter="");
+		virtual void read2DGrid(Grid2DObject& grid_out, const MeteoGrids::Parameters& parameter, const Date& date);
+		virtual void readDEM(DEMObject& dem_out);
+		virtual void readLanduse(Grid2DObject& landuse_out);
+
+		virtual void readStationData(const Date& date, std::vector<StationData>& vecStation);
+		virtual void readMeteoData(const Date& dateStart, const Date& dateEnd,
+		                           std::vector< std::vector<MeteoData> >& vecMeteo,
+		                           const size_t& stationindex=IOUtils::npos);
+
+		virtual void writeMeteoData(const std::vector< std::vector<MeteoData> >& vecMeteo,
+		                            const std::string& name="");
+
+		virtual void readAssimilationData(const Date&, Grid2DObject& da_out);
+		virtual void readPOI(std::vector<Coords>& pts);
+		virtual void write2DGrid(const Grid2DObject& grid_in, const std::string& filename);
+		virtual void write2DGrid(const Grid2DObject& grid_in, const MeteoGrids::Parameters& parameter, const Date& date);
+
+	private:
+		void parseInputOutputSection();
+		void cleanup() throw();
+		Date parseDINDate(const std::string& datum) const;
+		bool parseLine(const std::string& filename, const char& eoln, const size_t& nr_of_data_fields, const Date& dateStart, const Date& dateEnd, const std::string& line, MeteoData &md, bool &isValid) const;
+		void readMetoFile(const std::string& station_name, const Date& dateStart, const Date& dateEnd, 
+                                              std::vector<MeteoData>& vecM);
+
+		const Config cfg;
+		static const double plugin_nodata; //plugin specific nodata value, e.g. -999
+		static const std::string dflt_extension;
+		std::string coordin, coordinparam, coordout, coordoutparam; //projection parameters
+		std::vector<std::string> vecIDs, vecFields;  //read from the Config [Input] section
+		std::string inpath, outpath;                //read from the Config [Output] section
+		
+		double in_dflt_TZ, out_dflt_TZ;
+};
+
+} //namespace
+#endif

meteoio/plugins/CMakeLists.txt

@@ -3,6 +3,10 @@ LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/tools/cmake/")
 
 CONFIGURE_FILE( ${PROJECT_SOURCE_DIR}/meteoio/IOHandler.cmake.cc ${PROJECT_SOURCE_DIR}/meteoio/IOHandler.cc )
 
+IF(PLUGIN_ALPUG)
+	SET(plugins_sources ${plugins_sources} plugins/ALPUG.cc)
+ENDIF(PLUGIN_ALPUG)
+
 IF(PLUGIN_ARCIO)
 	SET(plugins_sources ${plugins_sources} plugins/ARCIO.cc)
 ENDIF(PLUGIN_ARCIO)

meteoio/plugins/libsmet.cc

@@ -510,49 +510,49 @@ void SMETWriter::write_header()
 	write_signature();
 
 	fout << "[HEADER]" << "\n";
-	fout << "station_id          = " << header["station_id"] << "\n";
+	fout << "station_id       = " << header["station_id"] << "\n";
 
 	map<string,string>::const_iterator it = header.find("station_name");
 	if (it != header.end())
-		fout << "station_name    = " << it->second << "\n";
+		fout << "station_name     = " << it->second << "\n";
 
 	if (location_in_header){
 		if (location_wgs84 == 7){
-			fout << "latitude              = " << header["latitude"]  << "\n";
-			fout << "longitude           = " << header["longitude"] << "\n";
-			fout << "altitude              = " << header["altitude"]  << "\n";
+			fout << "latitude         = " << header["latitude"]  << "\n";
+			fout << "longitude        = " << header["longitude"] << "\n";
+			fout << "altitude         = " << header["altitude"]  << "\n";
 		}
 
 		if (location_epsg == 15){
-			fout << "easting               = " << header["easting"]   << "\n";
-			fout << "northing             = " << header["northing"]  << "\n";
+			fout << "easting          = " << header["easting"]   << "\n";
+			fout << "northing         = " << header["northing"]  << "\n";
 			if (location_wgs84 != 7)
-				fout << "altitude           = " << header["altitude"]  << "\n";
-			fout << "epsg                   = " << header["epsg"]  << "\n";
+				fout << "altitude        = " << header["altitude"]  << "\n";
+			fout << "epsg             = " << header["epsg"]  << "\n";
 		}
 	} else {
 		if (location_in_data_epsg)
-			fout << "epsg                   = " << header["epsg"]      << "\n";
+			fout << "epsg             = " << header["epsg"]      << "\n";
 	}
 
-	fout << "nodata               = " << header["nodata"] << "\n";
+	fout << "nodata           = " << header["nodata"] << "\n";
 
 	//Optional header keys:
 	it = header.find("tz");
 	if (it != header.end())
-		fout << "tz                        = " << it->second << "\n";
+		fout << "tz               = " << it->second << "\n";
 
 	it = header.find("creation");
 	if (it != header.end())
-		fout << "creation             = " << it->second << "\n";
+		fout << "creation         = " << it->second << "\n";
 
 	it = header.find("source");
 	if (it != header.end())
-		fout << "source               = " << it->second << "\n";
+		fout << "source           = " << it->second << "\n";
 
 	it = header.find("units_offset");
 	if (it != header.end())
-		fout << "units_offset       = " << it->second << "\n";
+		fout << "units_offset     = " << it->second << "\n";
 
 	it = header.find("units_multiplier");
 	if (it != header.end())
@@ -566,7 +566,7 @@ void SMETWriter::write_header()
 		fout << other_header_keys[ii] << " = " << header[other_header_keys[ii]] << "\n";
 	}
 
-	fout << "fields                  = " << header["fields"] << "\n";
+	fout << "fields           = " << header["fields"] << "\n";
 	fout << "[DATA]" << endl;
 }