WSL/SLF GitLab Repository

Commit 0c0b0eda authored by Mathias Bavay's avatar Mathias Bavay
Browse files

A new plugin has been created to read the data from the ALPUG stations (ascii...

A new plugin has been created to read the data from the ALPUG stations (ascii format). This is still a work in progress...
parent 5d8f6009
......@@ -18,6 +18,7 @@
#include <meteoio/IOHandler.h>
#cmakedefine PLUGIN_ALPUG
#cmakedefine PLUGIN_ARCIO
#cmakedefine PLUGIN_A3DIO
#cmakedefine PLUGIN_ARPSIO
......@@ -36,6 +37,7 @@
#cmakedefine PLUGIN_PSQLIO
#cmakedefine PLUGIN_SASEIO
#include <meteoio/plugins/ALPUG.h>
#include <meteoio/plugins/ARCIO.h>
#include <meteoio/plugins/A3DIO.h>
#include <meteoio/plugins/ARPSIO.h>
......@@ -110,6 +112,7 @@ namespace mio {
* So far the following plugins have been implemented (by keyword for the io.ini key/value config file). Please read the documentation for each plugin in order to know the plugin-specific keywords:
* <center><table border="1">
* <tr><th>Plugin keyword</th><th>Provides</th><th>Description</th><th>Extra requirements</th></tr>
* <tr><td>\subpage alpug "ALPUG"</td><td>meteo</td><td>data files generated by the %ALPUG meteo stations</td><td></td></tr>
* <tr><td>\subpage a3d "A3D"</td><td>meteo, poi</td><td>original Alpine3D meteo files</td><td></td></tr>
* <tr><td>\subpage arc "ARC"</td><td>dem, landuse, grid2d</td><td>ESRI/ARC ascii grid files</td><td></td></tr>
* <tr><td>\subpage arps "ARPS"</td><td>dem, grid2d</td><td>ARPS ascii formatted grids</td><td></td></tr>
......@@ -167,6 +170,9 @@ namespace mio {
IOInterface* IOHandler::getPlugin(const std::string& plugin_name) const
{
#ifdef PLUGIN_ALPUG
if (plugin_name == "ALPUG") return new ALPUG(cfg);
#endif
#ifdef PLUGIN_ARCIO
if (plugin_name == "ARC") return new ARCIO(cfg);
#endif
......
/***********************************************************************************/
/* Copyright 2015 WSL Institute for Snow and Avalanche Research SLF-DAVOS */
/***********************************************************************************/
/* This file is part of MeteoIO.
MeteoIO is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
MeteoIO is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with MeteoIO. If not, see <http://www.gnu.org/licenses/>.
*/
#include "ALPUG.h"
#include "libsmet.h"
#include <meteoio/meteoLaws/Meteoconst.h>
#include <errno.h>
#include <string.h>
#include <sstream>
using namespace std;
namespace mio {
/**
* @page alpug Alpug
* This plugin reads the ASCII data produced by the automatic weather stations by <A href="http://www.alpug.ch/">ALPUG</A>. The metadata
* for the stations must be provided in an additional file as well as the "description" of the fields that must be provided through the configuration
* key ALPUG_FIELDS.
*
* @section alpug_format Format
* The files are named with the following schema: {YY}{ID}.met where {YY} represents the last two digits of the year and {ID} is the station ID.
*
* The files contain all the measurements but no metadata and no header. The following fields can be present:
* @code
* cod area,cod,id_AWS,date hour,Mean Wind,MaxWind,WD,AT (C),HR %, SWOR,HS (cm),empty,HTS0 (cm),empty,6697,empty,GST (C),empty,????,empty,TSS (C),ISWR,P(hpa)
* @endcode
*
* @section alpug_units Units
* Temperatures are in Celsius, relative humidity between 0 and 100%, snow heights in cm.
*
* @section alpug_keywords Keywords
* This plugin uses the following keywords:
* - COORDSYS: coordinate system (see Coords); [Input] and [Output] section
* - COORDPARAM: extra coordinates parameters (see Coords); [Input] and [Output] section
* - METEOPATH: where to find/write the meteo data; [Input] and [Output] section
* - STATION#: input filename (in METEOPATH). As many meteofiles as needed may be specified
* - ALPUG_FIELDS: comma delimited list of fields. The fields <b>MUST</b> use the \ref meteoparam "MeteoData" naming scheme. Unknown or ignored fields are replaced by "%".
*
* @code
* METEO = ALPUG
* METEOPATH = ./Met_files
* STATION1 = CAND5
* ALPUG_FIELDS = %,%,ID,timestamp,VW,VW_MAX,DW,TA,RH, RSWR,HS,%,%,%,%,%,TSG,%,%,%,TSS,ISWR,P
* @endcode
*/
const std::string ALPUG::dflt_extension = ".met";
const double ALPUG::plugin_nodata = -999.; //plugin specific nodata value. It can also be read by the plugin (depending on what is appropriate)
ALPUG::ALPUG(const std::string& configfile) : cfg(configfile)
{
parseInputOutputSection();
}
ALPUG::ALPUG(const Config& cfgreader) : cfg(cfgreader)
{
parseInputOutputSection();
}
ALPUG::~ALPUG() throw()
{}
void ALPUG::parseInputOutputSection()
{
//default timezones
in_dflt_TZ = out_dflt_TZ = IOUtils::nodata;
cfg.getValue("TIME_ZONE","Input",in_dflt_TZ,IOUtils::nothrow);
cfg.getValue("TIME_ZONE","Output",out_dflt_TZ,IOUtils::nothrow);
// Parse the [Input] and [Output] sections within Config object cfg
IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
//Parse input section: extract number of files to read and store filenames in vecFiles
std::string in_meteo;
cfg.getValue("METEO", "Input", in_meteo, IOUtils::nothrow);
if (in_meteo == "ALPUG") { //keep it synchronized with IOHandler.cc for plugin mapping!!
cfg.getValue("METEOPATH", "Input", inpath);
vecIDs.clear();
cfg.getValues("STATION", "INPUT", vecIDs);
string fields;
cfg.getValue("ALPUG_FIELDS", "Input", fields);
IOUtils::readLineToVec(fields, vecFields, ',');
if (vecFields.empty())
throw InvalidArgumentException("Please provide a comma delimited list of fields!", AT);
for (size_t ii=0; ii<vecFields.size(); ++ii) {
IOUtils::toUpper( vecFields[ii] );
IOUtils::trim( vecFields[ii] );
}
}
cfg.getValue("METEOPATH", "Output", outpath, IOUtils::nothrow);
}
void ALPUG::read2DGrid(Grid2DObject& /*grid_out*/, const std::string& /*name_in*/)
{
//Nothing so far
throw IOException("Nothing implemented here", AT);
}
void ALPUG::read2DGrid(Grid2DObject& /*grid_out*/, const MeteoGrids::Parameters& /*parameter*/, const Date& /*date*/)
{
//Nothing so far
throw IOException("Nothing implemented here", AT);
}
void ALPUG::readDEM(DEMObject& /*dem_out*/)
{
//Nothing so far
throw IOException("Nothing implemented here", AT);
}
void ALPUG::readLanduse(Grid2DObject& /*landuse_out*/)
{
//Nothing so far
throw IOException("Nothing implemented here", AT);
}
void ALPUG::readAssimilationData(const Date& /*date_in*/, Grid2DObject& /*da_out*/)
{
//Nothing so far
throw IOException("Nothing implemented here", AT);
}
void ALPUG::readStationData(const Date&, std::vector<StationData>& /*vecStation*/)
{
//Nothing so far
throw IOException("Nothing implemented here", AT);
}
Date ALPUG::parseDINDate(const std::string& datum) const
{
int year;
unsigned int month, day, hour, minute;
double second;
char rest[32] = "";
if (sscanf(datum.c_str(), "%u.%u.%d %u:%u:%lg%31s", &day, &month, &year, &hour, &minute, &second, rest) >= 6) {
return Date(year, month, day, hour, minute, in_dflt_TZ);
} else if (sscanf(datum.c_str(), "%u.%u.%d %u:%u%31s", &day, &month, &year, &hour, &minute, rest) >= 5) {
return Date(year, month, day, hour, minute, in_dflt_TZ);
}
return Date();
}
//return TRUE if we should keep reading lines
//if isValid==false, don't store the MeteoData object
bool ALPUG::parseLine(const std::string& filename, const char& eoln, const size_t& nr_of_data_fields, const Date& dateStart, const Date& dateEnd, const std::string& line, MeteoData &md, bool &isValid) const
{
md.reset();
isValid = false;
vector<string> tmp_vec;
if (IOUtils::readLineToVec(line, tmp_vec, ',') == nr_of_data_fields){
for (size_t ii=0; ii<nr_of_data_fields; ++ii) {
const string field = vecFields[ii];
if (field=="%" || field=="ID") continue;
if (field=="TIMESTAMP") {
Date date = parseDINDate(tmp_vec[ii]);
if (date.isUndef())
throw InvalidFormatException("Invalid date \'"+tmp_vec[ii]+"\' in file \'"+filename+"\'", AT);
if (date<dateStart) return true;
if (date>dateEnd) {
std::cout << date.toString(Date::ISO) << " > " << dateEnd.toString(Date::ISO) << "\n";
return false;
}
md.setDate(date);
continue;
}
double val;
IOUtils::convertString(val, tmp_vec[ii]);
if (field=="TA" || field=="TSG" || field=="TSS")
val += Cst::t_water_freezing_pt;
else if (field=="RH" || field=="HS")
val *= 0.01;
else if (field=="P")
val *= 100.;
md(field) = val;
}
isValid = true;
} else {
std::ostringstream ss;
ss << "File \'" << filename << "\' declares " << nr_of_data_fields << " columns ";
ss << "but this does not match the following line:\n" << line << "\n";
throw InvalidFormatException(ss.str(), AT);
}
return true;
}
void ALPUG::readMetoFile(const std::string& station_id, const Date& dateStart, const Date& dateEnd,
std::vector<MeteoData>& vecM)
{
vecM.clear();
//TODO: read accross multiple years
// start in the previous file, keep going in the next file (since files are restarted on Oct. 01)
stringstream ss;
ss << dateStart.getYear();
const string extension = IOUtils::getExtension(station_id);
const string filename = (extension!="")? inpath + "/" + ss.str().substr(2,2) + station_id : inpath + "/" + ss.str().substr(2,2) + station_id + dflt_extension;
std::ifstream fin; //Input file streams
fin.clear();
fin.open (filename.c_str(), ios::in|ios::binary); //ascii does end of line translation, which messes up the pointer code
if (fin.fail()) {
ostringstream ss;
ss << "Error opening file \"" << filename << "\" for reading, possible reason: " << strerror(errno);
ss << " Please check file existence and permissions!";
throw FileAccessException(ss.str(), AT);
}
const char eoln = smet::SMETCommon::getEoln(fin); //get the end of line character for the file
const size_t nr_of_data_fields = vecFields.size();
string line;
size_t linenr = 0;
while (!fin.eof()){
string line;
getline(fin, line, eoln);
linenr++;
if (line.empty())
continue; //Pure comment lines and empty lines are ignored
Coords pos;
StationData sd(pos, station_id, station_id);
MeteoData md;
md.meta = sd;
bool isValid;
if (!parseLine(filename, eoln, nr_of_data_fields, dateStart, dateEnd, line, md, isValid))
break;
if(isValid)
vecM.push_back( md );
}
fin.close();
}
void ALPUG::readMeteoData(const Date& dateStart, const Date& dateEnd,
std::vector< std::vector<MeteoData> >& vecMeteo,
const size_t&)
{
vecMeteo.clear();
for (size_t ii=0; ii<vecIDs.size(); ++ii) {
std::vector<MeteoData> vecM;
readMetoFile(vecIDs[ii], dateStart, dateEnd, vecM);
vecMeteo.push_back( vecM );
}
}
void ALPUG::writeMeteoData(const std::vector< std::vector<MeteoData> >& /*vecMeteo*/,
const std::string&)
{
//Nothing so far
throw IOException("Nothing implemented here", AT);
}
void ALPUG::readPOI(std::vector<Coords>&)
{
//Nothing so far
throw IOException("Nothing implemented here", AT);
}
void ALPUG::write2DGrid(const Grid2DObject& /*grid_in*/, const std::string& /*name*/)
{
//Nothing so far
throw IOException("Nothing implemented here", AT);
}
void ALPUG::write2DGrid(const Grid2DObject& /*grid_in*/, const MeteoGrids::Parameters& /*parameter*/, const Date& /*date*/)
{
//Nothing so far
throw IOException("Nothing implemented here", AT);
}
void ALPUG::cleanup() throw()
{
}
} //namespace
/***********************************************************************************/
/* Copyright 2015 WSL Institute for Snow and Avalanche Research SLF-DAVOS */
/***********************************************************************************/
/* This file is part of MeteoIO.
MeteoIO is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
MeteoIO is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with MeteoIO. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __ALPUG_H__
#define __ALPUG_H__
#include <meteoio/IOInterface.h>
#include <meteoio/Config.h>
#include <string>
namespace mio {
/**
* @class ALPUG
* @brief This plugin reads data as they come out of ALPUG stations, in ASCII
*
* @ingroup plugins
* @author Mathias Bavay
* @date 2015-03-31
*/
class ALPUG : public IOInterface {
public:
ALPUG(const std::string& configfile);
ALPUG(const ALPUG&);
ALPUG(const Config& cfgreader);
~ALPUG() throw();
virtual void read2DGrid(Grid2DObject& grid_out, const std::string& parameter="");
virtual void read2DGrid(Grid2DObject& grid_out, const MeteoGrids::Parameters& parameter, const Date& date);
virtual void readDEM(DEMObject& dem_out);
virtual void readLanduse(Grid2DObject& landuse_out);
virtual void readStationData(const Date& date, std::vector<StationData>& vecStation);
virtual void readMeteoData(const Date& dateStart, const Date& dateEnd,
std::vector< std::vector<MeteoData> >& vecMeteo,
const size_t& stationindex=IOUtils::npos);
virtual void writeMeteoData(const std::vector< std::vector<MeteoData> >& vecMeteo,
const std::string& name="");
virtual void readAssimilationData(const Date&, Grid2DObject& da_out);
virtual void readPOI(std::vector<Coords>& pts);
virtual void write2DGrid(const Grid2DObject& grid_in, const std::string& filename);
virtual void write2DGrid(const Grid2DObject& grid_in, const MeteoGrids::Parameters& parameter, const Date& date);
private:
void parseInputOutputSection();
void cleanup() throw();
Date parseDINDate(const std::string& datum) const;
bool parseLine(const std::string& filename, const char& eoln, const size_t& nr_of_data_fields, const Date& dateStart, const Date& dateEnd, const std::string& line, MeteoData &md, bool &isValid) const;
void readMetoFile(const std::string& station_name, const Date& dateStart, const Date& dateEnd,
std::vector<MeteoData>& vecM);
const Config cfg;
static const double plugin_nodata; //plugin specific nodata value, e.g. -999
static const std::string dflt_extension;
std::string coordin, coordinparam, coordout, coordoutparam; //projection parameters
std::vector<std::string> vecIDs, vecFields; //read from the Config [Input] section
std::string inpath, outpath; //read from the Config [Output] section
double in_dflt_TZ, out_dflt_TZ;
};
} //namespace
#endif
......@@ -3,6 +3,10 @@ LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/tools/cmake/")
CONFIGURE_FILE( ${PROJECT_SOURCE_DIR}/meteoio/IOHandler.cmake.cc ${PROJECT_SOURCE_DIR}/meteoio/IOHandler.cc )
IF(PLUGIN_ALPUG)
SET(plugins_sources ${plugins_sources} plugins/ALPUG.cc)
ENDIF(PLUGIN_ALPUG)
IF(PLUGIN_ARCIO)
SET(plugins_sources ${plugins_sources} plugins/ARCIO.cc)
ENDIF(PLUGIN_ARCIO)
......
......@@ -510,49 +510,49 @@ void SMETWriter::write_header()
write_signature();
fout << "[HEADER]" << "\n";
fout << "station_id = " << header["station_id"] << "\n";
fout << "station_id = " << header["station_id"] << "\n";
map<string,string>::const_iterator it = header.find("station_name");
if (it != header.end())
fout << "station_name = " << it->second << "\n";
fout << "station_name = " << it->second << "\n";
if (location_in_header){
if (location_wgs84 == 7){
fout << "latitude = " << header["latitude"] << "\n";
fout << "longitude = " << header["longitude"] << "\n";
fout << "altitude = " << header["altitude"] << "\n";
fout << "latitude = " << header["latitude"] << "\n";
fout << "longitude = " << header["longitude"] << "\n";
fout << "altitude = " << header["altitude"] << "\n";
}
if (location_epsg == 15){
fout << "easting = " << header["easting"] << "\n";
fout << "northing = " << header["northing"] << "\n";
fout << "easting = " << header["easting"] << "\n";
fout << "northing = " << header["northing"] << "\n";
if (location_wgs84 != 7)
fout << "altitude = " << header["altitude"] << "\n";
fout << "epsg = " << header["epsg"] << "\n";
fout << "altitude = " << header["altitude"] << "\n";
fout << "epsg = " << header["epsg"] << "\n";
}
} else {
if (location_in_data_epsg)
fout << "epsg = " << header["epsg"] << "\n";
fout << "epsg = " << header["epsg"] << "\n";
}
fout << "nodata = " << header["nodata"] << "\n";
fout << "nodata = " << header["nodata"] << "\n";
//Optional header keys:
it = header.find("tz");
if (it != header.end())
fout << "tz = " << it->second << "\n";
fout << "tz = " << it->second << "\n";
it = header.find("creation");
if (it != header.end())
fout << "creation = " << it->second << "\n";
fout << "creation = " << it->second << "\n";
it = header.find("source");
if (it != header.end())
fout << "source = " << it->second << "\n";
fout << "source = " << it->second << "\n";
it = header.find("units_offset");
if (it != header.end())
fout << "units_offset = " << it->second << "\n";
fout << "units_offset = " << it->second << "\n";
it = header.find("units_multiplier");
if (it != header.end())
......@@ -566,7 +566,7 @@ void SMETWriter::write_header()
fout << other_header_keys[ii] << " = " << header[other_header_keys[ii]] << "\n";
}
fout << "fields = " << header["fields"] << "\n";
fout << "fields = " << header["fields"] << "\n";
fout << "[DATA]" << endl;
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment