WSL/SLF GitLab Repository

Commit 963bb835 authored by Mathias Bavay's avatar Mathias Bavay
Browse files

The data_converter code has been cleaned up and made more similar to the...

The data_converter code has been cleaned up and made more similar to the snowpack operational "data_qa" tool. An invalid check (< when <= was needed) has been fixed in Meteo1DInterpolator (this was leading to segfaults when a buffer only contained 1 data point).

Code cleanup, documentation improvement (link to WFJ dataset), convenience method in StationData (getAltitude).
parent 75aefa3f
/*This example takes two ISO-formatted dates and a sampling rate (in h) on the command line
* for example ./data_converter 2008-12-01T00:00:00 2008-12-31T23:00 1
* It will retrieve the data for this time interval and write it out once per 1 hour as specified
* in the io.ini configuration
*/
#include <iostream>
#include <string.h>
#include <map>
#include <vector>
#include <meteoio/MeteoIO.h>
using namespace mio; //The MeteoIO namespace is called mio
//This example takes two ISO-formatted dates and a sampling rate (in h) on the command line
//for example ./data_converter 2008-12-01T00:00:00 2008-12-31T23:00 1
//It will retrieve the data for this time interval and write it out once per 1 hour as specified
//in the io.ini configuration
void real_main(int argc, char** argv) {
if(argc!=4) {
std::cout << "Invalid number of arguments! Please provide a date range and a sampling rate (in hours)\n";
exit(0);
}
void real_main(int argc, char** argv)
{
Config cfg("io.ini");
const double TZ = cfg.get("TIME_ZONE", "Input");
Date d1, d2;
double Tstep;
IOUtils::convertString(d1, argv[1], TZ);
IOUtils::convertString(d2, argv[2], TZ);
IOUtils::convertString(Tstep, argv[3]);
Tstep /= 24.; //convert to sampling rate in days
Date d_start, d_end;
IOUtils::convertString(d_start, argv[1], TZ);
IOUtils::convertString(d_end, argv[2], TZ);
IOManager io(cfg);
std::cout << "Reading input data" << std::endl;
std::cout << "Powered by " << getLibVersion() << "\n";
std::cout << "Reading data from " << d_start.toString(Date::ISO) << " to " << d_end.toString(Date::ISO) << "\n";
Timer timer;
timer.start();
std::map<std::string, size_t> mapIDs; //over a large time range, the number of stations might change... this is the way to make it work
std::vector<MeteoData> Meteo; //we need some intermediate storage, for storing data sets for 1 timestep
std::vector< std::vector<MeteoData> > vecMeteo; //so we can keep and output the data that has been read
for(; d1<=d2; d1+=Tstep) { //time loop
io.getMeteoData(d1, Meteo); //read 1 timestep at once, forcing resampling to the timestep
for(Date d=d_start; d<=d_end; d+=Tstep) { //time loop
io.getMeteoData(d, Meteo); //read 1 timestep at once, forcing resampling to the timestep
size_t insert_position = 0;
for(size_t ii=0; ii<Meteo.size(); ii++) {
const std::string stationID( Meteo[ii].meta.stationID );
......@@ -46,17 +48,23 @@ void real_main(int argc, char** argv) {
}
}
//io.getMeteoData(d1, d2, vecMeteo); //This would be the call that does NOT resample the data, instead of the above "for" loop
//io.getMeteoData(d_start, d_end, vecMeteo); //This would be the call that does NOT resample the data, instead of the above "for" loop
timer.stop();
//In both case, we write the data out
std::cout << "Writing output data" << std::endl;
io.writeMeteoData(vecMeteo);
timer.stop();
std::cout << "Done!! in " << timer.getElapsed() << " s" << std::endl;
}
int main(int argc, char** argv) {
int main(int argc, char** argv)
{
if(argc!=4) {
std::cout << "Invalid number of arguments! Please provide a date range and a sampling rate (in hours)\n";
exit(0);
}
try {
real_main(argc, argv);
} catch(const std::exception &e) {
......
......@@ -72,10 +72,10 @@ void Config::deleteKeys(const std::string& keymatch, const std::string& section,
//Loop through keys, look for match - delete matches
if (anywhere) {
const string key_pattern = IOUtils::strToUpper(keymatch);
const string section_pattern = IOUtils::strToUpper(section);
const string key_pattern( IOUtils::strToUpper(keymatch) );
const string section_pattern( IOUtils::strToUpper(section) );
map<string,string>::iterator it=properties.begin();
std::map<string,string>::iterator it=properties.begin();
while (it != properties.end()) {
const size_t found_section = (it->first).find(section_pattern, 0);
......@@ -86,9 +86,9 @@ void Config::deleteKeys(const std::string& keymatch, const std::string& section,
}
} else {
const string key_pattern = IOUtils::strToUpper(section) + "::" + IOUtils::strToUpper(keymatch);
const string key_pattern( IOUtils::strToUpper(section) + "::" + IOUtils::strToUpper(keymatch) );
map<string,string>::iterator it=properties.begin();
std::map<string,string>::iterator it=properties.begin();
while (it != properties.end()) {
if ( (it->first).find(key_pattern, 0)==0 ) //match found at start
properties.erase( it++ ); // advance before iterator become invalid
......@@ -100,8 +100,8 @@ void Config::deleteKeys(const std::string& keymatch, const std::string& section,
bool Config::keyExists(const std::string& key, const std::string& section) const
{
const string full_key = IOUtils::strToUpper(section) + "::" + IOUtils::strToUpper(key);
const map<string,string>::const_iterator it = properties.find(full_key);
const string full_key( IOUtils::strToUpper(section) + "::" + IOUtils::strToUpper(key) );
const std::map<string,string>::const_iterator it = properties.find(full_key);
return (it!=properties.end());
}
......@@ -246,7 +246,7 @@ void Config::parseLine(const unsigned int& linenr, std::vector<std::string> &imp
string key, value;
if (IOUtils::readKeyValuePair(line, "=", key, value, true)) {
if (key=="IMPORT_BEFORE") {
const std::string file_and_path = clean_import_path(value);
const std::string file_and_path( clean_import_path(value) );
if (!accept_import_before)
throw IOException("Error in \""+sourcename+"\": IMPORT_BEFORE key MUST occur before any other key!", AT);
if (std::find(imported.begin(), imported.end(), file_and_path)!=imported.end())
......@@ -255,7 +255,7 @@ void Config::parseLine(const unsigned int& linenr, std::vector<std::string> &imp
return;
}
if (key=="IMPORT_AFTER") {
const std::string file_and_path = clean_import_path(value);
const std::string file_and_path( clean_import_path(value) );
if (std::find(imported.begin(), imported.end(), file_and_path)!=imported.end())
throw IOException("Can not import \"" + value + "\" again: it has already been imported!", AT);
import_after.push_back(file_and_path);
......@@ -268,12 +268,12 @@ void Config::parseLine(const unsigned int& linenr, std::vector<std::string> &imp
ostringstream tmp;
tmp << linenr;
const string key_msg = (key.empty())? "" : "key "+key+" ";
const string key_value_link = (key.empty() && !value.empty())? "value " : "";
const string value_msg = (value.empty())? "" : value+" " ;
const string keyvalue_msg = (key.empty() && value.empty())? "key/value " : key_msg+key_value_link+value_msg;
const string section_msg = (section.empty())? "" : "in section "+section+" ";
const string source_msg = (sourcename.empty())? "" : "from \""+sourcename+"\" at line "+tmp.str();
const std::string key_msg = (key.empty())? "" : "key "+key+" ";
const std::string key_value_link = (key.empty() && !value.empty())? "value " : "";
const std::string value_msg = (value.empty())? "" : value+" " ;
const std::string keyvalue_msg = (key.empty() && value.empty())? "key/value " : key_msg+key_value_link+value_msg;
const std::string section_msg = (section.empty())? "" : "in section "+section+" ";
const std::string source_msg = (sourcename.empty())? "" : "from \""+sourcename+"\" at line "+tmp.str();
throw InvalidFormatException("Error reading "+keyvalue_msg+section_msg+source_msg, AT);
}
......@@ -299,8 +299,8 @@ size_t Config::findKeys(std::vector<std::string>& vecResult, const std::string&
//Loop through keys, look for match - push it into vecResult
if (anywhere) {
const string key_pattern( IOUtils::strToUpper(keymatch) );
const string section_pattern( IOUtils::strToUpper(section) );
const std::string key_pattern( IOUtils::strToUpper(keymatch) );
const std::string section_pattern( IOUtils::strToUpper(section) );
for (map<string,string>::const_iterator it=properties.begin(); it != properties.end(); ++it) {
const size_t found_section = (it->first).find(section_pattern, 0);
......@@ -308,17 +308,17 @@ size_t Config::findKeys(std::vector<std::string>& vecResult, const std::string&
const size_t found_pos = (it->first).find(key_pattern, section_len);
if (found_pos!=string::npos) { //found it!
const string key( (it->first).substr(section_len + 2) ); //from pos to the end
const std::string key( (it->first).substr(section_len + 2) ); //from pos to the end
vecResult.push_back(key);
}
}
} else {
const string key_pattern( IOUtils::strToUpper(section) + "::" + IOUtils::strToUpper(keymatch) );
const std::string key_pattern( IOUtils::strToUpper(section) + "::" + IOUtils::strToUpper(keymatch) );
for (map<string,string>::const_iterator it=properties.begin(); it != properties.end(); ++it) {
const size_t found_pos = (it->first).find(key_pattern, 0);
if (found_pos==0) { //found it!
const string key( (it->first).substr(section_len + 2) ); //from pos to the end
const std::string key( (it->first).substr(section_len + 2) ); //from pos to the end
vecResult.push_back(key);
}
}
......@@ -332,7 +332,7 @@ std::string Config::extract_section(std::string key) const
const string::size_type pos = key.find("::");
if (pos != string::npos){
const string sectionname = key.substr(0, pos);
const std::string sectionname( key.substr(0, pos) );
key.erase(key.begin(), key.begin() + pos + 2); //delete section name
return sectionname;
}
......@@ -343,8 +343,8 @@ std::string Config::clean_import_path(const std::string& in_path) const
{
//if this is a relative path, prefix the import path with the current path
const std::string prefix = ( FileUtils::isAbsolutePath(in_path) )? "" : FileUtils::getPath(sourcename, true)+"/";
const std::string path = FileUtils::getPath(prefix+in_path, true); //clean & resolve path
const std::string filename = FileUtils::getFilename(in_path);
const std::string path( FileUtils::getPath(prefix+in_path, true) ); //clean & resolve path
const std::string filename( FileUtils::getFilename(in_path) );
return path + "/" + filename;
}
......@@ -356,11 +356,11 @@ void Config::write(const std::string& filename) const
if (fout.fail()) throw AccessException(filename, AT);
try {
string current_section;
std::string current_section;
unsigned int sectioncount = 0;
for (map<string,string>::const_iterator it=properties.begin(); it != properties.end(); ++it) {
const string key_full = it->first;
const string section = extract_section(key_full);
const std::string key_full( it->first );
const std::string section( extract_section(key_full) );
if (current_section != section) {
current_section = section;
......@@ -371,7 +371,7 @@ void Config::write(const std::string& filename) const
}
const size_t key_start = key_full.find_first_of(":");
const string value = it->second;
const std::string value( it->second );
if (value.empty()) continue;
if (key_start!=string::npos) //start after the "::" marking the section prefix
......
......@@ -74,10 +74,10 @@ bool Meteo1DInterpolator::resampleData(const Date& date, const std::vector<Meteo
bool isResampled = true;
ResamplingAlgorithms::ResamplingPosition elementpos = ResamplingAlgorithms::exact_match;
if (index == IOUtils::npos) { //nothing found append new element at the left or right
if (vecM.front().date > date) {
if (date < vecM.front().date) {
elementpos = ResamplingAlgorithms::begin;
index = 0;
} else if (vecM.back().date < date) {
} else if (date >= vecM.back().date) {
elementpos = ResamplingAlgorithms::end;
index = vecM.size() - 1;
}
......
......@@ -98,6 +98,10 @@ std::string StationData::getHash() const {
return stationID+"::"+stationName;
}
double StationData::getAltitude() const {
return position.getAltitude();
}
double StationData::getSlopeAngle() const {
return slope;
}
......
......@@ -56,6 +56,7 @@ class StationData {
std::string getStationName() const;
Coords getPosition() const;
std::string getHash() const;
double getAltitude() const;
/**
* @brief Get local slope angle
......
......@@ -27,7 +27,8 @@ namespace mio {
* The Station meteo data files is a station centered, ascii file format that has been designed with flexibility and ease of use in mind. Please refer to its
* <a href="../SMET_specifications.pdf">official format specification</a> for more information (including the list of standard parameters: TA, TSS, TSG,
* RH, VW, DW, ISWR, OSWR, ILWR, OLWR, PINT, PSUM, HS). For PINT, it is assumed that the intensity (in mm/h) is valid for the whole period between the actual
* time step and the previous one.
* time step and the previous one. You can also have a look at the following <A HREF="http://www.envidat.ch/dataset/10-16904-1">Weissfluhjoch dataset</A>
* as (quite large) example SMET dataset.
*
* This plugin can also provide Points Of Interest, given as a SMET file containing either latitude/longitude/altitude or easting/northing/altitude. For the latter,
* the header must contain the epsg code (see example below).
......
......@@ -22,11 +22,10 @@
#include <meteoio/plugins/libsmet.h>
#include <string>
#include <vector>
#ifdef _MSC_VER
#pragma warning(disable:4512) //we don't need any = operator!
#endif
namespace mio {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment