WSL/SLF GitLab Repository

Commit 5cc7b300 authored by Mathias Bavay's avatar Mathias Bavay
Browse files

The ALPUG plugin now handles reading data split over multiple files. Cosmetic change in SMETIO.

parent 0a667927
......@@ -50,31 +50,31 @@ namespace mio {
* - METEOPATH: where to find/write the meteo data; [Input] and [Output] section
* - STATION#: input filename (in METEOPATH). As many meteofiles as needed may be specified
* - ALPUG_FIELDS: comma delimited list of fields. The fields <b>MUST</b> use the \ref meteoparam "MeteoData" naming scheme. Unknown or ignored fields are replaced by "%".
* - WRAP_MONTH: which month (numerical) triggers the start of a new file (belonging to the next year. Default: 10); [Input] section
*
* @code
* METEO = ALPUG
* METEOPATH = ./Met_files
* STATION1 = CAND5
* ALPUG_FIELDS = %,%,ID,timestamp,VW,VW_MAX,DW,TA,RH, RSWR,HS,%,%,%,%,%,TSG,%,%,%,TSS,ISWR,P
* WRAP_MONTH = 10
* @endcode
*/
const std::string ALPUG::dflt_extension = ".met";
const double ALPUG::plugin_nodata = -999.; //plugin specific nodata value. It can also be read by the plugin (depending on what is appropriate)
const size_t ALPUG::max_buffered_lines = 4; //how many lines to keep in buffer in order to detect and skip duplicates
ALPUG::ALPUG(const std::string& configfile) : cfg(configfile)
ALPUG::ALPUG(const std::string& configfile) : cfg(configfile), wrap_month(10)
{
parseInputOutputSection();
}
ALPUG::ALPUG(const Config& cfgreader) : cfg(cfgreader)
ALPUG::ALPUG(const Config& cfgreader) : cfg(cfgreader), wrap_month(10)
{
parseInputOutputSection();
}
ALPUG::~ALPUG() throw()
{}
void ALPUG::parseInputOutputSection()
{
//default timezones
......@@ -91,7 +91,9 @@ void ALPUG::parseInputOutputSection()
if (in_meteo == "ALPUG") { //keep it synchronized with IOHandler.cc for plugin mapping!!
cfg.getValue("METEOPATH", "Input", inpath);
vecIDs.clear();
cfg.getValues("STATION", "INPUT", vecIDs);
cfg.getValues("STATION", "Input", vecIDs);
cfg.getValue("WRAP_MONTH", "Input", wrap_month, IOUtils::nothrow);
string fields;
cfg.getValue("ALPUG_FIELDS", "Input", fields);
......@@ -177,10 +179,7 @@ bool ALPUG::parseLine(const std::string& filename, const char& eoln, const size_
throw InvalidFormatException("Invalid date \'"+tmp_vec[ii]+"\' in file \'"+filename+"\'", AT);
if (date<dateStart) return true;
if (date>dateEnd) {
std::cout << date.toString(Date::ISO) << " > " << dateEnd.toString(Date::ISO) << "\n";
return false;
}
if (date>dateEnd) return false;
md.setDate(date);
continue;
}
......@@ -207,55 +206,70 @@ bool ALPUG::parseLine(const std::string& filename, const char& eoln, const size_
return true;
}
//since ALPUG files seem to often contain duplicate lines, just skip them
bool ALPUG::isDuplicate(const std::string& line)
{
for(size_t ii=0; ii<LinesBuffer.size(); ++ii) {
if (line==LinesBuffer[ii]) return true;
}
if (LinesBuffer.size()>max_buffered_lines) LinesBuffer.pop_front();
LinesBuffer.push_back( line );
return false;
}
void ALPUG::readMetoFile(const std::string& station_id, const Date& dateStart, const Date& dateEnd,
std::vector<MeteoData>& vecM)
{
vecM.clear();
//TODO: read accross multiple years
// start in the previous file, keep going in the next file (since files are restarted on Oct. 01)
stringstream ss;
ss << dateStart.getYear();
const string extension = IOUtils::getExtension(station_id);
const string filename = (extension!="")? inpath + "/" + ss.str().substr(2,2) + station_id : inpath + "/" + ss.str().substr(2,2) + station_id + dflt_extension;
std::ifstream fin; //Input file streams
fin.clear();
fin.open (filename.c_str(), ios::in|ios::binary); //ascii does end of line translation, which messes up the pointer code
if (fin.fail()) {
ostringstream ss;
ss << "Error opening file \"" << filename << "\" for reading, possible reason: " << strerror(errno);
ss << " Please check file existence and permissions!";
throw FileAccessException(ss.str(), AT);
}
int start_year, start_month, start_day;
int end_year, end_month, end_day;
dateStart.getDate(start_year, start_month, start_day);
dateEnd.getDate(end_year, end_month, end_day);
const char eoln = smet::SMETCommon::getEoln(fin); //get the end of line character for the file
if (start_month>=wrap_month) start_year++;
if (end_month>=wrap_month) end_year++;
std::cout << "start_year=" << start_year << " end_year=" << end_year << "\n";
const size_t nr_of_data_fields = vecFields.size();
string line;
size_t linenr = 0;
while (!fin.eof()){
string line;
getline(fin, line, eoln);
linenr++;
if (line.empty())
continue; //Pure comment lines and empty lines are ignored
for(int year=start_year; year<=end_year; ++year) {
stringstream ss;
ss << year;
const string extension = IOUtils::getExtension(station_id);
const string filename = (!extension.empty())? inpath + "/" + ss.str().substr(2,2) + station_id : inpath + "/" + ss.str().substr(2,2) + station_id + dflt_extension;
Coords pos;
StationData sd(pos, station_id, station_id);
MeteoData md;
md.meta = sd;
bool isValid;
if (!parseLine(filename, eoln, nr_of_data_fields, dateStart, dateEnd, line, md, isValid))
break;
std::ifstream fin; //Input file streams
fin.clear();
fin.open (filename.c_str(), ios::in|ios::binary); //ascii does end of line translation, which messes up the pointer code
if (fin.fail()) {
ostringstream ss;
ss << "Error opening file \"" << filename << "\" for reading, possible reason: " << strerror(errno);
ss << " Please check file existence and permissions!";
throw FileAccessException(ss.str(), AT);
}
const char eoln = smet::SMETCommon::getEoln(fin); //get the end of line character for the file
const size_t nr_of_data_fields = vecFields.size();
string line;
while (!fin.eof()){
string line;
getline(fin, line, eoln);
if (line.empty())
continue; //Pure comment lines and empty lines are ignored
if (isDuplicate(line))
continue;
Coords pos;
MeteoData md(Date(), StationData(pos, station_id, station_id));
bool isValid;
if (!parseLine(filename, eoln, nr_of_data_fields, dateStart, dateEnd, line, md, isValid))
break;
if(isValid)
vecM.push_back( md );
}
if(isValid)
vecM.push_back( md );
fin.close();
}
fin.close();
}
void ALPUG::readMeteoData(const Date& dateStart, const Date& dateEnd,
......@@ -295,9 +309,4 @@ void ALPUG::write2DGrid(const Grid2DObject& /*grid_in*/, const MeteoGrids::Param
throw IOException("Nothing implemented here", AT);
}
void ALPUG::cleanup() throw()
{
}
} //namespace
......@@ -22,6 +22,7 @@
#include <meteoio/Config.h>
#include <string>
#include <deque>
namespace mio {
......@@ -38,7 +39,7 @@ class ALPUG : public IOInterface {
ALPUG(const std::string& configfile);
ALPUG(const ALPUG&);
ALPUG(const Config& cfgreader);
~ALPUG() throw();
~ALPUG() throw() {};
virtual void read2DGrid(Grid2DObject& grid_out, const std::string& parameter="");
virtual void read2DGrid(Grid2DObject& grid_out, const MeteoGrids::Parameters& parameter, const Date& date);
......@@ -60,7 +61,7 @@ class ALPUG : public IOInterface {
private:
void parseInputOutputSection();
void cleanup() throw();
bool isDuplicate(const std::string& line) ;
Date parseDINDate(const std::string& datum) const;
bool parseLine(const std::string& filename, const char& eoln, const size_t& nr_of_data_fields, const Date& dateStart, const Date& dateEnd, const std::string& line, MeteoData &md, bool &isValid) const;
void readMetoFile(const std::string& station_name, const Date& dateStart, const Date& dateEnd,
......@@ -68,12 +69,15 @@ class ALPUG : public IOInterface {
const Config cfg;
static const double plugin_nodata; //plugin specific nodata value, e.g. -999
static const size_t max_buffered_lines; //how many lines to keep in buffer to check for duplicates?
static const std::string dflt_extension;
std::deque<std::string> LinesBuffer;
std::string coordin, coordinparam, coordout, coordoutparam; //projection parameters
std::vector<std::string> vecIDs, vecFields; //read from the Config [Input] section
std::string inpath, outpath; //read from the Config [Output] section
double in_dflt_TZ, out_dflt_TZ;
unsigned short wrap_month;
};
} //namespace
......
......@@ -158,7 +158,7 @@ void SMETIO::parseInputOutputSection()
for (size_t ii=0; ii<vecFilenames.size(); ii++) {
const string filename = vecFilenames[ii];
const string extension = IOUtils::getExtension(filename);
const std::string file_and_path = (extension!="")? inpath+"/"+filename : inpath+"/"+filename+dflt_extension;
const std::string file_and_path = (!extension.empty())? inpath+"/"+filename : inpath+"/"+filename+dflt_extension;
if (!IOUtils::validFileName(file_and_path)) //Check whether filename is valid
throw InvalidFileNameException(file_and_path, AT);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment