WSL/SLF GitLab Repository

Commit 514fa75f authored by Mathias Bavay's avatar Mathias Bavay
Browse files

The SNIO and SMETIO plugins now use METEOPATH for both inputs and outputs....

The SNIO and SMETIO plugins now use METEOPATH for both inputs and outputs. This is more consistent with other plugins and usage.

Some unused variables/methods have been removed from BufferedIOHandler (namely, the "buffer_always" functionality).

A first implementation of fileExists() and readDirectory() for Windows has been written. This has not yet been tested (even for compilation) since it requires Visual C++. This moves forward to address issues 143 and 144
parent a62e26d6
......@@ -44,17 +44,22 @@ GRID2D = ARC
#Snowpack input
; METEO = SNOWPACK
; METEOPATH = ./
; METEOPATH = input
; NROFSTATIONS = 1
; METAFILE = IMIS_Extracted_Info.txt ;metadata for all stations
; STATION1 = DAV1 ;this is used as a stationID to get meta data from METAFILE
; METEOFILE1 = MST96_RR.inp
#SMET meteorological file format
METEO = SMET
STATION1 = input/meteo/FLU2.smet
STATION2 = input/meteo/FIR2.smet
STATION3 = input/meteo/FRA2.smet
STATION4 = input/meteo/GLA2.smet
STATION5 = input/meteo/ILI2.smet
STATION6 = input/meteo/OTT2.smet
STATION7 = input/meteo/TUJ3.smet
METEOPATH = ./input/meteo
STATION1 = FLU2.smet
STATION2 = FIR2.smet
STATION3 = FRA2.smet
STATION4 = GLA2.smet
STATION5 = ILI2.smet
STATION6 = OTT2.smet
STATION7 = TUJ3.smet
#IMIS network database input -> IMIS plugin
; METEO = IMIS
......
......@@ -123,8 +123,6 @@ void BufferedIOHandler::writeMeteoData(const std::vector< METEO_TIMESERIE >& vec
void BufferedIOHandler::setDfltBufferProperties()
{
always_rebuffer = false;
double chunk_size_days = 15.; //default chunk size value
chunks = 1;
cfg.getValue("BUFF_CHUNK_SIZE", "General", chunk_size_days, Config::nothrow); //in days
......@@ -153,15 +151,6 @@ void BufferedIOHandler::setDfltBufferProperties()
}
}
void BufferedIOHandler::setBufferPolicy(const buffer_policy& policy)
{
if (policy==RECHECK_NODATA){
always_rebuffer=true;
} else {
always_rebuffer=false;
}
}
double BufferedIOHandler::getAvgSamplingRate()
{
if (vec_buffer_meteo.size() > 0){
......@@ -308,7 +297,6 @@ std::ostream& operator<<(std::ostream& os, const BufferedIOHandler& data)
os << "Config& cfg = " << hex << &data.cfg << dec << "\n";
os << "IOHandler &iohandler = " << hex << &data.iohandler << dec << "\n";
os << "Rebuffer if not found: " << data.always_rebuffer << "\n";
os << "Buffering " << data.chunks << " chunk(s) of " <<data.chunk_size.getJulianDate() << " days\n";
os << "Current buffer content (" << data.vec_buffer_meteo.size() << " stations, "
......
......@@ -116,12 +116,6 @@ class BufferedIOHandler : public IOInterface {
#endif
virtual void write2DGrid(const Grid2DObject& grid_in, const std::string& options="");
/**
* @brief Manually tune the buffering policy
* @param policy flag to define how to handle nodata (see BufferedIOHandler::buffer_policy)
*/
void setBufferPolicy(const buffer_policy& policy);
double getAvgSamplingRate();
friend std::ostream& operator<<(std::ostream& os, const BufferedIOHandler& data);
......@@ -140,7 +134,6 @@ class BufferedIOHandler : public IOInterface {
IOHandler& iohandler;
const Config& cfg;
bool always_rebuffer;
Date buffer_start, buffer_end;
Duration chunk_size; ///< How much data to read at once
Duration buff_before; ///< How much data to read before the requested date in buffer
......
......@@ -20,6 +20,14 @@
#include <meteoio/Config.h> // to avoid forward declaration hell
#include <meteoio/MeteoData.h> // to avoid forward declaration hell
#ifdef _WIN32
#include <windows.h>
#include <strsafe.h>
#else
#include <dirent.h>
#include <sys/stat.h>
#endif
namespace mio {
bool IOUtils::checkEpsilonEquality(const double& val1, const double& val2, const double& epsilon)
......@@ -134,25 +142,65 @@ bool IOUtils::readKeyValuePair(const std::string& in_line, const std::string& de
return true;
}
bool IOUtils::validFileName(const std::string& filename)
{
size_t startpos = filename.find_first_not_of(" \t\n"); // Find the first character position after excluding leading blank spaces
if((startpos!=0) || (filename==".") || (filename=="..")) {
return false;
}
return true;
}
#ifdef _WIN32
bool IOUtils::fileExists(const std::string& filename)
{
struct stat buffer ;
return ( GetFileAttributes( (CString)filename ) != INVALID_FILE_ATTRIBUTES );
}
if ((stat( filename.c_str(), &buffer))==0) {//File exists if stat returns 0
return true ;
void IOUtils::readDirectory(const std::string& path, std::list<std::string>& dirlist, const std::string& pattern)
{
WIN32_FIND_DATA ffd;
HANDLE hFind = INVALID_HANDLE_VALUE;
size_t path_length = path.length();
if (path_length > (MAX_PATH - 3)) {
std::cout << "Path " << path << "is too long (" << path_length << " characters)" << std::endl;
throw FileAccessException("Error opening directory " + path, AT);
}
return false;
}
hFind = FindFirstFile((LPCSTR)(path+"\\"+pattern), &ffd);
if (INVALID_HANDLE_VALUE == hFind) {
throw FileAccessException("Error opening directory " + path, AT);
}
bool IOUtils::validFileName(const std::string& filename)
do {
if (ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
//this is a directory -> do nothing
} else {
std::string filename(ffd.cFileName);
dirlist.push_back(filename);
}
}
while (FindNextFile(hFind, &ffd) != 0);
const DWORD dwError = GetLastError();
if (dwError != ERROR_NO_MORE_FILES) {
throw FileAccessException("Error listing files in directory " + path, AT);
}
FindClose(hFind);
}
#else
bool IOUtils::fileExists(const std::string& filename)
{
size_t startpos = filename.find_first_not_of(" \t\n"); // Find the first character position after excluding leading blank spaces
if((startpos!=0) || (filename==".") || (filename=="..")) {
return false;
struct stat buffer ;
if ((stat( filename.c_str(), &buffer))==0) {//File exists if stat returns 0
return true ;
}
return true;
return false;
}
void IOUtils::readDirectory(const std::string& path, std::list<std::string>& dirlist, const std::string& pattern)
......@@ -179,6 +227,7 @@ void IOUtils::readDirectory(const std::string& path, std::list<std::string>& dir
}
closedir(dp);
}
#endif
void IOUtils::readKeyValueHeader(std::map<std::string,std::string>& headermap,
std::istream& fin,
......
......@@ -32,8 +32,6 @@
#include <iostream>
#include <cstdio>
#include <fstream>
#include <dirent.h>
#include <sys/stat.h>
#include <cctype>
#include <limits>
......
......@@ -31,16 +31,17 @@ namespace mio {
*
* @section template_keywords Keywords
* This plugin uses the following keywords:
* - STATION#: input filename and path. As many meteofiles as needed may be specified
* - METEOPATH: output directory where to write the output meteofiles
* - STATION#: input filename (in METEOPATH). As many meteofiles as needed may be specified
* - METEOPATH: meteo files directory where to read/write the meteofiles; [Input] and [Output] sections
* - METEOPARAM: output file format options (ASCII or BINARY that might be followed by GZIP)
*
* Example:
* @code
* [Input]
* STATION1 = ./input/uppper_station.smet
* STATION2 = ./input/lower_station.smet
* STATION3 = ./input/outlet_station.smet
* METEOPATH = ./input
* STATION1 = uppper_station.smet
* STATION2 = lower_station.smet
* STATION3 = outlet_station.smet
* [Output]
* METEOPATH = ./output
* METEOPARAM = ASCII GZIP
......@@ -247,6 +248,8 @@ void SMETIO::parseInputOutputSection()
IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
//Parse input section: extract number of files to read and store filenames in vecFiles
std::string inpath;
cfg.getValue("METEOPATH", "Input", inpath);
unsigned int counter = 1;
string filename = "";
do {
......@@ -257,10 +260,12 @@ void SMETIO::parseInputOutputSection()
cfg.getValue(ss.str(), "Input", filename, Config::nothrow);
if (filename != ""){
if (!IOUtils::validFileName(filename)) //Check whether filename is valid
throw InvalidFileNameException(filename, AT);
stringstream file_and_path;
file_and_path << inpath << "/" << filename;
if (!IOUtils::validFileName(file_and_path.str())) //Check whether filename is valid
throw InvalidFileNameException(file_and_path.str(), AT);
vecFiles.push_back(filename);
vecFiles.push_back(file_and_path.str());
}
counter++;
} while (filename != "");
......
......@@ -42,10 +42,10 @@ namespace mio {
* This plugin uses the following keywords:
* - COORDSYS: coordinate system (see Coords); [Input] and [Output] section
* - COORDPARAM: extra coordinates parameters (see Coords); [Input] and [Output] section
* - METEOPATH: path to the output directory; [Output] section
* - METEOPATH: path to the meteo files directory; [Input] and [Output] sections
* - METEOFILE#: input meteo data file, e.g. METEOFILE1, METEOFILE2; [Input] section
* - STATION#: station name as listed in the METAFILE, e.g. STATION1, STATION2; [Input] section
* - METAFILE: filename of the meta data file; [Input] section
* - METAFILE: filename of the meta data file (in METEOPATH); [Input] section
* - NROFSTATIONS: integer, the number of stations for which meteo files are provided; [Input] section
*/
......@@ -133,19 +133,16 @@ void SNIO::readStationData(const Date&, std::vector<StationData>& vecStation)
}
void SNIO::readMetaData(unsigned int& nrOfStations)
{
/*
* The format of the meta data file is as follows:
* SHORTNAME LONGNAME altitude latitude longitude
* ALI2 Allieres:Chenau 1767 6.993 46.489 1.22
*/
string stationname, metafile;
{
string stationname, metafile, inpath;
cfg.getValue("METAFILE", "Input", metafile);
if (!IOUtils::validFileName(metafile))
throw InvalidFileNameException(metafile, AT);
if (!IOUtils::fileExists(metafile))
throw FileNotFoundException(metafile, AT);
cfg.getValue("METEOPATH", "Input", inpath);
stringstream meta_with_path;
meta_with_path << inpath << "/" << metafile;
if ( !IOUtils::validFileName(meta_with_path.str()) )
throw InvalidFileNameException(meta_with_path.str(), AT);
if ( !IOUtils::fileExists(meta_with_path.str()) )
throw FileNotFoundException(meta_with_path.str(), AT);
fin.clear();
//Loop over all stations
......@@ -156,11 +153,11 @@ void SNIO::readMetaData(unsigned int& nrOfStations)
cfg.getValue("STATION" + snum.str(), "Input", stationname);
fin.open (metafile.c_str(), std::ifstream::in);
fin.open (meta_with_path.str().c_str(), std::ifstream::in);
if (fin.fail())
throw FileAccessException(metafile, AT);
throw FileAccessException(meta_with_path.str(), AT);
try{
try{
char eoln = IOUtils::getEoln(fin); //get the end of line character for the file
unsigned int linenr = 0;
......@@ -179,7 +176,7 @@ void SNIO::readMetaData(unsigned int& nrOfStations)
if (ncols==0){
//Ignore empty lines
} else if ((ncols<6) || (ncols>6)){
throw InvalidFormatException(metafile+":"+ss.str() + " each line must have 6 columns", AT);
throw InvalidFormatException(meta_with_path.str()+":"+ss.str() + " each line must have 6 columns", AT);
} else {
//6 columns exist
if (tmpvec.at(0) == stationname){
......@@ -211,12 +208,12 @@ void SNIO::parseMetaDataLine(const std::vector<std::string>& vecLine, StationDat
Coords stationcoord(coordin, coordinparam);
stationcoord.setLatLon(tmpdata[3], tmpdata[4], tmpdata[2]);
sd.setStationData(stationcoord, vecLine[0], vecLine[0]);
sd.setStationData(stationcoord, vecLine[0], vecLine[1]);
}
void SNIO::readMeteoData(const Date& dateStart, const Date& dateEnd, std::vector< std::vector<MeteoData> >& vecMeteo,
const unsigned int&)
const unsigned int&)
{
/*
* Read the meteorological snowpack input file, formatted as follows:
......@@ -226,10 +223,11 @@ void SNIO::readMeteoData(const Date& dateStart, const Date& dateEnd, std::vector
*/
vector<string> tmpvec;
string strNrOfStations;
string strNrOfStations, inpath;
unsigned int nrOfStations = 0;
cfg.getValue("NROFSTATIONS", "Input", strNrOfStations);
cfg.getValue("METEOPATH", "Input", inpath);
if (!IOUtils::convertString(nrOfStations, strNrOfStations, std::dec))
throw ConversionFailedException("Error while reading value for NROFSTATIONS", AT);
......@@ -242,23 +240,24 @@ void SNIO::readMeteoData(const Date& dateStart, const Date& dateEnd, std::vector
for (unsigned int ii=0; ii<vecAllStations.size(); ii++){
string filename="", line="";
stringstream ss;
stringstream ss, file_with_path;
ss << ii+1;
cfg.getValue("METEOFILE"+ss.str(), "Input", filename);
file_with_path << inpath << "/" << filename;
if (!IOUtils::validFileName(filename))
throw InvalidFileNameException(filename, AT);
if (!IOUtils::fileExists(filename))
throw FileNotFoundException(filename, AT);
if ( !IOUtils::validFileName(file_with_path.str()) )
throw InvalidFileNameException(file_with_path.str(), AT);
if ( !IOUtils::fileExists(file_with_path.str()) )
throw FileNotFoundException(file_with_path.str(), AT);
fin.clear();
fin.open (filename.c_str(), std::ifstream::in);
fin.open (file_with_path.str().c_str(), std::ifstream::in);
if (fin.fail())
throw FileAccessException(filename, AT);
throw FileAccessException(file_with_path.str(), AT);
if (fin.eof())
throw InvalidFileNameException(filename + ": Empty file", AT);
throw InvalidFileNameException(file_with_path.str() + ": Empty file", AT);
char eoln = IOUtils::getEoln(fin); //get the end of line character for the file
......@@ -268,7 +267,7 @@ void SNIO::readMeteoData(const Date& dateStart, const Date& dateEnd, std::vector
if (line.substr(0,3) != "MTO") //if its not meta information rewind to the beginning
fin.seekg (0, ios::beg);
}else {
throw InvalidFormatException(filename + ": first line in invalid format", AT);
throw InvalidFormatException(file_with_path.str() + ": first line in invalid format", AT);
}
......@@ -286,7 +285,7 @@ void SNIO::readMeteoData(const Date& dateStart, const Date& dateEnd, std::vector
if (ncols >= 15){//valid length for MeteoData
MeteoData md;
md.meta = vecAllStations[ii];
parseMeteoLine(tmpvec, filename + ":" + ss.str(), md);
parseMeteoLine(tmpvec, file_with_path.str() + ":" + ss.str(), md);
if ((md.date >= dateStart) && (md.date <= dateEnd)){//check date and add to vectors
convertUnits(md);
......@@ -296,12 +295,12 @@ void SNIO::readMeteoData(const Date& dateStart, const Date& dateEnd, std::vector
if (tmpvec.at(0) == "END") {
break; //reached end of MeteoData
} else {
throw InvalidFormatException(filename + ":line " + ss.str() + " premature end of line", AT);
throw InvalidFormatException(file_with_path.str() + ":line " + ss.str() + " premature end of line", AT);
}
} else if (ncols == 0){
//Ignore empty lines
} else {
throw InvalidFormatException(filename + ":line " + ss.str() + " premature end of line", AT);
throw InvalidFormatException(file_with_path.str() + ":line " + ss.str() + " premature end of line", AT);
}
}
} catch (std::exception& e){
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment