The SNIO and SMETIO plugins now use METEOPATH for both inputs and outputs....

The SNIO and SMETIO plugins now use METEOPATH for both inputs and outputs. This is more consistent with other plugins and usage.

Some unused variables/methods have been removed from BufferedIOHandler (namely, the "buffer_always" functionality).

A first implementation of fileExists() and readDirectory() for Windows has been written. This has not yet been tested (even for compilation) since it requires Visual C++. This moves forward to address issues 143 and 144

doc/examples/io.ini

@@ -44,17 +44,22 @@ GRID2D		= ARC
 
 #Snowpack input
 ; METEO		= SNOWPACK
-; METEOPATH	= ./
+; METEOPATH	= input
+; NROFSTATIONS	= 1
+; METAFILE	= IMIS_Extracted_Info.txt ;metadata for all stations
+; STATION1	= DAV1 ;this is used as a stationID to get meta data from METAFILE
+; METEOFILE1	= MST96_RR.inp
 
 #SMET meteorological file format
 METEO		= SMET
-STATION1	= input/meteo/FLU2.smet
-STATION2	= input/meteo/FIR2.smet
-STATION3	= input/meteo/FRA2.smet
-STATION4	= input/meteo/GLA2.smet
-STATION5	= input/meteo/ILI2.smet
-STATION6	= input/meteo/OTT2.smet
-STATION7	= input/meteo/TUJ3.smet
+METEOPATH	= ./input/meteo
+STATION1	= FLU2.smet
+STATION2	= FIR2.smet
+STATION3	= FRA2.smet
+STATION4	= GLA2.smet
+STATION5	= ILI2.smet
+STATION6	= OTT2.smet
+STATION7	= TUJ3.smet
 
 #IMIS network database input -> IMIS plugin
 ; METEO		= IMIS

meteoio/BufferedIOHandler.cc

@@ -123,8 +123,6 @@ void BufferedIOHandler::writeMeteoData(const std::vector< METEO_TIMESERIE >& vec
 
 void BufferedIOHandler::setDfltBufferProperties()
 {
-	always_rebuffer = false;
-
 	double chunk_size_days = 15.; //default chunk size value
 	chunks = 1;
 	cfg.getValue("BUFF_CHUNK_SIZE", "General", chunk_size_days, Config::nothrow); //in days
@@ -153,15 +151,6 @@ void BufferedIOHandler::setDfltBufferProperties()
 	}
 }
 
-void BufferedIOHandler::setBufferPolicy(const buffer_policy& policy)
-{
-	if (policy==RECHECK_NODATA){
-		always_rebuffer=true;
-	} else {
-		always_rebuffer=false;
-	}
-}
-
 double BufferedIOHandler::getAvgSamplingRate()
 {
 	if (vec_buffer_meteo.size() > 0){
@@ -308,7 +297,6 @@ std::ostream& operator<<(std::ostream& os, const BufferedIOHandler& data)
 	os << "Config& cfg = " << hex << &data.cfg << dec << "\n";
 	os << "IOHandler &iohandler = " << hex << &data.iohandler << dec << "\n";
 
-	os << "Rebuffer if not found: " << data.always_rebuffer << "\n";
 	os << "Buffering " << data.chunks << " chunk(s) of " <<data.chunk_size.getJulianDate() << " days\n";
 	
 	os << "Current buffer content (" << data.vec_buffer_meteo.size() << " stations, " 

meteoio/BufferedIOHandler.h

@@ -116,12 +116,6 @@ class BufferedIOHandler : public IOInterface {
 #endif
 		virtual void write2DGrid(const Grid2DObject& grid_in, const std::string& options="");
 
-		/**
-		 * @brief Manually tune the buffering policy
-		 * @param policy flag to define how to handle nodata (see BufferedIOHandler::buffer_policy)
-		 */
-		void setBufferPolicy(const buffer_policy& policy);
-
 		double getAvgSamplingRate();
 
 		friend std::ostream& operator<<(std::ostream& os, const BufferedIOHandler& data);
@@ -140,7 +134,6 @@ class BufferedIOHandler : public IOInterface {
 		IOHandler& iohandler;
 		const Config& cfg;
 
-		bool always_rebuffer;
 		Date buffer_start, buffer_end;
 		Duration chunk_size; ///< How much data to read at once
 		Duration buff_before; ///< How much data to read before the requested date in buffer

meteoio/IOUtils.cc

@@ -20,6 +20,14 @@
 #include <meteoio/Config.h>    // to avoid forward declaration hell
 #include <meteoio/MeteoData.h> // to avoid forward declaration hell
 
+#ifdef _WIN32
+	#include <windows.h>
+	#include <strsafe.h>
+#else
+	#include <dirent.h>
+	#include <sys/stat.h>
+#endif
+
 namespace mio {
 
 bool IOUtils::checkEpsilonEquality(const double& val1, const double& val2, const double& epsilon)
@@ -134,25 +142,65 @@ bool IOUtils::readKeyValuePair(const std::string& in_line, const std::string& de
 	return true;
 }
 
+bool IOUtils::validFileName(const std::string& filename)
+{
+	size_t startpos = filename.find_first_not_of(" \t\n"); // Find the first character position after excluding leading blank spaces
+	if((startpos!=0) || (filename==".") || (filename=="..")) {
+		return false;
+	}
+
+	return true;
+}
+
+#ifdef _WIN32
 bool IOUtils::fileExists(const std::string& filename)
 {
-	struct stat buffer ;
+	return ( GetFileAttributes( (CString)filename ) != INVALID_FILE_ATTRIBUTES );
+}
 
-	if ((stat( filename.c_str(), &buffer))==0) {//File exists if stat returns 0
-		return true ;
+void IOUtils::readDirectory(const std::string& path, std::list<std::string>& dirlist, const std::string& pattern)
+{
+	WIN32_FIND_DATA ffd;
+	HANDLE hFind = INVALID_HANDLE_VALUE;
+
+	size_t path_length = path.length();
+	if (path_length > (MAX_PATH - 3)) {
+		std::cout << "Path " << path << "is too long (" << path_length << " characters)" << std::endl;
+		throw FileAccessException("Error opening directory " + path, AT);
 	}
 
-	return false;
-}
+	hFind = FindFirstFile((LPCSTR)(path+"\\"+pattern), &ffd);
+	if (INVALID_HANDLE_VALUE == hFind) {
+		throw FileAccessException("Error opening directory " + path, AT);
+	}
 
-bool IOUtils::validFileName(const std::string& filename)
+	do {
+		if (ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
+			//this is a directory -> do nothing
+		} else {
+			std::string filename(ffd.cFileName);
+			dirlist.push_back(filename);
+		}
+	}
+	while (FindNextFile(hFind, &ffd) != 0);
+
+	const DWORD dwError = GetLastError();
+	if (dwError != ERROR_NO_MORE_FILES) {
+		throw FileAccessException("Error listing files in directory " + path, AT);
+	}
+
+	FindClose(hFind);
+}
+#else
+bool IOUtils::fileExists(const std::string& filename)
 {
-	size_t startpos = filename.find_first_not_of(" \t\n"); // Find the first character position after excluding leading blank spaces
-	if((startpos!=0) || (filename==".") || (filename=="..")) {
-		return false;
+	struct stat buffer ;
+
+	if ((stat( filename.c_str(), &buffer))==0) {//File exists if stat returns 0
+		return true ;
 	}
 
-	return true;
+	return false;
 }
 
 void IOUtils::readDirectory(const std::string& path, std::list<std::string>& dirlist, const std::string& pattern)
@@ -179,6 +227,7 @@ void IOUtils::readDirectory(const std::string& path, std::list<std::string>& dir
 	}
 	closedir(dp);
 }
+#endif
 
 void IOUtils::readKeyValueHeader(std::map<std::string,std::string>& headermap,
 				  std::istream& fin,

meteoio/IOUtils.h

@@ -32,8 +32,6 @@
 #include <iostream>
 #include <cstdio>
 #include <fstream>
-#include <dirent.h>
-#include <sys/stat.h>
 #include <cctype>
 #include <limits>
 

meteoio/plugins/SMETIO.cc

@@ -31,16 +31,17 @@ namespace mio {
  *
  * @section template_keywords Keywords
  * This plugin uses the following keywords:
- * - STATION#: input filename and path. As many meteofiles as needed may be specified
- * - METEOPATH: output directory where to write the output meteofiles
+ * - STATION#: input filename (in METEOPATH). As many meteofiles as needed may be specified
+ * - METEOPATH: meteo files directory where to read/write the meteofiles; [Input] and [Output] sections
  * - METEOPARAM: output file format options (ASCII or BINARY that might be followed by GZIP)
  *
  * Example:
  * @code
  * [Input]
- * STATION1 = ./input/uppper_station.smet
- * STATION2 = ./input/lower_station.smet
- * STATION3 = ./input/outlet_station.smet
+ * METEOPATH = ./input
+ * STATION1 = uppper_station.smet
+ * STATION2 = lower_station.smet
+ * STATION3 = outlet_station.smet
  * [Output]
  * METEOPATH = ./output
  * METEOPARAM = ASCII GZIP
@@ -247,6 +248,8 @@ void SMETIO::parseInputOutputSection()
 	IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
 
 	//Parse input section: extract number of files to read and store filenames in vecFiles
+	std::string inpath;
+	cfg.getValue("METEOPATH", "Input", inpath);
 	unsigned int counter = 1;
 	string filename = "";
 	do {
@@ -257,10 +260,12 @@ void SMETIO::parseInputOutputSection()
 		cfg.getValue(ss.str(), "Input", filename, Config::nothrow);
 
 		if (filename != ""){
-			if (!IOUtils::validFileName(filename)) //Check whether filename is valid
-				throw InvalidFileNameException(filename, AT);
+			stringstream file_and_path;
+			file_and_path << inpath << "/" << filename;
+			if (!IOUtils::validFileName(file_and_path.str())) //Check whether filename is valid
+				throw InvalidFileNameException(file_and_path.str(), AT);
 
-			vecFiles.push_back(filename);
+			vecFiles.push_back(file_and_path.str());
 		}
 		counter++;
 	} while (filename != "");

meteoio/plugins/SNIO.cc

@@ -42,10 +42,10 @@ namespace mio {
  * This plugin uses the following keywords:
  * - COORDSYS: coordinate system (see Coords); [Input] and [Output] section
  * - COORDPARAM: extra coordinates parameters (see Coords); [Input] and [Output] section
- * - METEOPATH: path to the output directory; [Output] section
+ * - METEOPATH: path to the meteo files directory; [Input] and [Output] sections
  * - METEOFILE#: input meteo data file, e.g. METEOFILE1, METEOFILE2; [Input] section
  * - STATION#: station name as listed in the METAFILE, e.g. STATION1, STATION2; [Input] section
- * - METAFILE: filename of the meta data file; [Input] section
+ * - METAFILE: filename of the meta data file (in METEOPATH); [Input] section
  * - NROFSTATIONS: integer, the number of stations for which meteo files are provided; [Input] section
  */
 
@@ -133,19 +133,16 @@ void SNIO::readStationData(const Date&, std::vector<StationData>& vecStation)
 }
 
 void SNIO::readMetaData(unsigned int& nrOfStations)
-{	
-	/*
-	 * The format of the meta data file is as follows:
-	 * SHORTNAME LONGNAME altitude latitude longitude 
-	 * ALI2 Allieres:Chenau 1767 6.993 46.489 1.22
-	 */
-
-	string stationname, metafile;
+{
+	string stationname, metafile, inpath;
 	cfg.getValue("METAFILE", "Input", metafile);
-	if (!IOUtils::validFileName(metafile))
-		throw InvalidFileNameException(metafile, AT);
-	if (!IOUtils::fileExists(metafile))
-		throw FileNotFoundException(metafile, AT);
+	cfg.getValue("METEOPATH", "Input", inpath);
+	stringstream meta_with_path;
+	meta_with_path << inpath << "/" << metafile;
+	if ( !IOUtils::validFileName(meta_with_path.str()) )
+		throw InvalidFileNameException(meta_with_path.str(), AT);
+	if ( !IOUtils::fileExists(meta_with_path.str()) )
+		throw FileNotFoundException(meta_with_path.str(), AT);
 	fin.clear();
 	
 	//Loop over all stations
@@ -156,11 +153,11 @@ void SNIO::readMetaData(unsigned int& nrOfStations)
 
 		cfg.getValue("STATION" + snum.str(), "Input", stationname);
 
-		fin.open (metafile.c_str(), std::ifstream::in);	
+		fin.open (meta_with_path.str().c_str(), std::ifstream::in);
 		if (fin.fail())
-			throw FileAccessException(metafile, AT);
+			throw FileAccessException(meta_with_path.str(), AT);
 		
-		try{ 
+		try{
 			char eoln = IOUtils::getEoln(fin); //get the end of line character for the file
 			
 			unsigned int linenr = 0;
@@ -179,7 +176,7 @@ void SNIO::readMetaData(unsigned int& nrOfStations)
 				if (ncols==0){
 					//Ignore empty lines
 				} else if ((ncols<6) || (ncols>6)){
-					throw InvalidFormatException(metafile+":"+ss.str() + " each line must have 6 columns", AT);
+					throw InvalidFormatException(meta_with_path.str()+":"+ss.str() + " each line must have 6 columns", AT);
 				} else {
 					//6 columns exist
 					if (tmpvec.at(0) == stationname){
@@ -211,12 +208,12 @@ void SNIO::parseMetaDataLine(const std::vector<std::string>& vecLine, StationDat
 
 	Coords stationcoord(coordin, coordinparam);
 	stationcoord.setLatLon(tmpdata[3], tmpdata[4], tmpdata[2]);
-	sd.setStationData(stationcoord, vecLine[0], vecLine[0]);
+	sd.setStationData(stationcoord, vecLine[0], vecLine[1]);
 }
 
 
 void SNIO::readMeteoData(const Date& dateStart, const Date& dateEnd, std::vector< std::vector<MeteoData> >& vecMeteo, 
-					const unsigned int&)
+                         const unsigned int&)
 {
 	/*
 	 * Read the meteorological snowpack input file, formatted as follows:
@@ -226,10 +223,11 @@ void SNIO::readMeteoData(const Date& dateStart, const Date& dateEnd, std::vector
 	 */
 
 	vector<string> tmpvec;
-	string strNrOfStations;
+	string strNrOfStations, inpath;
 	unsigned int nrOfStations = 0;
 
 	cfg.getValue("NROFSTATIONS", "Input", strNrOfStations);
+	cfg.getValue("METEOPATH", "Input", inpath);
 
 	if (!IOUtils::convertString(nrOfStations, strNrOfStations, std::dec))
 		throw ConversionFailedException("Error while reading value for NROFSTATIONS", AT);
@@ -242,23 +240,24 @@ void SNIO::readMeteoData(const Date& dateStart, const Date& dateEnd, std::vector
 
 	for (unsigned int ii=0; ii<vecAllStations.size(); ii++){
 		string filename="", line="";
-		stringstream ss;
+		stringstream ss, file_with_path;
 
 		ss << ii+1;
 		cfg.getValue("METEOFILE"+ss.str(), "Input", filename);
+		file_with_path << inpath << "/" << filename;
 
-		if (!IOUtils::validFileName(filename))
-			throw InvalidFileNameException(filename, AT);
-		if (!IOUtils::fileExists(filename))
-			throw FileNotFoundException(filename, AT);
+		if ( !IOUtils::validFileName(file_with_path.str()) )
+			throw InvalidFileNameException(file_with_path.str(), AT);
+		if ( !IOUtils::fileExists(file_with_path.str()) )
+			throw FileNotFoundException(file_with_path.str(), AT);
   
 		fin.clear();
-		fin.open (filename.c_str(), std::ifstream::in);
+		fin.open (file_with_path.str().c_str(), std::ifstream::in);
 	
 		if (fin.fail())
-			throw FileAccessException(filename, AT);
+			throw FileAccessException(file_with_path.str(), AT);
 		if (fin.eof())
-			throw InvalidFileNameException(filename + ": Empty file", AT);
+			throw InvalidFileNameException(file_with_path.str() + ": Empty file", AT);
 	
 		char eoln = IOUtils::getEoln(fin); //get the end of line character for the file
 		
@@ -268,7 +267,7 @@ void SNIO::readMeteoData(const Date& dateStart, const Date& dateEnd, std::vector
 				if (line.substr(0,3) != "MTO") //if its not meta information rewind to the beginning
 					fin.seekg (0, ios::beg);
 			}else {
-				throw InvalidFormatException(filename + ": first line in invalid format", AT);
+				throw InvalidFormatException(file_with_path.str() + ": first line in invalid format", AT);
 			}
 		
 
@@ -286,7 +285,7 @@ void SNIO::readMeteoData(const Date& dateStart, const Date& dateEnd, std::vector
 				if (ncols >= 15){//valid length for MeteoData
 					MeteoData md;
 					md.meta = vecAllStations[ii];
-					parseMeteoLine(tmpvec, filename + ":" + ss.str(), md);
+					parseMeteoLine(tmpvec, file_with_path.str() + ":" + ss.str(), md);
 					
 					if ((md.date >= dateStart) && (md.date <= dateEnd)){//check date and add to vectors
 						convertUnits(md);
@@ -296,12 +295,12 @@ void SNIO::readMeteoData(const Date& dateStart, const Date& dateEnd, std::vector
 					if (tmpvec.at(0) == "END") {
 						break; //reached end of MeteoData
 					} else {
-						throw InvalidFormatException(filename + ":line " + ss.str() + " premature end of line", AT);
+						throw InvalidFormatException(file_with_path.str() + ":line " + ss.str() + " premature end of line", AT);
 					}
 				} else if (ncols == 0){
 					//Ignore empty lines
 				} else {
-					throw InvalidFormatException(filename + ":line " + ss.str() + " premature end of line", AT);
+					throw InvalidFormatException(file_with_path.str() + ":line " + ss.str() + " premature end of line", AT);
 				}
 			}				
 		} catch (std::exception& e){