The bufferAlways method of BufferedIOHandler has been renamed as...

The bufferAlways method of BufferedIOHandler has been renamed as setBufferProperties. The A3DIO, IMISIO, SMETIO and SNIO plugins now correctly support time zones. The plugin howto documentation now warns the developer to take care of time zones issues.

meteoio/A3DIO.cc

@@ -63,17 +63,26 @@ const unsigned int A3DIO::buffer_reserve = 23*24*2; //kind of average size of a
 A3DIO::A3DIO(const std::string& configfile) : IOInterface(NULL), cfg(configfile)
 {
 	IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
+	in_tz = out_tz = 0.;
+	cfg.getValue("TZ","Input",in_tz,Config::nothrow);
+	cfg.getValue("TZ","Output",out_tz,Config::nothrow);
 }
 
 //Copy constructor
 A3DIO::A3DIO(const A3DIO& aio) : IOInterface(NULL), cfg(aio.cfg)
 {
 	IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
+	in_tz = out_tz = 0.;
+	cfg.getValue("TZ","Input",in_tz,Config::nothrow);
+	cfg.getValue("TZ","Output",out_tz,Config::nothrow);
 }
 
 A3DIO::A3DIO(const Config& cfgreader) : IOInterface(NULL), cfg(cfgreader)
 {
 	IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
+	in_tz = out_tz = 0.;
+	cfg.getValue("TZ","Input",in_tz,Config::nothrow);
+	cfg.getValue("TZ","Output",out_tz,Config::nothrow);
 }
 
 A3DIO::~A3DIO() throw()
@@ -267,6 +276,7 @@ void A3DIO::read1DMeteo(const Date& dateStart, const Date& dateEnd,
 	Date tmp_date;
 	
 	MeteoData tmpdata;
+	tmpdata.date.setTimeZone(in_tz);
 	StationData sd;
 	bool eofreached = false;
 
@@ -337,6 +347,7 @@ void A3DIO::read1DMeteo(const Date& dateStart, const Date& dateEnd,
 bool A3DIO::readMeteoDataLine(std::string& line, MeteoData& tmpdata, std::string filename)
 {
 	Date tmp_date;
+	tmp_date.setTimeZone(in_tz);
 	int tmp_ymdh[4];
 	std::vector<std::string> tmpvec;
 	tmpvec.reserve(6);
@@ -421,8 +432,8 @@ void A3DIO::read2DStations(const Date& timestamp, std::vector<StationData>& vecS
 	//1D and 2D data must correspond, that means that if there is 1D data
 	//for a certain date (e.g. 1.1.2006) then 2D data must exist (prec2006.txt etc),
 	//otherwise throw FileNotFoundException
-	Date startDate(vecMeteo[0][0].date);
-	Date endDate(vecMeteo[0][vecMeteo[0].size()-1].date);
+	Date startDate(vecMeteo[0][0].date.getJulianDate(), in_tz, false); //so that the correct filenames for the TZ will be constructed
+	Date endDate(vecMeteo[0][vecMeteo[0].size()-1].date.getJulianDate(), in_tz, false);
 
 	constructMeteo2DFilenames(startDate, endDate, filenames);//get all files for all years
 	stations = getNrOfStations(filenames, hashStations);
@@ -577,6 +588,7 @@ void A3DIO::read2DMeteoData(const std::string& filename, const std::string& para
 	unsigned int columns;
 	std::vector<std::string> tmpvec, vec_names;
 	Date tmp_date;
+	tmp_date.setTimeZone(in_tz);
 	int tmp_ymdh[4];
 
 	fin.clear();
@@ -799,7 +811,7 @@ int A3DIO::create1DFile(const std::vector< std::vector<MeteoData> >& data, const
 			file.flags ( std::ios::fixed );
 			for(unsigned int j=0; j<size; j++) {
 				int yyyy, mm, dd, hh;
-				data[ii][j].date.getDate(yyyy, mm, dd, hh);
+				data[ii][j].date.getDate(yyyy, mm, dd, hh); //HACK: use out_tz!!
 				file.fill('0');
 				file << setw(4) << yyyy << " " << setw(2) << mm << " " << setw(2) << dd << " " << setw(2) << hh << " ";
 				file.fill(' ');
@@ -898,7 +910,7 @@ int A3DIO::write2DmeteoFile(const std::vector< std::vector<MeteoData> >& data,
 	file.flags ( ios::fixed );
 
 	for(unsigned int ii=0; ii<nb_timesteps; ii++) {
-		data[0][ii].date.getDate(year, month, day, hour);
+		data[0][ii].date.getDate(year, month, day, hour); //HACK: use out_tz!!
 		if(year!=startyear) {
 			//if the year has changed, we need to write to a new file
 			file.close();

meteoio/A3DIO.h

@@ -96,6 +96,7 @@ class A3DIO : public IOInterface {
 		                      const std::string& label);
 		void write2DMeteo(const std::vector< std::vector<MeteoData> >& data, const std::vector< std::vector<StationData> >& stations);
 
+		double in_tz, out_tz; //timezones
 		Config cfg;
 		std::ifstream fin; //Input file streams
 		std::string coordin, coordinparam, coordout, coordoutparam; //projection parameters

meteoio/BufferedIOHandler.cc

@@ -29,7 +29,7 @@ BufferedIOHandler::BufferedIOHandler(IOHandler& _iohandler, const Config& _cfg)
 	  : IOInterface(NULL), iohandler(_iohandler), cfg(_cfg), meteoprocessor(_cfg), meteoBuffer(), stationBuffer(), startDateBuffer(), endDateBuffer(), mapBufferedGrids()
 #endif
 {
-	setBufferProperties();
+	setDfltBufferProperties();
 }
 
 #ifdef _POPC_
@@ -38,7 +38,7 @@ BufferedIOHandler::~BufferedIOHandler()
 BufferedIOHandler::~BufferedIOHandler() throw()
 #endif
 {
-	setBufferProperties();
+	setDfltBufferProperties();
 }
 
 void BufferedIOHandler::read2DGrid(Grid2DObject& _grid2Dobj, const std::string& _filename)
@@ -131,7 +131,7 @@ void BufferedIOHandler::readMeteoData(const Date& dateStart, const Date& dateEnd
 	readMeteoData(dateStart, dateEnd, vecMeteo, vecStation);
 }
 
-void BufferedIOHandler::setBufferProperties()
+void BufferedIOHandler::setDfltBufferProperties()
 {
 	always_rebuffer = true;
 	bufferbefore = Date(2.0);  //minus 2 days
@@ -294,9 +294,12 @@ bool BufferedIOHandler::bufferData(const Date& _date, const unsigned int& statio
 	return true;
 }
 
-void BufferedIOHandler::bufferAlways(const bool& bufferalways)
+void BufferedIOHandler::setBufferPolicy(const buffer_policy& policy)
 {
-	always_rebuffer = bufferalways;
+	if(policy==RECHECK_NODATA)
+		always_rebuffer=true;
+	else
+		always_rebuffer=false;
 }
 
 void BufferedIOHandler::setBufferDuration(const Date& _beforeDate, const Date& _afterDate)

meteoio/BufferedIOHandler.h

@@ -71,6 +71,12 @@ class BufferedIOHandler : public IOInterface {
 		virtual ~BufferedIOHandler() throw();
 	#endif
 
+		///Keywords for slope computation algorithm
+		typedef enum BUFFER_POLICY {
+			KEEP_NODATA, ///< when a data point is nodata in the buffer, return the buffered value
+			RECHECK_NODATA ///< when a data point is nodata in the buffer, refresh the buffer to see if a value could be found
+		} buffer_policy;
+
 		/**
 		 * @brief The function returns the next MeteoData object for each station with a
 		 *        date >= to the parameter _date. vecMeteo and vecStation will be empty if there
@@ -145,9 +151,11 @@ class BufferedIOHandler : public IOInterface {
 #endif
 		virtual void write2DGrid(const Grid2DObject& grid_in, const std::string& options="");
 
-		//HACK: please do NOT use these methods, they will be replaced/changed/tweaked asap
-		//These methods should be considered experimental and NOT stable (from an API point of view)
-		void bufferAlways(const bool& bufferalways);
+		/**
+		 * @brief Manually tune the buffering policy
+		 * @param policy flag to define how to handle nodata (see BufferedIOHandler::buffer_policy)
+		 */
+		void setBufferPolicy(const buffer_policy& policy);
 
 		/**
 		 * @brief Manually tune the buffer
@@ -162,7 +170,7 @@ class BufferedIOHandler : public IOInterface {
 	private:
 		bool bufferData(const Date& _date, const unsigned int& stationindex);
 		void bufferAllData(const Date& _date);
-		void setBufferProperties();
+		void setDfltBufferProperties();
 
 		IOHandler& iohandler;
 		Config cfg;

meteoio/DEMObject.cc

@@ -50,7 +50,7 @@ DEMObject::DEMObject(const slope_type& _algorithm) : Grid2DObject(), slope(), az
 * @param _algorithm specify the default algorithm to use for slope computation (default=DFLT)
 */
 DEMObject::DEMObject(const unsigned int& _ncols, const unsigned int& _nrows,
-				const double& _cellsize, const Coords& _llcorner, const slope_type& _algorithm)
+                     const double& _cellsize, const Coords& _llcorner, const slope_type& _algorithm)
 	: Grid2DObject(_ncols, _nrows, _cellsize, _llcorner),
 	  slope(), azi(), curvature(), Nx(), Ny(), Nz()
 {
@@ -72,8 +72,8 @@ DEMObject::DEMObject(const unsigned int& _ncols, const unsigned int& _nrows,
 * @param _algorithm specify the default algorithm to use for slope computation (default=DFLT)
 */
 DEMObject::DEMObject(const unsigned int& _ncols, const unsigned int& _nrows,
-				const double& _cellsize, const Coords& _llcorner, const Array2D<double>& _altitude,
-				const bool& _update, const slope_type& _algorithm)
+                     const double& _cellsize, const Coords& _llcorner, const Array2D<double>& _altitude,
+                     const bool& _update, const slope_type& _algorithm)
 	: Grid2DObject(_ncols, _nrows, _cellsize, _llcorner, _altitude),
 	  slope(), azi(), curvature(), Nx(), Ny(), Nz()
 {

meteoio/IOInterface.h

@@ -47,6 +47,8 @@ namespace mio {
  * 
  * It is the responsibility of the plugin to properly convert the units toward the SI as used in MeteoIO (see the MeteoData class for a list of parameters and their units). Moreover, it is required by the BufferedIOHandler that each plugin that implements readMeteoData @em also implements the readStationData method. This is required so that the metadata is available even if not data exists for the requested time period.
  *
+ * Finally, plugins must properly handle time zones. The Date class provides everything that is necessary, but the plugin developer must still properly set the time zone to each Date object (using the "TZ" key in the io.ini configuration file at least as a default value and afterwards overwriting with a plugin specified time zone specification if available). The time zone should be set \em before setting the date (so that the date that is given is understood as a date within the specified time zone).
+ *
  * The meteorological data must be returned in a vector of vectors of MeteoData (and similarly, of StationData in order to provide the metadata). This consists of building a vector of MeteoData objects, each containing a set of measurements for a given timestamp, at a given location. This vector that contains the time series at one specific location is then added to a vector (pushed) that will then contain all locations.
  * \image html vector_vector.png "vector of vector structure"
  * \image latex vector_vector.eps "vector of vector structure" width=0.9\textwidth

meteoio/plugins/ImisIO.cc

@@ -49,6 +49,7 @@ namespace mio {
  */
 
 const double ImisIO::plugin_nodata = -999.0; //plugin specific nodata value
+const double ImisIO::in_tz = 1.; //All IMIS data is in gmt+1
 
 const string ImisIO::sqlQueryMeteoData = "SELECT to_char(datum, 'YYYY-MM-DD HH24:MI') as datum, avg(ta) as ta, avg(iswr) as iswr, avg(vw) as vw, avg(dw) as dw, avg(rh) as rh, avg(ilwr) as ilwr, avg(hnw) as hnw, avg(tsg) as tsg, avg(tss) as tss, avg(hs) as hs, avg(rswr) as rswr from ams.v_ams_raw WHERE stat_abk=:1 and stao_nr=:2 and datum>=:3 and datum<=:4 group by datum order by datum asc";
 
@@ -281,8 +282,12 @@ void ImisIO::readData(const Date& dateStart, const Date& dateEnd, std::vector< s
 
 	parseStationName(vecMyStation.at(stationindex).getStationID(), stationName, stationNumber);
 
-	dateStart.getDate(datestart[0], datestart[1], datestart[2], datestart[3], datestart[4]);
-	dateEnd.getDate(dateend[0], dateend[1], dateend[2], dateend[3], dateend[4]);
+	//IMIS is in TZ=+1, so moving back to this timezone
+	Date dateS(dateStart), dateE(dateEnd);
+	dateS.setTimeZone(in_tz);
+	dateE.setTimeZone(in_tz);
+	dateS.getDate(datestart[0], datestart[1], datestart[2], datestart[3], datestart[4]);
+	dateE.getDate(dateend[0], dateend[1], dateend[2], dateend[3], dateend[4]);
 
 	//Oracle can't deal with an integer for the hour of 24, hence the following workaround
 	if (datestart[3] == 24){
@@ -297,6 +302,7 @@ void ImisIO::readData(const Date& dateStart, const Date& dateEnd, std::vector< s
 	getImisData(stationName, stationNumber, datestart, dateend, vecResult);
 
 	MeteoData tmpmd;
+	tmpmd.date.setTimeZone(in_tz);
 	for (unsigned int ii=0; ii<vecResult.size(); ii++){
 		parseDataSet(vecResult[ii], tmpmd);
 		convertUnits(tmpmd);

meteoio/plugins/ImisIO.h

@@ -85,6 +85,7 @@ class ImisIO : public IOInterface {
 		void readStationMetaData();
 		void convertUnits(MeteoData& meteo);
 
+		static const double in_tz; //timezone
 		Config cfg;
 		std::string coordin, coordinparam, coordout, coordoutparam; //projection parameters
 		std::vector<StationData> vecMyStation;

meteoio/plugins/SMETIO.cc

@@ -188,7 +188,7 @@ void SMETIO::readAssimilationData(const Date& /*date_in*/, Grid2DObject& /*da_ou
 
 void SMETIO::readStationData(const Date&, std::vector<StationData>& vecStation)
 {//big HACK: this is a barbaric code duplication!! Plus it should support coordinates in the data
-//ie: it should use the given date!
+//ie: it should use the given date! (and TZ)
 	vecStation.clear();
 	vecStation.reserve(nr_stations);
 
@@ -234,9 +234,12 @@ void SMETIO::readStationData(const Date&, std::vector<StationData>& vecStation)
 
 void SMETIO::parseInputOutputSection()
 {
-	/*
-	 * Parse the [Input] and [Output] sections within Config object cfg
-	 */
+	//default timezones
+	in_dflt_TZ = out_dflt_TZ = 0.;
+	cfg.getValue("TZ","Input",in_dflt_TZ,Config::nothrow);
+	cfg.getValue("TZ","Output",out_dflt_TZ,Config::nothrow);
+
+	// Parse the [Input] and [Output] sections within Config object cfg
 	IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
 
 	//Parse input section: extract number of files to read and store filenames in vecFiles
@@ -388,6 +391,8 @@ void SMETIO::readDataBinary(const char&, const std::string&, const double& timez
 
 	while (!fin.eof()){
 		MeteoData md;
+		if ((timezone != IOUtils::nodata) && (timezone != 0.0))
+			md.date.setTimeZone(timezone);
 		StationData tmpsd = sd;
 		double lat=IOUtils::nodata, lon=IOUtils::nodata, alt=IOUtils::nodata;
 		unsigned int poscounter = 0;
@@ -399,9 +404,6 @@ void SMETIO::readDataBinary(const char&, const std::string&, const double& timez
 
 				md.date.setDate(tmpval);
 
-				if ((timezone != IOUtils::nodata) && (timezone != 0.0))
-					md.date.setTimeZone(timezone);
-
 				if (md.date < dateStart)
 					continue;
 				if (md.date > dateEnd)
@@ -469,6 +471,8 @@ void SMETIO::readDataAscii(const char& eoln, const std::string& filename, const
 			throw InvalidFormatException("In "+ filename + ": Invalid amount of data in data line "+line, AT);
 
 		MeteoData md;
+		if ((timezone != IOUtils::nodata) && (timezone != 0.0))
+			md.date.setTimeZone(timezone);
 		StationData tmpsd = sd;
 		double lat, lon, alt;
 		unsigned int poscounter = 0;
@@ -476,10 +480,6 @@ void SMETIO::readDataAscii(const char& eoln, const std::string& filename, const
 			if (vecDataSequence[ii] == "timestamp"){
 				if (!IOUtils::convertString(md.date, tmpvec[ii]))
 					throw InvalidFormatException("In "+filename+": Timestamp "+tmpvec[ii]+" invalid in data line", AT);
-
-				if ((timezone != IOUtils::nodata) && (timezone != 0.0))
-					md.date.setTimeZone(timezone);
-
 				if (md.date < dateStart)
 					continue;
 				if (md.date > dateEnd)
@@ -544,6 +544,7 @@ void SMETIO::readHeader(const char& eoln, const std::string& filename, bool& loc
 	//Now extract info from mapHeader
 	IOUtils::getValueForKey(mapHeader, "station_id", sd.stationID);
 	IOUtils::getValueForKey(mapHeader, "station_name", sd.stationName, IOUtils::nothrow);
+	timezone = in_dflt_TZ;
 	IOUtils::getValueForKey(mapHeader, "tz", timezone, IOUtils::nothrow);
 	sd.position.setProj(coordin, coordinparam); //set the default projection from config file
 
@@ -556,7 +557,7 @@ void SMETIO::readHeader(const char& eoln, const std::string& filename, bool& loc
 	}
 	
 	IOUtils::getValueForKey(mapHeader, "easting", easting, IOUtils::nothrow);
-	if (easting != IOUtils::nodata){ //HACK
+	if (easting != IOUtils::nodata){
 		IOUtils::getValueForKey(mapHeader, "northing", northing);
 		IOUtils::getValueForKey(mapHeader, "altitude", alt);
 		sd.position.setXY(easting, northing, alt);
@@ -568,7 +569,7 @@ void SMETIO::readHeader(const char& eoln, const std::string& filename, bool& loc
 	//now trying to read lat/long (second, so that it has precedence over east/north coordinates)
 	double lat=IOUtils::nodata, lon=IOUtils::nodata;
 	IOUtils::getValueForKey(mapHeader, "latitude", lat, IOUtils::nothrow);
-	if (lat != IOUtils::nodata){ //HACK
+	if (lat != IOUtils::nodata){ 
 		IOUtils::getValueForKey(mapHeader, "longitude", lon);
 		IOUtils::getValueForKey(mapHeader, "altitude", alt);
 		sd.position.setLatLon(lat, lon, alt);

meteoio/plugins/SMETIO.h

@@ -99,6 +99,7 @@ class SMETIO : public IOInterface {
 		std::vector<std::string> vecFiles;  //read from the Config [Input] section
 		std::string outpath;                //read from the Config [Output] section
 		bool outputIsAscii, outputIsGzipped;//read from the Config [Output] section
+		double in_dflt_TZ, out_dflt_TZ;     //default time zones
 
 		Config cfg;
 		std::ifstream fin; //Input file streams

meteoio/plugins/SNIO.cc

@@ -54,16 +54,25 @@ const double SNIO::plugin_nodata = -999.0; //plugin specific nodata value
 SNIO::SNIO(void (*delObj)(void*), const Config& i_cfg) : IOInterface(delObj), cfg(i_cfg)
 {
 	IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
+	in_tz = out_tz = 0.;
+	cfg.getValue("TZ","Input",in_tz,Config::nothrow);
+	cfg.getValue("TZ","Output",out_tz,Config::nothrow);
 }
 
 SNIO::SNIO(const std::string& configfile) : IOInterface(NULL), cfg(configfile)
 {
 	IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
+	in_tz = out_tz = 0.;
+	cfg.getValue("TZ","Input",in_tz,Config::nothrow);
+	cfg.getValue("TZ","Output",out_tz,Config::nothrow);
 }
 
 SNIO::SNIO(const Config& cfgreader) : IOInterface(NULL), cfg(cfgreader)
 {
 	IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
+	in_tz = out_tz = 0.;
+	cfg.getValue("TZ","Input",in_tz,Config::nothrow);
+	cfg.getValue("TZ","Output",out_tz,Config::nothrow);
 }
 
 SNIO::~SNIO() throw()
@@ -279,6 +288,7 @@ void SNIO::readMeteoData(const Date& dateStart, const Date& dateEnd,
 			
 				if (ncols >= 15){//valid length for MeteoData
 					MeteoData md;
+					md.date.setTimeZone(in_tz);
 					parseMeteoLine(tmpvec, filename + ":" + ss.str(), md);
 					
 					if ((md.date >= dateStart) && (md.date <= dateEnd)){//check date and add to vectors

meteoio/plugins/SNIO.h

@@ -76,6 +76,8 @@ class SNIO : public IOInterface {
 		void readMetaData(unsigned int& nrOfStations);
 		void parseMetaDataLine(const std::vector<std::string>& vecLine, StationData& sd);
 		void cleanup() throw();
+
+		double in_tz, out_tz;
 		Config cfg;
 		std::ifstream fin; //Input file streams
 		std::ofstream fout;//Output file streams