WSL/SLF GitLab Repository

Commit a47911f4 authored by Mathias Bavay's avatar Mathias Bavay
Browse files

The bufferAlways method of BufferedIOHandler has been renamed as...

The bufferAlways method of BufferedIOHandler has been renamed as setBufferProperties. The A3DIO, IMISIO, SMETIO and SNIO plugins now correctly support time zones. The plugin howto documentation now warns the developer to take care of time zones issues.
parent 8e574ceb
......@@ -63,17 +63,26 @@ const unsigned int A3DIO::buffer_reserve = 23*24*2; //kind of average size of a
A3DIO::A3DIO(const std::string& configfile) : IOInterface(NULL), cfg(configfile)
{
IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
in_tz = out_tz = 0.;
cfg.getValue("TZ","Input",in_tz,Config::nothrow);
cfg.getValue("TZ","Output",out_tz,Config::nothrow);
}
//Copy constructor
A3DIO::A3DIO(const A3DIO& aio) : IOInterface(NULL), cfg(aio.cfg)
{
IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
in_tz = out_tz = 0.;
cfg.getValue("TZ","Input",in_tz,Config::nothrow);
cfg.getValue("TZ","Output",out_tz,Config::nothrow);
}
A3DIO::A3DIO(const Config& cfgreader) : IOInterface(NULL), cfg(cfgreader)
{
IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
in_tz = out_tz = 0.;
cfg.getValue("TZ","Input",in_tz,Config::nothrow);
cfg.getValue("TZ","Output",out_tz,Config::nothrow);
}
A3DIO::~A3DIO() throw()
......@@ -267,6 +276,7 @@ void A3DIO::read1DMeteo(const Date& dateStart, const Date& dateEnd,
Date tmp_date;
MeteoData tmpdata;
tmpdata.date.setTimeZone(in_tz);
StationData sd;
bool eofreached = false;
......@@ -337,6 +347,7 @@ void A3DIO::read1DMeteo(const Date& dateStart, const Date& dateEnd,
bool A3DIO::readMeteoDataLine(std::string& line, MeteoData& tmpdata, std::string filename)
{
Date tmp_date;
tmp_date.setTimeZone(in_tz);
int tmp_ymdh[4];
std::vector<std::string> tmpvec;
tmpvec.reserve(6);
......@@ -421,8 +432,8 @@ void A3DIO::read2DStations(const Date& timestamp, std::vector<StationData>& vecS
//1D and 2D data must correspond, that means that if there is 1D data
//for a certain date (e.g. 1.1.2006) then 2D data must exist (prec2006.txt etc),
//otherwise throw FileNotFoundException
Date startDate(vecMeteo[0][0].date);
Date endDate(vecMeteo[0][vecMeteo[0].size()-1].date);
Date startDate(vecMeteo[0][0].date.getJulianDate(), in_tz, false); //so that the correct filenames for the TZ will be constructed
Date endDate(vecMeteo[0][vecMeteo[0].size()-1].date.getJulianDate(), in_tz, false);
constructMeteo2DFilenames(startDate, endDate, filenames);//get all files for all years
stations = getNrOfStations(filenames, hashStations);
......@@ -577,6 +588,7 @@ void A3DIO::read2DMeteoData(const std::string& filename, const std::string& para
unsigned int columns;
std::vector<std::string> tmpvec, vec_names;
Date tmp_date;
tmp_date.setTimeZone(in_tz);
int tmp_ymdh[4];
fin.clear();
......@@ -799,7 +811,7 @@ int A3DIO::create1DFile(const std::vector< std::vector<MeteoData> >& data, const
file.flags ( std::ios::fixed );
for(unsigned int j=0; j<size; j++) {
int yyyy, mm, dd, hh;
data[ii][j].date.getDate(yyyy, mm, dd, hh);
data[ii][j].date.getDate(yyyy, mm, dd, hh); //HACK: use out_tz!!
file.fill('0');
file << setw(4) << yyyy << " " << setw(2) << mm << " " << setw(2) << dd << " " << setw(2) << hh << " ";
file.fill(' ');
......@@ -898,7 +910,7 @@ int A3DIO::write2DmeteoFile(const std::vector< std::vector<MeteoData> >& data,
file.flags ( ios::fixed );
for(unsigned int ii=0; ii<nb_timesteps; ii++) {
data[0][ii].date.getDate(year, month, day, hour);
data[0][ii].date.getDate(year, month, day, hour); //HACK: use out_tz!!
if(year!=startyear) {
//if the year has changed, we need to write to a new file
file.close();
......
......@@ -96,6 +96,7 @@ class A3DIO : public IOInterface {
const std::string& label);
void write2DMeteo(const std::vector< std::vector<MeteoData> >& data, const std::vector< std::vector<StationData> >& stations);
double in_tz, out_tz; //timezones
Config cfg;
std::ifstream fin; //Input file streams
std::string coordin, coordinparam, coordout, coordoutparam; //projection parameters
......
......@@ -29,7 +29,7 @@ BufferedIOHandler::BufferedIOHandler(IOHandler& _iohandler, const Config& _cfg)
: IOInterface(NULL), iohandler(_iohandler), cfg(_cfg), meteoprocessor(_cfg), meteoBuffer(), stationBuffer(), startDateBuffer(), endDateBuffer(), mapBufferedGrids()
#endif
{
setBufferProperties();
setDfltBufferProperties();
}
#ifdef _POPC_
......@@ -38,7 +38,7 @@ BufferedIOHandler::~BufferedIOHandler()
BufferedIOHandler::~BufferedIOHandler() throw()
#endif
{
setBufferProperties();
setDfltBufferProperties();
}
void BufferedIOHandler::read2DGrid(Grid2DObject& _grid2Dobj, const std::string& _filename)
......@@ -131,7 +131,7 @@ void BufferedIOHandler::readMeteoData(const Date& dateStart, const Date& dateEnd
readMeteoData(dateStart, dateEnd, vecMeteo, vecStation);
}
void BufferedIOHandler::setBufferProperties()
void BufferedIOHandler::setDfltBufferProperties()
{
always_rebuffer = true;
bufferbefore = Date(2.0); //minus 2 days
......@@ -294,9 +294,12 @@ bool BufferedIOHandler::bufferData(const Date& _date, const unsigned int& statio
return true;
}
void BufferedIOHandler::bufferAlways(const bool& bufferalways)
void BufferedIOHandler::setBufferPolicy(const buffer_policy& policy)
{
always_rebuffer = bufferalways;
if(policy==RECHECK_NODATA)
always_rebuffer=true;
else
always_rebuffer=false;
}
void BufferedIOHandler::setBufferDuration(const Date& _beforeDate, const Date& _afterDate)
......
......@@ -71,6 +71,12 @@ class BufferedIOHandler : public IOInterface {
virtual ~BufferedIOHandler() throw();
#endif
///Keywords for slope computation algorithm
typedef enum BUFFER_POLICY {
KEEP_NODATA, ///< when a data point is nodata in the buffer, return the buffered value
RECHECK_NODATA ///< when a data point is nodata in the buffer, refresh the buffer to see if a value could be found
} buffer_policy;
/**
* @brief The function returns the next MeteoData object for each station with a
* date >= to the parameter _date. vecMeteo and vecStation will be empty if there
......@@ -145,9 +151,11 @@ class BufferedIOHandler : public IOInterface {
#endif
virtual void write2DGrid(const Grid2DObject& grid_in, const std::string& options="");
//HACK: please do NOT use these methods, they will be replaced/changed/tweaked asap
//These methods should be considered experimental and NOT stable (from an API point of view)
void bufferAlways(const bool& bufferalways);
/**
* @brief Manually tune the buffering policy
* @param policy flag to define how to handle nodata (see BufferedIOHandler::buffer_policy)
*/
void setBufferPolicy(const buffer_policy& policy);
/**
* @brief Manually tune the buffer
......@@ -162,7 +170,7 @@ class BufferedIOHandler : public IOInterface {
private:
bool bufferData(const Date& _date, const unsigned int& stationindex);
void bufferAllData(const Date& _date);
void setBufferProperties();
void setDfltBufferProperties();
IOHandler& iohandler;
Config cfg;
......
......@@ -50,7 +50,7 @@ DEMObject::DEMObject(const slope_type& _algorithm) : Grid2DObject(), slope(), az
* @param _algorithm specify the default algorithm to use for slope computation (default=DFLT)
*/
DEMObject::DEMObject(const unsigned int& _ncols, const unsigned int& _nrows,
const double& _cellsize, const Coords& _llcorner, const slope_type& _algorithm)
const double& _cellsize, const Coords& _llcorner, const slope_type& _algorithm)
: Grid2DObject(_ncols, _nrows, _cellsize, _llcorner),
slope(), azi(), curvature(), Nx(), Ny(), Nz()
{
......@@ -72,8 +72,8 @@ DEMObject::DEMObject(const unsigned int& _ncols, const unsigned int& _nrows,
* @param _algorithm specify the default algorithm to use for slope computation (default=DFLT)
*/
DEMObject::DEMObject(const unsigned int& _ncols, const unsigned int& _nrows,
const double& _cellsize, const Coords& _llcorner, const Array2D<double>& _altitude,
const bool& _update, const slope_type& _algorithm)
const double& _cellsize, const Coords& _llcorner, const Array2D<double>& _altitude,
const bool& _update, const slope_type& _algorithm)
: Grid2DObject(_ncols, _nrows, _cellsize, _llcorner, _altitude),
slope(), azi(), curvature(), Nx(), Ny(), Nz()
{
......
......@@ -47,6 +47,8 @@ namespace mio {
*
* It is the responsibility of the plugin to properly convert the units toward the SI as used in MeteoIO (see the MeteoData class for a list of parameters and their units). Moreover, it is required by the BufferedIOHandler that each plugin that implements readMeteoData @em also implements the readStationData method. This is required so that the metadata is available even if not data exists for the requested time period.
*
* Finally, plugins must properly handle time zones. The Date class provides everything that is necessary, but the plugin developer must still properly set the time zone to each Date object (using the "TZ" key in the io.ini configuration file at least as a default value and afterwards overwriting with a plugin specified time zone specification if available). The time zone should be set \em before setting the date (so that the date that is given is understood as a date within the specified time zone).
*
* The meteorological data must be returned in a vector of vectors of MeteoData (and similarly, of StationData in order to provide the metadata). This consists of building a vector of MeteoData objects, each containing a set of measurements for a given timestamp, at a given location. This vector that contains the time series at one specific location is then added to a vector (pushed) that will then contain all locations.
* \image html vector_vector.png "vector of vector structure"
* \image latex vector_vector.eps "vector of vector structure" width=0.9\textwidth
......
......@@ -49,6 +49,7 @@ namespace mio {
*/
const double ImisIO::plugin_nodata = -999.0; //plugin specific nodata value
const double ImisIO::in_tz = 1.; //All IMIS data is in gmt+1
const string ImisIO::sqlQueryMeteoData = "SELECT to_char(datum, 'YYYY-MM-DD HH24:MI') as datum, avg(ta) as ta, avg(iswr) as iswr, avg(vw) as vw, avg(dw) as dw, avg(rh) as rh, avg(ilwr) as ilwr, avg(hnw) as hnw, avg(tsg) as tsg, avg(tss) as tss, avg(hs) as hs, avg(rswr) as rswr from ams.v_ams_raw WHERE stat_abk=:1 and stao_nr=:2 and datum>=:3 and datum<=:4 group by datum order by datum asc";
......@@ -281,8 +282,12 @@ void ImisIO::readData(const Date& dateStart, const Date& dateEnd, std::vector< s
parseStationName(vecMyStation.at(stationindex).getStationID(), stationName, stationNumber);
dateStart.getDate(datestart[0], datestart[1], datestart[2], datestart[3], datestart[4]);
dateEnd.getDate(dateend[0], dateend[1], dateend[2], dateend[3], dateend[4]);
//IMIS is in TZ=+1, so moving back to this timezone
Date dateS(dateStart), dateE(dateEnd);
dateS.setTimeZone(in_tz);
dateE.setTimeZone(in_tz);
dateS.getDate(datestart[0], datestart[1], datestart[2], datestart[3], datestart[4]);
dateE.getDate(dateend[0], dateend[1], dateend[2], dateend[3], dateend[4]);
//Oracle can't deal with an integer for the hour of 24, hence the following workaround
if (datestart[3] == 24){
......@@ -297,6 +302,7 @@ void ImisIO::readData(const Date& dateStart, const Date& dateEnd, std::vector< s
getImisData(stationName, stationNumber, datestart, dateend, vecResult);
MeteoData tmpmd;
tmpmd.date.setTimeZone(in_tz);
for (unsigned int ii=0; ii<vecResult.size(); ii++){
parseDataSet(vecResult[ii], tmpmd);
convertUnits(tmpmd);
......
......@@ -85,6 +85,7 @@ class ImisIO : public IOInterface {
void readStationMetaData();
void convertUnits(MeteoData& meteo);
static const double in_tz; //timezone
Config cfg;
std::string coordin, coordinparam, coordout, coordoutparam; //projection parameters
std::vector<StationData> vecMyStation;
......
......@@ -188,7 +188,7 @@ void SMETIO::readAssimilationData(const Date& /*date_in*/, Grid2DObject& /*da_ou
void SMETIO::readStationData(const Date&, std::vector<StationData>& vecStation)
{//big HACK: this is a barbaric code duplication!! Plus it should support coordinates in the data
//ie: it should use the given date!
//ie: it should use the given date! (and TZ)
vecStation.clear();
vecStation.reserve(nr_stations);
......@@ -234,9 +234,12 @@ void SMETIO::readStationData(const Date&, std::vector<StationData>& vecStation)
void SMETIO::parseInputOutputSection()
{
/*
* Parse the [Input] and [Output] sections within Config object cfg
*/
//default timezones
in_dflt_TZ = out_dflt_TZ = 0.;
cfg.getValue("TZ","Input",in_dflt_TZ,Config::nothrow);
cfg.getValue("TZ","Output",out_dflt_TZ,Config::nothrow);
// Parse the [Input] and [Output] sections within Config object cfg
IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
//Parse input section: extract number of files to read and store filenames in vecFiles
......@@ -388,6 +391,8 @@ void SMETIO::readDataBinary(const char&, const std::string&, const double& timez
while (!fin.eof()){
MeteoData md;
if ((timezone != IOUtils::nodata) && (timezone != 0.0))
md.date.setTimeZone(timezone);
StationData tmpsd = sd;
double lat=IOUtils::nodata, lon=IOUtils::nodata, alt=IOUtils::nodata;
unsigned int poscounter = 0;
......@@ -399,9 +404,6 @@ void SMETIO::readDataBinary(const char&, const std::string&, const double& timez
md.date.setDate(tmpval);
if ((timezone != IOUtils::nodata) && (timezone != 0.0))
md.date.setTimeZone(timezone);
if (md.date < dateStart)
continue;
if (md.date > dateEnd)
......@@ -469,6 +471,8 @@ void SMETIO::readDataAscii(const char& eoln, const std::string& filename, const
throw InvalidFormatException("In "+ filename + ": Invalid amount of data in data line "+line, AT);
MeteoData md;
if ((timezone != IOUtils::nodata) && (timezone != 0.0))
md.date.setTimeZone(timezone);
StationData tmpsd = sd;
double lat, lon, alt;
unsigned int poscounter = 0;
......@@ -476,10 +480,6 @@ void SMETIO::readDataAscii(const char& eoln, const std::string& filename, const
if (vecDataSequence[ii] == "timestamp"){
if (!IOUtils::convertString(md.date, tmpvec[ii]))
throw InvalidFormatException("In "+filename+": Timestamp "+tmpvec[ii]+" invalid in data line", AT);
if ((timezone != IOUtils::nodata) && (timezone != 0.0))
md.date.setTimeZone(timezone);
if (md.date < dateStart)
continue;
if (md.date > dateEnd)
......@@ -544,6 +544,7 @@ void SMETIO::readHeader(const char& eoln, const std::string& filename, bool& loc
//Now extract info from mapHeader
IOUtils::getValueForKey(mapHeader, "station_id", sd.stationID);
IOUtils::getValueForKey(mapHeader, "station_name", sd.stationName, IOUtils::nothrow);
timezone = in_dflt_TZ;
IOUtils::getValueForKey(mapHeader, "tz", timezone, IOUtils::nothrow);
sd.position.setProj(coordin, coordinparam); //set the default projection from config file
......@@ -556,7 +557,7 @@ void SMETIO::readHeader(const char& eoln, const std::string& filename, bool& loc
}
IOUtils::getValueForKey(mapHeader, "easting", easting, IOUtils::nothrow);
if (easting != IOUtils::nodata){ //HACK
if (easting != IOUtils::nodata){
IOUtils::getValueForKey(mapHeader, "northing", northing);
IOUtils::getValueForKey(mapHeader, "altitude", alt);
sd.position.setXY(easting, northing, alt);
......@@ -568,7 +569,7 @@ void SMETIO::readHeader(const char& eoln, const std::string& filename, bool& loc
//now trying to read lat/long (second, so that it has precedence over east/north coordinates)
double lat=IOUtils::nodata, lon=IOUtils::nodata;
IOUtils::getValueForKey(mapHeader, "latitude", lat, IOUtils::nothrow);
if (lat != IOUtils::nodata){ //HACK
if (lat != IOUtils::nodata){
IOUtils::getValueForKey(mapHeader, "longitude", lon);
IOUtils::getValueForKey(mapHeader, "altitude", alt);
sd.position.setLatLon(lat, lon, alt);
......
......@@ -99,6 +99,7 @@ class SMETIO : public IOInterface {
std::vector<std::string> vecFiles; //read from the Config [Input] section
std::string outpath; //read from the Config [Output] section
bool outputIsAscii, outputIsGzipped;//read from the Config [Output] section
double in_dflt_TZ, out_dflt_TZ; //default time zones
Config cfg;
std::ifstream fin; //Input file streams
......
......@@ -54,16 +54,25 @@ const double SNIO::plugin_nodata = -999.0; //plugin specific nodata value
SNIO::SNIO(void (*delObj)(void*), const Config& i_cfg) : IOInterface(delObj), cfg(i_cfg)
{
IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
in_tz = out_tz = 0.;
cfg.getValue("TZ","Input",in_tz,Config::nothrow);
cfg.getValue("TZ","Output",out_tz,Config::nothrow);
}
SNIO::SNIO(const std::string& configfile) : IOInterface(NULL), cfg(configfile)
{
IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
in_tz = out_tz = 0.;
cfg.getValue("TZ","Input",in_tz,Config::nothrow);
cfg.getValue("TZ","Output",out_tz,Config::nothrow);
}
SNIO::SNIO(const Config& cfgreader) : IOInterface(NULL), cfg(cfgreader)
{
IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
in_tz = out_tz = 0.;
cfg.getValue("TZ","Input",in_tz,Config::nothrow);
cfg.getValue("TZ","Output",out_tz,Config::nothrow);
}
SNIO::~SNIO() throw()
......@@ -279,6 +288,7 @@ void SNIO::readMeteoData(const Date& dateStart, const Date& dateEnd,
if (ncols >= 15){//valid length for MeteoData
MeteoData md;
md.date.setTimeZone(in_tz);
parseMeteoLine(tmpvec, filename + ":" + ss.str(), md);
if ((md.date >= dateStart) && (md.date <= dateEnd)){//check date and add to vectors
......
......@@ -76,6 +76,8 @@ class SNIO : public IOInterface {
void readMetaData(unsigned int& nrOfStations);
void parseMetaDataLine(const std::vector<std::string>& vecLine, StationData& sd);
void cleanup() throw();
double in_tz, out_tz;
Config cfg;
std::ifstream fin; //Input file streams
std::ofstream fout;//Output file streams
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment