WSL/SLF GitLab Repository

Commit ab05786a authored by Fierz's avatar Fierz
Browse files

Revisiting ImisIO plugin:

- enhance and simplify sqlQueries. The joint snow_drift query now synchronizes the two stations queried (for example IMIS-snow w/ ENET-wind). The data query is efficient in terms of fetching time, that is 1.58 s for one full year of data (17484 rows a 17 columns) but the first conection may take forever (~50 s)
- disentangle stationID and stationName (staion_code and station_name, respectively, in station2.v_snow_drift_standort); parse/edit station_name to not contain spaces ;-)
- get temperature sensor depths and insert them in vecMeteoData
=> we now have a robust mapping of the MeteoData object (see r549) and can pass meta data to the application snowpack ;-)
parent 278e3e63
......@@ -152,6 +152,7 @@ Coords& Coords::operator=(const Coords& source) {
Coords Coords::merge(const Coords& coord1, const Coords& coord2) {
Coords tmp(coord1);
tmp.merge(coord2);
return tmp;
}
/**
......
......@@ -52,27 +52,6 @@ double IOManager::getAvgSamplingRate()
}
}
void IOManager::push_meteo_data(const ProcessingLevel& level, const Date& date_start, const Date& date_end,
const std::vector< METEO_TIMESERIE >& vecMeteo)
{
//perform check on date_start and date_end
if (date_end < date_start)
throw InvalidArgumentException("date_start cannot be greater than date_end", AT);
if (level == IOManager::filtered){
fcache_start = date_start;
fcache_end = date_end;
filtered_cache = vecMeteo;
} else if (level == IOManager::raw){
//push data into the BufferedIOHandler
fcache_start = fcache_end = Date(0.0, 0.);
filtered_cache.clear();
bufferedio.push_meteo_data(date_start, date_end, vecMeteo);
} else {
throw InvalidArgumentException("The processing level is invalid (should be raw OR filtered)", AT);
}
}
unsigned int IOManager::getStationData(const Date& date, STATION_TIMESERIE& vecStation)
{
vecStation.clear();
......@@ -128,9 +107,8 @@ void IOManager::fill_filtered_cache()
//ask the bufferediohandler for the whole buffer
const vector< METEO_TIMESERIE >& buffer = bufferedio.get_complete_buffer(fcache_start, fcache_end);
//cout << "Now filtering ... data for " << buffer.size() << " stations" << endl; clock_t cstart = std::clock();
//cout << "Now filtering ..." << endl;
meteoprocessor.process(buffer, filtered_cache);
//cout << "Filtering: " << ( std::clock() - cstart ) / (double)CLOCKS_PER_SEC << " seconds" << endl;
}
}
......
......@@ -45,10 +45,10 @@
#endif
#ifndef C_TO_K
#define C_TO_K( T ) ( T + 273.15 ) // degree Celsius to kelvin
#define C_TO_K( T ) ( T + Cst::t_water_freezing_pt ) // degree Celsius to kelvin
#endif
#ifndef K_TO_C
#define K_TO_C( T ) ( T - 273.15 ) // kelvin to degree Celsius
#define K_TO_C( T ) ( T - Cst::t_water_freezing_pt ) // kelvin to degree Celsius
#endif
namespace mio {
......
......@@ -31,18 +31,19 @@ const bool MeteoData::__init = MeteoData::initStaticData();
bool MeteoData::initStaticData()
{
//Associate unsigned int value and a string representation of a meteo parameter
static_meteoparamname[TA] = "TA";
static_meteoparamname[ISWR] = "ISWR";
static_meteoparamname[VW] = "VW";
static_meteoparamname[DW] = "DW";
static_meteoparamname[RH] = "RH";
static_meteoparamname[ILWR] = "ILWR";
static_meteoparamname[HNW] = "HNW";
static_meteoparamname[TSG] = "TSG";
static_meteoparamname[TSS] = "TSS";
static_meteoparamname[HS] = "HS";
static_meteoparamname[RSWR] = "RSWR";
static_meteoparamname[P] = "P";
static_meteoparamname[TA] = "TA";
static_meteoparamname[ISWR] = "ISWR";
static_meteoparamname[VW] = "VW";
static_meteoparamname[DW] = "DW";
static_meteoparamname[VW_MAX] = "VW_MAX";
static_meteoparamname[RH] = "RH";
static_meteoparamname[ILWR] = "ILWR";
static_meteoparamname[HNW] = "HNW";
static_meteoparamname[TSG] = "TSG";
static_meteoparamname[TSS] = "TSS";
static_meteoparamname[HS] = "HS";
static_meteoparamname[RSWR] = "RSWR";
static_meteoparamname[P] = "P";
return true;
}
......@@ -79,6 +80,7 @@ void MeteoData::associateMemberVariables()
meteoparam[ISWR] = &iswr;
meteoparam[VW] = &vw;
meteoparam[DW] = &dw;
meteoparam[VW_MAX] = &vw_max;
meteoparam[RH] = &rh;
meteoparam[ILWR] = &ilwr;
meteoparam[HNW] = &hnw;
......@@ -92,18 +94,19 @@ void MeteoData::associateMemberVariables()
void MeteoData::initParameterMap()
{//NOTE: any performace improvement here would make a big difference...
//Associate unsigned int value and a string representation of a meteo parameter
meteoparamname[TA] = "TA";
meteoparamname[ISWR] = "ISWR";
meteoparamname[VW] = "VW";
meteoparamname[DW] = "DW";
meteoparamname[RH] = "RH";
meteoparamname[ILWR] = "ILWR";
meteoparamname[HNW] = "HNW";
meteoparamname[TSG] = "TSG";
meteoparamname[TSS] = "TSS";
meteoparamname[HS] = "HS";
meteoparamname[RSWR] = "RSWR";
meteoparamname[P] = "P";
meteoparamname[TA] = "TA";
meteoparamname[ISWR] = "ISWR";
meteoparamname[VW] = "VW";
meteoparamname[DW] = "DW";
meteoparamname[VW_MAX] = "VW_MAX";
meteoparamname[RH] = "RH";
meteoparamname[ILWR] = "ILWR";
meteoparamname[HNW] = "HNW";
meteoparamname[TSG] = "TSG";
meteoparamname[TSS] = "TSS";
meteoparamname[HS] = "HS";
meteoparamname[RSWR] = "RSWR";
meteoparamname[P] = "P";
//The following mapping needs to be done for every instance of MeteoData
associateMemberVariables();
......@@ -335,7 +338,7 @@ std::ostream& operator<<(std::ostream& os, const MeteoData& data) {
std::map<unsigned int, double*>::const_iterator it1;
for (it1=data.meteoparam.begin(); it1 != data.meteoparam.end(); it1++){
if( (*it1->second) != IOUtils::nodata ) {
os << setw(7) << data.getNameForParameter(it1->first) << ":" << setw(15) << *it1->second << "\n";
os << setw(8) << data.getNameForParameter(it1->first) << ":" << setw(15) << *it1->second << "\n";
}
}
os << "</meteo>\n";
......@@ -366,6 +369,7 @@ void MeteoData::Serialize(POPBuffer &buf, bool pack)
buf.Pack(&ta,1);
buf.Pack(&vw,1);
buf.Pack(&dw,1);
buf.Pack(&vw_max,1);
buf.Pack(&rh,1);
buf.Pack(&hnw,1);
buf.Pack(&iswr,1);
......@@ -384,6 +388,7 @@ void MeteoData::Serialize(POPBuffer &buf, bool pack)
buf.UnPack(&ta,1);
buf.UnPack(&vw,1);
buf.UnPack(&dw,1);
buf.UnPack(&vw_max,1);
buf.UnPack(&rh,1);
buf.UnPack(&hnw,1);
buf.UnPack(&iswr,1);
......
......@@ -53,6 +53,7 @@ class MeteoData {
RH, ///< Relative humidity
VW, ///< Wind velocity
DW, ///< Wind direction
VW_MAX, ///< Maximum wind velocity
ISWR, ///< Incoming short wave radiation
RSWR, ///< Reflected short wave radiation
ILWR, ///< Incoming long wave radiation
......@@ -138,18 +139,19 @@ class MeteoData {
//direct access allowed
Date date; ///<Timestamp of the measurement
StationData meta; ///<The meta data of the measurement
double ta; ///<Air temperature in Kelvin
double vw; ///<Wind velocity in m s-1
double ta; ///<Air temperature (K)
double vw; ///<Wind velocity (m s-1)
double dw; ///<Wind direction in degrees
double rh; ///<Relative humidity between 0 and 1
double hnw; ///<Precipitations in mm h-1
double iswr; ///<Incoming shortwave radiation in W m-2
double rswr; ///<Reflected Short Wave Radiation in W m-2
double ilwr; ///<Incoming Long wave radiation in W m-2
double tsg; ///<Soil or snow bottom temperature in Kelvin
double tss; ///<Soil or snow surface temperature in Kelvin
double hs; ///<Snow height in m
double p; ///<Atmospheric pressure in Pa
double vw_max; ///<Maximum wind velocity (m s-1)
double rh; ///<Relative humidity between 0 and 1 (1)
double hnw; ///<Precipitations in mm h-1, that is (kg m-2)
double iswr; ///<Incoming shortwave radiation (W m-2)
double rswr; ///<Reflected Short Wave Radiation (W m-2)
double ilwr; ///<Incoming Long wave radiation (W m-2)
double tsg; ///<Soil or snow bottom temperature (K)
double tss; ///<Soil or snow surface temperature (K)
double hs; ///<Snow height (m)
double p; ///<Atmospheric pressure (Pa)
unsigned int getNrOfParameters() const;
private:
......
......@@ -68,6 +68,7 @@ bool StationData::operator!=(const StationData& in) const {
StationData StationData::merge(const StationData& sd1, const StationData& sd2) {
StationData tmp(sd1);
tmp.merge(sd2);
return tmp;
}
void StationData::merge(const StationData& sd2) {
......
......@@ -30,6 +30,7 @@ namespace Cst {
const double gaz_constant = 8.31451; // (J mol-1 K-1)
const double p_water_triple_pt = 610.78; // (Pa)
const double t_water_freezing_pt = 273.15; // (K)
const double t_water_triple_pt = 273.16; // (K)
const double l_water_sublimation = 2.838e6; // (J Kg-1)
const double l_water_vaporization = 2.504e6; // (J Kg-1)
......
This diff is collapsed.
......@@ -110,21 +110,31 @@ class ImisIO : public IOInterface {
void closeDBConnection(oracle::occi::Environment*& env, oracle::occi::Connection*& conn);
void getDBParameters();
void getStationData(const std::string& stat_abk, const std::string& stao_nr, std::vector<std::string>& data2S,
oracle::occi::Connection*& conn);
std::string getDriftStation(std::string stationName, std::string stationNumber, oracle::occi::Connection*& conn);
void getImisData(const std::string& stat_abk, const std::string& stao_nr,
const std::vector<int>& datestart, const std::vector<int>& dateend,
std::vector< std::vector<std::string> >& dataImis,
oracle::occi::Environment*& env, oracle::occi::Connection*& conn);
void parseDataSet(const std::vector<std::string>& meteo_in, MeteoData& md);
unsigned int getStationIDs(const std::string& stat_code,
const std::string& sqlQuery, std::vector<std::string>& vecStationMetaData,
oracle::occi::Connection*& conn);
unsigned int getStationMetaData(const std::string& stat_abk, const std::string& stao_nr,
const std::string& sqlQuery, std::vector<std::string>& vecStationMetaData,
oracle::occi::Connection*& conn);
unsigned int getSensorDepths(const std::string& stat_abk, const std::string& stao_nr,
const std::string& sqlQuery, std::vector<std::string>& vecHts1,
oracle::occi::Connection*& conn);
bool getStationData(const std::string& stat_abk, const std::string& stao_nr,
const std::vector<int>& datestart, const std::vector<int>& dateend,
const std::vector<std::string>& i_vecHts,
std::vector< std::vector<std::string> >& vecMeteoData,
oracle::occi::Environment*& env, oracle::occi::Connection*& conn);
void parseDataSet(const std::vector<std::string>& meteo_in, MeteoData& md, bool& _fullStation);
void readData(const Date& dateStart, const Date& dateEnd, std::vector< std::vector<MeteoData> >& vecMeteo,
const unsigned int& stationindex, const std::vector<StationData>& vecStationNames,
const unsigned int& stationindex, const std::vector<StationData>& vecStationID,
oracle::occi::Environment*& env, oracle::occi::Connection*& conn);
void readStationNames(std::vector<std::string>& vecStationName);
void parseStationName(const std::string& stationName, std::string& stName, std::string& stNumber);
void readStationIDs(std::vector<std::string>& vecStationID);
void parseStationID(const std::string& stationID, std::string& stnAbbrev, std::string& stnNumber);
void readStationMetaData(oracle::occi::Connection*& conn);
void convertSnowTemperature(MeteoData& meteo, const std::string& parameter);
void convertSensorDepth(MeteoData& meteo, const std::string& parameter);
void convertUnits(MeteoData& meteo);
//helper functions for the Anetz coefficient mangling:
......@@ -136,19 +146,22 @@ class ImisIO : public IOInterface {
const std::vector< std::vector<double> > vec_of_psums,
const std::map<std::string, unsigned int>& mapAnetzNames, const unsigned int& stationindex,
std::vector< std::vector<MeteoData> >& vecMeteo);
void calculatePsum(const Date& dateStart, const Date& dateEnd, const std::vector< std::vector<MeteoData> >& vecMeteoAnetz,
std::vector< std::vector<double> >& vec_of_psums);
void calculatePsum(const Date& dateStart, const Date& dateEnd,
const std::vector< std::vector<MeteoData> >& vecMeteoAnetz,
std::vector< std::vector<double> >& vec_of_psums);
static const double in_tz; //timezone
Config cfg;
std::string coordin, coordinparam, coordout, coordoutparam; //projection parameters
std::vector<StationData> vecMyStation;
std::vector<StationData> vecStationMetaData;
std::map<std::string, std::string> mapDriftStation;
std::vector<double> vecHts;
static const double plugin_nodata; //plugin specific nodata value, e.g. -999
static const std::string sqlQueryMeteoData;
static const std::string sqlQueryStationData;
static const std::string sqlQueryDriftStation;
static const std::string sqlQueryMeteoDataDrift;
static const std::string sqlQueryStationIDs;
static const std::string sqlQueryStationMetaData;
static const std::string sqlQuerySensorDepths;
static const std::string sqlQueryMeteoDataDrift; // combined snow_drift query from two stations (ams.v_ams_raw)
static const std::string sqlQueryMeteoData; // single station query (ams.v_ams_raw)
std::string oracleUserName_in;
std::string oraclePassword_in;
std::string oracleDBName_in;
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment