WSL/SLF GitLab Repository

Commit ed0de23f authored by Mathias Bavay's avatar Mathias Bavay
Browse files

An unpleasant bug has been fixed in data_converter when handling large number...

An unpleasant bug has been fixed in data_converter when handling large number of stations and time ranges. Minor code cleanup in some plugins.
parent a47fc595
......@@ -15,8 +15,8 @@ void real_main(int argc, char** argv) {
}
Config cfg("io.ini");
Date d1, d2;
const double TZ = cfg.get("TIME_ZONE", "Input");
Date d1, d2;
double Tstep;
IOUtils::convertString(d1, argv[1], TZ);
IOUtils::convertString(d2, argv[2], TZ);
......@@ -25,23 +25,24 @@ void real_main(int argc, char** argv) {
std::vector< std::vector<MeteoData> > vecMeteo;
IOManager io(cfg);
//io.setProcessingLevel(IOManager::raw);
std::cout << "Reading input data" << std::endl;
//Very basic conversion: get the whole data set at once, with its original sampling rate
//io.getMeteoData(d1, d2, vecMeteo);
Timer timer;
timer.start();
//More elaborate conversion: sample the data to a specific rate
//by looping over the time and calling readMeteoData for each timestep
std::map<std::string, size_t> mapIDs; //over a large time range, the number of stations might change... this is the way to make it work
std::vector<MeteoData> Meteo; //we need some intermediate storage, for storing data sets for 1 timestep
io.getMeteoData(d1, Meteo); //we need to know how many stations will be available
vecMeteo.insert(vecMeteo.begin(), Meteo.size(), std::vector<MeteoData>()); //allocation for the vectors
for(; d1<=d2; d1+=Tstep) { //time loop
io.getMeteoData(d1, Meteo); //read 1 timestep at once, forcing resampling to the timestep
for(size_t ii=0; ii<Meteo.size(); ii++) {
vecMeteo.at(ii).push_back(Meteo[ii]); //fill the data manually into the vector of vectors
const std::string stationID( Meteo[ii].meta.stationID );
if (mapIDs.count( stationID )==0) { //if this is the first time we encounter this station, save where it should be inserted
mapIDs[ stationID ] = ii;
}
vecMeteo[ mapIDs[stationID] ].push_back(Meteo[ii]); //fill the data manually into the vector of vectors
}
}
......
......@@ -340,7 +340,7 @@ void ImisIO::readStationIDs(std::vector<std::string>& vecStationID)
}
void ImisIO::readMeteoData(const Date& dateStart, const Date& dateEnd,
std::vector< std::vector<MeteoData> >& vecMeteo, const size_t& stationindex)
std::vector< std::vector<MeteoData> >& vecMeteo, const size_t& /*stationindex*/)
{
Environment *env = NULL;
Connection *conn = NULL;
......@@ -360,17 +360,8 @@ void ImisIO::readMeteoData(const Date& dateStart, const Date& dateEnd,
size_t indexStart=0, indexEnd=vecStationMetaData.size();
//The following part decides whether all the stations are rebuffered or just one station
if (stationindex == IOUtils::npos) {
vecMeteo.clear();
vecMeteo.insert(vecMeteo.begin(), vecStationMetaData.size(), vector<MeteoData>());
} else {
if (stationindex < vecMeteo.size()) {
indexStart = stationindex;
indexEnd = stationindex+1;
} else {
throw IndexOutOfBoundsException("You tried to access a stationindex in readMeteoData that is out of bounds", AT);
}
}
vecMeteo.clear();
vecMeteo.insert(vecMeteo.begin(), vecStationMetaData.size(), vector<MeteoData>());
if ((env == NULL) || (conn == NULL))
openDBConnection(env, conn);
......
......@@ -396,19 +396,20 @@ void SMETIO::readMeteoData(const Date& dateStart, const Date& dateEnd,
void SMETIO::writeMeteoData(const std::vector< std::vector<MeteoData> >& vecMeteo, const std::string&)
{
//Loop through all stations
for (size_t ii=0; ii<vecMeteo.size(); ii++){
for (size_t ii=0; ii<vecMeteo.size(); ii++) {
if (vecMeteo[ii].empty()) continue; //this station does not have any data in this vecMeteo
//1. check consistency of station data position -> write location in header or data section
StationData sd;
sd.position.setProj(coordout, coordoutparam);
const bool isConsistent = checkConsistency(vecMeteo.at(ii), sd);
if (sd.stationID.empty()){
if (sd.stationID.empty()) {
ostringstream ss;
ss << "Station" << ii+1;
sd.stationID = ss.str();
}
const string filename = outpath + "/" + sd.stationID + ".smet";
const string filename( outpath + "/" + sd.stationID + ".smet" );
if (!FileUtils::validFileAndPath(filename)) //Check whether filename is valid
throw InvalidNameException(filename, AT);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment