WSL/SLF GitLab Repository

Commit 4fca1874 authored by Mathias Bavay's avatar Mathias Bavay
Browse files

More robust memory allocation in data_converter (as well as simpler), better...

More robust memory allocation in data_converter (as well as simpler), better handling of larger header sections in smet_range and improved error messages in IOUtils and IOHandler
parent 349e9d43
......@@ -32,15 +32,14 @@ void real_main(int argc, char** argv) {
std::map<std::string, size_t> mapIDs; //over a large time range, the number of stations might change... this is the way to make it work
std::vector<MeteoData> Meteo; //we need some intermediate storage, for storing data sets for 1 timestep
io.getMeteoData(d1, Meteo); //we need to know how many stations will be available
vecMeteo.insert(vecMeteo.begin(), Meteo.size(), std::vector<MeteoData>()); //allocation for the vectors
for(; d1<=d2; d1+=Tstep) { //time loop
io.getMeteoData(d1, Meteo); //read 1 timestep at once, forcing resampling to the timestep
for(size_t ii=0; ii<Meteo.size(); ii++) {
const std::string stationID( Meteo[ii].meta.stationID );
if (mapIDs.count( stationID )==0) { //if this is the first time we encounter this station, save where it should be inserted
mapIDs[ stationID ] = ii;
if (ii>=vecMeteo.size()) vecMeteo.push_back( std::vector<MeteoData>() ); //allocating the new station
}
vecMeteo[ mapIDs[stationID] ].push_back(Meteo[ii]); //fill the data manually into the vector of vectors
}
......
......@@ -572,7 +572,7 @@ void IOHandler::merge_stations(STATIONS_SET& vecStation) const
}
}
if (!found)
throw InvalidArgumentException("Station ID '"+fromStationID+"' not found when merging toward station '"+toStationID+"'", AT);
throw InvalidArgumentException("Station ID '"+fromStationID+"' not found when merging toward station '"+toStationID+"'. Consider increasing BUFF_CHUNK_SIZE!", AT);
}
}
......
......@@ -481,9 +481,10 @@ bool convertString(Date& t, const std::string& str, const double& time_zone, std
return true;
} else {
const size_t wrong_dash = s.find("–");
if (wrong_dash!=std::string::npos)
throw InvalidFormatException("Invalid date '"+s+"', replace the '–' character by '-'", AT);
const size_t wrong_dash1 = s.find("–");
const size_t wrong_dash2 = s.find("Ð");
if (wrong_dash1!=std::string::npos || wrong_dash2!=std::string::npos)
throw InvalidFormatException("Invalid date '"+s+"', please use the '-' character as date delimiter", AT);
//try to read purely numerical date, potentially surrounded by other chars
//and potentially containing an ISO time zone string
......
......@@ -21,12 +21,12 @@ files=`find ${INPUT_DIR}/* -maxdepth 0 -type f -name "*.smet"`
if [ "${param}" = "time" ]; then
for SMET in ${files}; do
NAME=`basename "${SMET}" .smet`
ALT=`head -25 "${SMET}" | grep altitude | tr -s '\t' ' ' | tr -s ' ' | cut -d' ' -f3 | cut -d'.' -f1`
JULIAN=`head -25 "${SMET}" | grep fields | grep julian`
ISO=`head -25 "${SMET}" | grep fields | grep timestamp`
start=`head -25 "${SMET}" | grep -E "^[0-9][0-9][0-9][0-9]" | head -1 | tr -s '\t' ' ' | tr -s ' ' | cut -d' ' -f1`
ALT=`head -100 "${SMET}" | grep altitude | tr -s '\t' ' ' | tr -s ' ' | cut -d' ' -f3 | cut -d'.' -f1`
JULIAN=`head -100 "${SMET}" | grep fields | grep julian`
ISO=`head -100 "${SMET}" | grep fields | grep timestamp`
start=`head -100 "${SMET}" | grep -E "^[0-9][0-9][0-9][0-9]" | head -1 | tr -s '\t' ' ' | tr -s ' ' | cut -d' ' -f1`
end=`tail -5 "${SMET}" | grep -E "^[0-9][0-9][0-9][0-9]" | tail -1 | tr -s '\t' ' ' | tr -s ' ' | cut -d' ' -f1`
header_nr_lines=`head -25 "${SMET}" | grep -n "\[DATA\]" | cut -d':' -f1`
header_nr_lines=`head -100 "${SMET}" | grep -n "\[DATA\]" | cut -d':' -f1`
full_nr_lines=`wc -l "${SMET}" | cut -d' ' -f1`
nr_lines=`expr ${full_nr_lines} - ${header_nr_lines}`
......@@ -65,15 +65,15 @@ if [ "${param}" = "time" ]; then
fi
for SMET in ${files}; do
IJ=`head -15 ${SMET} | grep station_id | tr -s ' \t' | cut -d' ' -f3`
IJ=`head -100 ${SMET} | grep station_id | tr -s ' \t' | cut -d' ' -f3`
if [ -z "${IJ}" ]; then
IJ=`echo ${SMET} | cut -d'.' -f 1 | cut -d'_' -f2,3 | tr "_" ","`
fi
LAT=`head -15 ${SMET} | grep latitude | tr -s ' \t' | cut -d' ' -f3`
LON=`head -15 ${SMET} | grep longitude | tr -s ' \t' | cut -d' ' -f3`
ALT=`head -15 ${SMET} | grep altitude | tr -s ' \t' | cut -d' ' -f3 | cut -d'.' -f1`
NODATA=`head -15 ${SMET} | grep nodata | tr -s ' \t' | cut -d' ' -f3`
JULIAN=`head -25 "${SMET}" | grep fields | grep julian`
LAT=`head -100 ${SMET} | grep latitude | tr -s ' \t' | cut -d' ' -f3`
LON=`head -100 ${SMET} | grep longitude | tr -s ' \t' | cut -d' ' -f3`
ALT=`head -100 ${SMET} | grep altitude | tr -s ' \t' | cut -d' ' -f3 | cut -d'.' -f1`
NODATA=`head -100 ${SMET} | grep nodata | tr -s ' \t' | cut -d' ' -f3`
JULIAN=`head -100 "${SMET}" | grep fields | grep julian`
awk '
function toJul(ts){
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment