WSL/SLF GitLab Repository

Commit 5b52f4c6 authored by Mathias Bavay's avatar Mathias Bavay
Browse files

A OSHD_DEBUG key has been added and the units are now properly supported when...

A OSHD_DEBUG key has been added and the units are now properly supported when using commit https://github.com/tbeu/matio/commit/12b7d4051369716ef21a53540d1f3354fe0d4e97 of libmatio (the UTF-16 is now returned as ISO-8859-1). 
parent c76f55a5
......@@ -33,6 +33,11 @@ namespace mio {
* library to read this file format: the Open Source <A HREF="https://sourceforge.net/projects/matio/">MatIO</A> library. This can be installed directly from
* the default repositories under Linux or installed by downloading the proper package for Windows or OsX.
*
* \note If non-ascii characters have been used and the file has been created under Windows, some of the strings might end up using the UTF-16 encoding.
* This requires a recent version of libmatio (see <A HREF="https://github.com/tbeu/matio/issues/34">this issue</A>). Another option would be to
* add at the begining of the Matlab routine a call to *feature('DefaultCharacterSet', 'UTF8')* in order to switch from the current default (which can be read by the same call,
* ommitting the 'UTF8' option) to the (<A HREF="http://blog.omega-prime.co.uk/?p=150">partial</A>) UTF-8 encoding of Matlab.
*
* @section oshd_data_structure Data structure
* The files are named with the following schema: <i>{parameter}_{timestep}_{cosmo model version}run{run time}.mat</i> with the following possible values:
* + *parameter* is one of idif, idir, ilwr, pair, prec, rhum, tair, tswe, wcor, wdir, wind;
......@@ -70,6 +75,7 @@ namespace mio {
* - METEOPATH: directory containing all the data files with the proper file naming schema; [Input] section
* - STATION#: input stations' IDs (in METEOPATH). As many meteofiles as needed may be specified
* - METAFILE: file within METEOPATH containing the stations' IDs, names and location; [Input] section
* - OSHD_DEBUG: write out extra information to better show what is in the files
*
* @section oshd_example Example use
* @code
......@@ -91,14 +97,16 @@ void listFields(matvar_t *matvar)
{
const unsigned int nrFields = Mat_VarGetNumberOfFields(matvar);
char * const *fields = Mat_VarGetStructFieldnames(matvar);
for (unsigned int ii=0; ii<nrFields; ii++) printf("field[%d] = %s\n", ii, fields[ii]);
for (unsigned int ii=0; ii<nrFields; ii++)
printf("field[%d] = %s\n", ii, fields[ii]);
}
void printStructure(matvar_t *matvar)
{
//Mat_VarPrint(field, 0);
printf("name=%s class_type=%d data_type=%d rank=%d", matvar->name, matvar->class_type, matvar->data_type, matvar->rank);
for (int ii=0; ii<matvar->rank; ii++) printf("\tdims[%d]=%d", ii, (int)matvar->dims[ii]);
for (int ii=0; ii<matvar->rank; ii++)
printf("\tdims[%d]=%d", ii, (int)matvar->dims[ii]);
printf("\n");
}
......@@ -110,8 +118,7 @@ std::string readString(const std::string &filename, const std::string &fieldname
throw InvalidFormatException("could not read field '"+fieldname+"' in file '"+filename+"'", AT);
if (field->class_type!=MAT_C_CHAR) throw InvalidFormatException("field '"+fieldname+"' in file '"+filename+"' is not a type string", AT);
const std::string tmp( static_cast<char*>(field->data) );
return tmp;
return std::string( static_cast<char*>(field->data) );
}
void readStringVector(const std::string &filename, const std::string &fieldname, mat_t *matfp, matvar_t *matvar, std::vector<std::string> &vecString)
......@@ -159,7 +166,7 @@ void readDoubleVector(const std::string &filename, const std::string &fieldname,
const size_t ncols = field->dims[1];
if (nrows!=1) throw InvalidFormatException("invalid nrows for field '"+fieldname+"' in file '"+filename+"'", AT);
const double* matData = static_cast<double*>( field->data );
const double* matData( static_cast<double*>( field->data ) );
vecData.resize( ncols );
for (size_t ii=0; ii<ncols; ii++) {
vecData[ii] = matData[ii];
......@@ -174,19 +181,20 @@ const double OshdIO::plugin_nodata = -999.; //plugin specific nodata value. It c
const double OshdIO::in_dflt_TZ = 0.; //COSMO data is always GMT
OshdIO::OshdIO(const std::string& configfile) : cfg(configfile), cache_meteo_files(), vecMeta(), vecIDs(), params_map(), vecIdx(),
in_meteopath(), in_metafile()
in_meteopath(), in_metafile(), debug(false)
{
parseInputOutputSection();
}
OshdIO::OshdIO(const Config& cfgreader) : cfg(cfgreader), cache_meteo_files(), vecMeta(), vecIDs(), params_map(), vecIdx(),
in_meteopath(), in_metafile()
in_meteopath(), in_metafile(), debug(false)
{
parseInputOutputSection();
}
void OshdIO::parseInputOutputSection()
{
cfg.getValue("OSHD_DEBUG", "INPUT", debug, IOUtils::nothrow);
//IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
cfg.getValue("METAFILE", "INPUT", in_metafile);
......@@ -268,10 +276,10 @@ void OshdIO::readMeteoData(const Date& dateStart, const Date& dateEnd,
std::vector< std::vector<MeteoData> >& vecMeteo,
const size_t&)
{
const size_t nr_files = cache_meteo_files.size();
const size_t nrIDs = vecIDs.size();
vecMeteo.clear();
const size_t nr_files = cache_meteo_files.size();
size_t file_idx = getFileIdx( dateStart );
Date station_date( cache_meteo_files[file_idx].first );
if (station_date<dateStart || station_date>dateEnd) return; //the requested period is NOT in the available files
......@@ -290,7 +298,7 @@ void OshdIO::readMeteoData(const Date& dateStart, const Date& dateEnd,
std::vector<double> vecData;
for (size_t ii=0; ii<params_map.size(); ii++) {
const MeteoData::Parameters param( params_map[ii].first );
const std::string prefix(params_map[ii].second );
const std::string prefix( params_map[ii].second );
const std::string filename = in_meteopath + "/" + prefix + "_" + file_suffix;
vecData.resize( nrIDs, IOUtils::nodata );
readFromFile(filename, param, station_date, vecData);
......@@ -304,7 +312,6 @@ void OshdIO::readMeteoData(const Date& dateStart, const Date& dateEnd,
file_idx++;
station_date = ((file_idx)<nr_files)? cache_meteo_files[file_idx].first : dateEnd+1.;
} while (file_idx<nr_files && station_date<=dateEnd);
}
void OshdIO::readSWRad(const Date& station_date, const std::string& file_suffix, const size_t& nrIDs, std::vector< std::vector<MeteoData> >& vecMeteo) const
......@@ -333,7 +340,7 @@ void OshdIO::readFromFile(const std::string& filename, const MeteoData::Paramete
matvar_t *matvar = Mat_VarReadInfo(matfp, "stat");
if (matvar==NULL) throw NotFoundException("structure 'stat' not found in file '"+filename+"'", AT);
if (matvar->class_type!=MAT_C_STRUCT) throw InvalidFormatException("The matlab file should contain 1 structure", AT);
const std::string units( readString(filename, "dunit", matfp, matvar) );
const std::string type( readString(filename, "type", matfp, matvar) );
checkFieldType(param, type);
......@@ -354,6 +361,7 @@ void OshdIO::readFromFile(const std::string& filename, const MeteoData::Paramete
}
//extract the data for the selected stations
const std::string units( readString(filename, "dunit", matfp, matvar) );
std::vector<double> vecRaw;
readDoubleVector(filename, "data", matfp, matvar, vecRaw);
if (vecAcro.size() != vecRaw.size()) throw InvalidFormatException("'acro' and 'data' arrays don't match in file '"+filename+"'", AT);
......@@ -382,25 +390,26 @@ double OshdIO::convertUnits(const double& val, const std::string& units, const M
{
if (units=="%") return val/100.;
if (units=="cm") return val/100.;
if (units=="°C") return val+Cst::t_water_freezing_pt;
if (units=="°") return val;
if (units.empty()) return val;
if (units=="Pa") return val;
if (units=="mm") {
if (param==MeteoData::PSUM) return val;
else return val/1000.;
}
if (units=="W/m2") return val;
if (units=="m/s") return val;
else
throw IOException("Unknown units '"+units+"'", AT);
if (units=="\xB0\x43") return val+Cst::t_water_freezing_pt; //ISO-8859-1 hex for '°C'
// //usually skip these extra tests
// if (units=="\xB0") return val; //ISO-8859-1 hex for '°'
// if (units.empty()) return val;
// if (units=="Pa") return val;
// if (units=="W/m2") return val;
// if (units=="m/s") return val;
// else
// throw IOException("Unknown units '"+units+"'", AT);
return val;
}
void OshdIO::fillStationMeta()
{
vecMeta.resize( vecIDs.size() );
vecMeta.resize( vecIDs.size(), StationData() );
const std::string filename( in_meteopath+"/"+in_metafile );
mat_t *matfp = Mat_Open(filename.c_str(), MAT_ACC_RDONLY);
if ( NULL == matfp ) throw AccessException(filename, AT);
......@@ -422,6 +431,12 @@ void OshdIO::fillStationMeta()
Mat_VarFree(matvar);
Mat_Close(matfp);
if (debug) {
for (size_t ii=0; ii<vecAcro.size(); ii++)
std::cout << std::setw(8) << vecAcro[ii] << std::setw(40) << vecNames[ii] << std::setw(8) << easting[ii] << std::setw(8) << northing[ii] << std::setw(8) << altitude[ii] << "\n";
std::cout << endl;
}
buildVecIdx(vecAcro);
for (size_t ii=0; ii<vecIdx.size(); ii++) {
Coords position("CH1903", "");
......@@ -440,17 +455,16 @@ void OshdIO::buildVecIdx(const std::vector<std::string>& vecAcro)
vecIdx.resize( nrIDs, 0 );
for (size_t ii=0; ii<nrIDs; ii++) {
const std::string stationID( vecIDs[ii] );
bool found = false;
for (size_t jj=0; jj<vecAcro.size(); jj++) {
if (stationID==vecAcro[jj]) {
if (vecIDs[ii]==vecAcro[jj]) {
vecIdx[ii] = jj;
found = true;
break;
}
}
if (!found)
throw NotFoundException("station ID '"+stationID+"' could not be found in the provided data", AT);
throw NotFoundException("station ID '"+vecIDs[ii]+"' could not be found in the provided data", AT);
}
}
......
......@@ -62,8 +62,9 @@ class OshdIO : public IOInterface {
std::vector<StationData> vecMeta;
std::vector<std::string> vecIDs; ///< IDs of the stations that have to be read
std::vector< std::pair<MeteoData::Parameters, std::string> > params_map; ///< parameters to extract from the files
std::vector<size_t> vecIdx; ///< index of each ID that should be read within the 'acro' vector
std::vector<size_t> vecIdx; ///< index of each ID that should be read within the 'acro', 'names' and 'data' vectors
std::string in_meteopath, in_metafile;
bool debug; ///< write out extra information to help understand what is being read
//std::string coordin, coordinparam, coordout, coordoutparam; //projection parameters
static const char* meteo_ext; //for the file naming scheme
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment