WSL/SLF GitLab Repository

Commit f55578e6 authored by Mathias Bavay's avatar Mathias Bavay
Browse files

The COSMOXMLIO documentation has been updated and the plugin can now read all...

The COSMOXMLIO documentation has been updated and the plugin can now read all files in a given directory (but they must follow a loose naming scheme). Otherwise, small fixes (documentation typo, constification).
parent 537cb767
......@@ -115,7 +115,7 @@ namespace IOUtils {
* The matching is very primitive: it only looks for the substring "pattern" in the file names.
* If this substrings exists, the file matches.
* @param path directory containing the files
* @param dirlist list of mathcing file names
* @param dirlist list of matching file names
* @param pattern optional pattern that must be part of the file names
*/
void readDirectory(const std::string& path, std::list<std::string>& dirlist, const std::string& pattern="");
......
......@@ -39,11 +39,24 @@ namespace mio {
/**
* @page cosmoxml COSMOXML
* @section cosmoxml_format Format
* This plugin reads the XML files as generated by the Cosmo system.
* The files are outputted in Grib format and preprocessed by FieldExtra (MeteoSwiss)
* to get XML files.
* It requires libxml to compile and run.
* When writing files, it creates one file per station nammed as {stationID}_{numerical date}.xml
* This plugin reads the XML files as generated by <A HREF="http://www.cosmo-model.org/">Cosmo</A>'s <A HREF="http://www.cosmo-model.org/content/support/software/default.htm#fieldextra">FieldExtra</A>.
* The files are written out by COSMO in Grib format and preprocessed by FieldExtra (MeteoSwiss) to get XML files.
* It requires <A HREF="http://xmlsoft.org/">libxml2</A> to compile and run.
*
* @section cosmo_partners COSMO Group
* This plugin has been developed primarily for reading XML files produced by COSMO (http://www.cosmo-model.org/) at MeteoSwiss.
* COSMO (COnsortium for Small scale MOdelling) represents a non-hydrostatic limited-area atmospheric model, to be used both for operational and for research applications by the members of the consortium. The Consortium has the following members:
* - Germany, DWD, Deutscher Wetterdienst
* - Switzerland, MCH, MeteoSchweiz
* - Italy, USAM, Ufficio Generale Spazio Aereo e Meteorologia
* - Greece, HNMS, Hellenic National Meteorological Service
* - Poland, IMGW, Institute of Meteorology and Water Management
* - Romania, NMA, National Meteorological Administration
* - Russia, RHM, Federal Service for Hydrometeorology and Environmental Monitoring
* - Germany, AGeoBw, Amt für GeoInformationswesen der Bundeswehr
* - Italy, CIRA, Centro Italiano Ricerche Aerospaziali
* - Italy, ARPA-SIMC, ARPA Emilia Romagna Servizio Idro Meteo Clima
* - Italy, ARPA Piemonte, Agenzia Regionale per la Protezione Ambientale Piemonte
*
* @section cosmoxml_units Units
* The units are assumed to be the following:
......@@ -60,9 +73,13 @@ namespace mio {
* - COORDSYS: input coordinate system (see Coords) specified in the [Input] section
* - METEO: specify COSMOXML for [Input] section
* - METEOPATH: string containing the path to the xml files to be read, specified in the [Input] section
* - METEOFILE: specify the xml file to read the data from
* - METEOFILE: specify the xml file to read the data from (optional)
* - STATION#: ID of the station to read
*
* If no METEOFILE is provided, all ".xml" files in the METEOPATH directory will be read. They <i>must</i> contain the date of
* the first data formatted as ISO8601 numerical UTC date in their file name. For example, a file containing simulated
* meteorological fields from 2014-03-03T12:00 until 2014-03-05T00:00 could be named such as "cosmo_201403031200.xml"
*
* Example:
* @code
* [Input]
......@@ -80,39 +97,50 @@ const double CosmoXMLIO::out_tz = 0.; //Plugin specific time zone
const std::string CosmoXMLIO::xml_namespace = "http://www.meteoswiss.ch/xmlns/modeltemplate/2";
const std::string CosmoXMLIO::StationData_xpath = "//ch:datainformation/ch:data-tables/ch:data/ch:row/ch:col";
const std::string CosmoXMLIO::MeteoData_xpath = "//ch:valueinformation/ch:values-tables/ch:data/ch:row/ch:col";
const std::string CosmoXMLIO::meteo_ext = "xml";
CosmoXMLIO::CosmoXMLIO(const std::string& configfile)
: xml_stations_id(), input_id(), meteofile(), plugin_nodata(-999.), in_doc(NULL), in_xpathCtx(NULL),
: cache_meteo_files(), xml_stations_id(), input_id(), plugin_nodata(-999.),
in_doc(NULL), in_xpathCtx(NULL),
coordin(), coordinparam(), coordout(), coordoutparam()
{
LIBXML_TEST_VERSION
Config cfg(configfile);
IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
cfg.getValues("STATION", "INPUT", input_id);
cfg.getValue("METEOFILE", "INPUT", meteofile);
std::string meteopath;
cfg.getValue("METEOPATH", "INPUT", meteopath);
meteofile = meteopath + "/" + meteofile;
init(cfg);
}
CosmoXMLIO::CosmoXMLIO(const Config& cfg)
: xml_stations_id(), input_id(), meteofile(), plugin_nodata(-999.), in_doc(NULL), in_xpathCtx(NULL),
: cache_meteo_files(), xml_stations_id(), input_id(), plugin_nodata(-999.),
in_doc(NULL), in_xpathCtx(NULL),
coordin(), coordinparam(), coordout(), coordoutparam()
{
init(cfg);
}
void CosmoXMLIO::init(const Config& cfg)
{
LIBXML_TEST_VERSION
IOUtils::getProjectionParameters(cfg, coordin, coordinparam, coordout, coordoutparam);
cfg.getValues("STATION", "INPUT", input_id);
cfg.getValue("METEOFILE", "INPUT", meteofile);
std::string meteopath;
cfg.getValue("METEOPATH", "INPUT", meteopath);
meteofile = meteopath + "/" + meteofile;
std::string meteofile;
cfg.getValue("METEOFILE", "INPUT", meteofile, IOUtils::nothrow);
if(!meteofile.empty()) {
meteofile = meteopath + "/" + meteofile;
const std::pair<Date,std::string> tmp(Date(), meteofile);
cache_meteo_files.push_back( tmp );
} else {
scanMeteoPath(meteopath, cache_meteo_files);
}
}
CosmoXMLIO& CosmoXMLIO::operator=(const CosmoXMLIO& source) {
if(this != &source) {
cache_meteo_files = source.cache_meteo_files;
xml_stations_id = source.xml_stations_id;
input_id = source.input_id;
meteofile = source.meteofile;
plugin_nodata = source.plugin_nodata;
in_doc = NULL;
in_xpathCtx = NULL;
......@@ -129,6 +157,27 @@ CosmoXMLIO::~CosmoXMLIO() throw()
closeIn_XML();
}
void CosmoXMLIO::scanMeteoPath(const std::string& meteopath_in, std::vector< std::pair<Date,std::string> > &meteo_files)
{
meteo_files.clear();
std::list<std::string> dirlist;
IOUtils::readDirectory(meteopath_in, dirlist, meteo_ext);
dirlist.sort();
//Check date in every filename and cache it
std::list<std::string>::const_iterator it = dirlist.begin();
while ((it != dirlist.end())) {
const std::string& filename = *it;
const std::string::size_type spos = filename.find_first_of("0123456789");
Date date;
IOUtils::convertString(date, filename.substr(spos,10), in_tz);
const std::pair<Date,std::string> tmp(date, filename);
meteo_files.push_back(tmp);
it++;
}
}
void CosmoXMLIO::openIn_XML(const std::string& in_meteofile)
{
if(in_doc!=NULL) return; //the file has already been read
......@@ -249,9 +298,35 @@ bool CosmoXMLIO::parseStationData(const std::string& station_id, const xmlXPathC
return true;
}
void CosmoXMLIO::readStationData(const Date& /*station_date*/, std::vector<StationData>& vecStation)
size_t CosmoXMLIO::getFileIdx(const Date& start_date) const
{
if(cache_meteo_files.empty())
throw InvalidArgumentException("No input files found or configured!", AT);
//find which file we should open
if(cache_meteo_files.size()==1) {
return 0;
} else {
size_t idx;
for(idx=1; idx<cache_meteo_files.size(); idx++) {
if(start_date>=cache_meteo_files[idx-1].first && start_date<cache_meteo_files[idx].first) {
return idx--;
}
}
//not found, we take the closest timestamp we have
if(start_date<cache_meteo_files.front().first)
return 0;
else
return cache_meteo_files.size()-1;
}
}
void CosmoXMLIO::readStationData(const Date& station_date, std::vector<StationData>& vecStation)
{
vecStation.clear();
const std::string meteofile( cache_meteo_files[ getFileIdx(station_date) ].second );
openIn_XML(meteofile);
//read all the stations' metadata
......@@ -264,10 +339,10 @@ void CosmoXMLIO::readStationData(const Date& /*station_date*/, std::vector<Stati
vecStation.push_back(sd);
}
//closeIn_XML(); //let the destructor close it when appropriate
closeIn_XML();
}
bool CosmoXMLIO::parseMeteoDataPoint(const Date& dateStart, const Date& dateEnd, const xmlNodePtr &element, MeteoData &md) const
CosmoXMLIO::MeteoReadStatus CosmoXMLIO::parseMeteoDataPoint(const Date& dateStart, const Date& dateEnd, const xmlNodePtr &element, MeteoData &md) const
{
const std::string attribute = "id";
double iswr_dir = IOUtils::nodata, iswr_diff = IOUtils::nodata;
......@@ -283,7 +358,8 @@ bool CosmoXMLIO::parseMeteoDataPoint(const Date& dateStart, const Date& dateEnd,
const std::string value( (char*)(cur_node->children->content) );
if(field=="reference_ts") {
IOUtils::convertString(md.date, value, in_tz);
if(md.date<dateStart || md.date>dateEnd) return false; //HACK return for real if >dateEnd
if(md.date<dateStart) return read_continue;
if(md.date>dateEnd) return read_stop;
} else {
double tmp;
IOUtils::convertString(tmp, value);
......@@ -310,7 +386,7 @@ bool CosmoXMLIO::parseMeteoDataPoint(const Date& dateStart, const Date& dateEnd,
if(iswr_diff!=IOUtils::nodata && iswr_dir!=IOUtils::nodata)
md(MeteoData::ISWR) = iswr_diff+iswr_dir;
return true;
return read_ok;
}
bool CosmoXMLIO::parseMeteoData(const Date& dateStart, const Date& dateEnd, const std::string& station_id, const StationData& sd, const xmlXPathContextPtr& xpathCtx, std::vector<MeteoData> &vecMeteo) const
......@@ -330,8 +406,9 @@ bool CosmoXMLIO::parseMeteoData(const Date& dateStart, const Date& dateEnd, cons
for(int ii=0; ii<nr_data; ii++) {
MeteoData md( Date(), sd);
if(parseMeteoDataPoint(dateStart, dateEnd, data->nodeTab[ii], md)) //add to vector if this is a valid point
vecMeteo.push_back( md );
const MeteoReadStatus status = parseMeteoDataPoint(dateStart, dateEnd, data->nodeTab[ii], md);
if(status==read_ok) vecMeteo.push_back( md );
if(status==read_stop) break;
}
xmlXPathFreeObject(xpathObj);
......@@ -343,23 +420,44 @@ void CosmoXMLIO::readMeteoData(const Date& dateStart, const Date& dateEnd,
const size_t&)
{
vecMeteo.clear();
openIn_XML(meteofile);
std::vector<StationData> sd;
readStationData(dateStart, sd);
const size_t nr_files = cache_meteo_files.size();
size_t file_idx = getFileIdx(dateStart);
Date nextDate;
do {
//since files contain overlapping data, we will only read the non-overlapping part
//ie from start to the start date of the next file
nextDate = ((file_idx+1)<nr_files)? cache_meteo_files[file_idx+1].first : dateEnd;
const std::string meteofile( cache_meteo_files[file_idx].second );
openIn_XML(meteofile);
//read all the stations' metadata
std::vector<StationData> vecStation;
for(size_t ii=0; ii<input_id.size(); ii++) {
StationData sd;
if(!parseStationData(input_id[ii], in_xpathCtx, sd)) {
closeIn_XML();
throw IOException("Unable to evaluate xpath expression \""+input_id[ii]+"\"", AT);
}
vecStation.push_back(sd);
}
//read all the stations' data
for(size_t ii=0; ii<input_id.size(); ii++) {
const string station_id = xml_stations_id[ input_id[ii] ];
vector<MeteoData> vecTmp;
if(!parseMeteoData(dateStart, dateEnd, station_id, sd[ii], in_xpathCtx, vecTmp)) {
closeIn_XML();
throw IOException("Unable to evaluate xpath expression \""+input_id[ii]+"\"", AT);
//read all the stations' data
for(size_t ii=0; ii<input_id.size(); ii++) {
const string station_id = xml_stations_id[ input_id[ii] ];
vector<MeteoData> vecTmp;
if(!parseMeteoData(dateStart, nextDate, station_id, vecStation[ii], in_xpathCtx, vecTmp)) {
closeIn_XML();
throw IOException("Unable to evaluate xpath expression \""+input_id[ii]+"\"", AT);
}
vecMeteo.push_back(vecTmp);
}
vecMeteo.push_back(vecTmp);
}
closeIn_XML();
closeIn_XML();
file_idx++;
} while (file_idx<nr_files && nextDate<=dateEnd);
}
void CosmoXMLIO::writeMeteoData(const std::vector< std::vector<MeteoData> >& /*vecMeteo*/,
......
......@@ -68,17 +68,22 @@ class CosmoXMLIO : public IOInterface {
virtual void write2DGrid(const Grid2DObject& grid_in, const MeteoGrids::Parameters& parameter, const Date& date);
private:
typedef enum METEOREADSTATUS { read_ok, read_continue, read_stop } MeteoReadStatus;
void init(const Config& cfg);
static void scanMeteoPath(const std::string& meteopath_in, std::vector< std::pair<Date,std::string> > &meteo_files);
size_t getFileIdx(const Date& start_date) const;
void openIn_XML(const std::string& in_meteofile);
void closeIn_XML() throw();
bool parseStationData(const std::string& station_id, const xmlXPathContextPtr& xpathCtx, StationData &sd);
bool parseMeteoDataPoint(const Date& dateStart, const Date& dateEnd, const xmlNodePtr &element, MeteoData &md) const;
MeteoReadStatus parseMeteoDataPoint(const Date& dateStart, const Date& dateEnd, const xmlNodePtr &element, MeteoData &md) const;
bool parseMeteoData(const Date& dateStart, const Date& dateEnd, const std::string& station_id,
const StationData& sd, const xmlXPathContextPtr& xpathCtx, std::vector<MeteoData> &vecMeteo) const;
std::vector< std::pair<Date,std::string> > cache_meteo_files; //cache of meteo files in METEOPATH
std::map<std::string, std::string> xml_stations_id; //mapping between true station ID and the messy id used in the xml
std::vector<std::string> input_id; //user specified stations to read
std::string meteofile; //file containing all the data, for all the stations
double plugin_nodata; //plugin specific no data value
xmlDocPtr in_doc;
......@@ -86,6 +91,7 @@ class CosmoXMLIO : public IOInterface {
static const double in_tz, out_tz; //plugin specific time zones
static const std::string xml_namespace, StationData_xpath, MeteoData_xpath;
static const std::string meteo_ext;
std::string coordin, coordinparam, coordout, coordoutparam; //projection parameters
};
......
......@@ -719,13 +719,13 @@ void GRIBIO::scanMeteoPath()
dirlist.sort();
//Check date in every filename and cache it
std::list<std::string>::iterator it = dirlist.begin();
std::list<std::string>::const_iterator it = dirlist.begin();
while ((it != dirlist.end())) {
const std::string& filename = *it;
std::string::size_type spos = filename.find_first_of("0123456789");
const std::string::size_type spos = filename.find_first_of("0123456789");
Date date;
IOUtils::convertString(date, filename.substr(spos,10), tz_in);
std::pair<Date,std::string> tmp(date, filename);
const std::pair<Date,std::string> tmp(date, filename);
cache_meteo_files.push_back(tmp);
it++;
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment