libsmet.cc

/***********************************************************************************/
/*  Copyright 2009 WSL Institute for Snow and Avalanche Research    SLF-DAVOS      */
/***********************************************************************************/
/* This file is part of MeteoIO.
    MeteoIO is free software: you can redistribute it and/or modify
    it under the terms of the GNU Lesser General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    MeteoIO is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public License
    along with MeteoIO.  If not, see <http://www.gnu.org/licenses/>.
*/
#include <meteoio/plugins/libsmet.h>
#include <errno.h>
#include <string.h>
#include <limits>

using namespace std;
using namespace smet; //HACK POPC

namespace smet {

const std::string SMETCommon::smet_version = "1.1";
set<string> SMETCommon::all_mandatory_header_keys = set<std::string>();
set<string> SMETCommon::all_optional_header_keys  = set<std::string>();
set<string> SMETCommon::all_decimal_header_values = set<std::string>();

SMETException::SMETException(const std::string& message, const std::string& position) : msg()
{
	if (position.empty()) {
		msg = "At unknown position: " + message;
	} else {
		msg = position + ": " + message;
	}
}

SMETException::~SMETException() throw() {}

const char* SMETException::what() const throw()
{
	return msg.c_str();
}

const bool SMETCommon::__init = SMETCommon::initStaticData();
bool SMETCommon::initStaticData()
{
	all_mandatory_header_keys.insert("station_id");
	all_mandatory_header_keys.insert("nodata");
	all_mandatory_header_keys.insert("fields");

	all_optional_header_keys.insert("latitude");
	all_optional_header_keys.insert("longitude");
	all_optional_header_keys.insert("altitude");
	all_optional_header_keys.insert("easting");
	all_optional_header_keys.insert("northing");
	all_optional_header_keys.insert("epsg");

	all_optional_header_keys.insert("station_name");
	all_optional_header_keys.insert("tz");
	all_optional_header_keys.insert("creation");
	all_optional_header_keys.insert("source");
	all_optional_header_keys.insert("units_offset");
	all_optional_header_keys.insert("units_multiplier");
	all_optional_header_keys.insert("comment");

	all_decimal_header_values.insert("nodata");
	all_decimal_header_values.insert("latitude");
	all_decimal_header_values.insert("longitude");
	all_decimal_header_values.insert("altitude");
	all_decimal_header_values.insert("easting");
	all_decimal_header_values.insert("northing");
	all_decimal_header_values.insert("epsg");
	all_decimal_header_values.insert("tz");

	return true;
}

double SMETCommon::convert_to_double(const std::string& in_string)
{
	istringstream ss(in_string);
	double value;
	if (!(ss >> value)) throw SMETException("Value \"" + in_string + "\" cannot be converted to double", SMET_AT);

	return value;
}

int SMETCommon::convert_to_int(const std::string& in_string)
{
	istringstream ss(in_string);
	int value;
	if (!(ss >> value)) throw SMETException("Value \"" + in_string + "\" cannot be converted to int", SMET_AT);

	return value;
}

void SMETCommon::trim(std::string& str)
{
	const std::string whitespaces(" \t\f\v\n\r");
	const size_t startpos = str.find_first_not_of(whitespaces); // Find the first character position after excluding leading blank spaces
	const size_t endpos = str.find_last_not_of(whitespaces); // Find the first character position from reverse af

	// if all spaces or empty return an empty string
	if(( std::string::npos == startpos ) || ( std::string::npos == endpos)) {
		str.clear();
	} else {
		str.erase(endpos+1); //right trim
		str.erase(0, startpos); //left trim
	}
}

bool SMETCommon::readKeyValuePair(const std::string& in_line, const std::string& delimiter,
                                  std::map<std::string,std::string>& out_map)
{
	//size_t pos = in_line.find(delimiter); //first occurence of '='

	size_t pos = std::string::npos;
	if ((delimiter==" ") || (delimiter=="\t")) {
		pos = in_line.find_first_of(" \t", 0);
	} else {
		pos = in_line.find(delimiter); //first occurence of '='
	}


	if(pos != std::string::npos) { //ignore in_lines that are empty or without '='
		std::string key = in_line.substr(0, pos);
		std::string value = in_line.substr(pos + 1);

		SMETCommon::trim(key);
		SMETCommon::trim(value);

		if (key.empty() || value.empty()) {
			return false;
		}

		out_map[key] = value;
	} else {
		return false;
	}

	return true;
}

void SMETCommon::stripComments(std::string& str)
{
	const size_t found = str.find_first_of("#;");
	if (found != std::string::npos){
		str.erase(found); //rest of line disregarded
	}
}

void SMETCommon::toUpper(std::string& str)
{
	std::transform(str.begin(), str.end(),str.begin(), ::toupper);
}

char SMETCommon::getEoln(std::istream& fin)
{
	std::streambuf* pbuf;
	char tmp = '0';
	int chars = 0;

	const std::streampos position = fin.tellg();

	do {
		fin.get(tmp);
		chars++;

		if ((tmp == '\r') || (tmp == '\n')) {
			char peekc = tmp;
			while ((!fin.eof() && ((peekc=='\r') || (peekc=='\n')))) {
				tmp = peekc;
				fin.get(peekc);
				chars++;
			}
			pbuf = fin.rdbuf();
			pbuf->pubseekpos(position); //rewind
			fin.clear(); //reset eof flag, etc
			return tmp;
		}
	} while ((chars < 3000) && (!fin.eof()));

	pbuf = fin.rdbuf();
	pbuf->pubseekpos(position); //rewind
	fin.clear(); //reset eof flag, etc

	return '\n';
}

bool SMETCommon::is_decimal(const std::string& value)
{
	istringstream ss(value);
	double doublevalue;
	if (!(ss >> doublevalue)) return false;

	return true;
}

size_t SMETCommon::readLineToVec(const std::string& line_in, std::vector<std::string>& vec_string)
{
	vec_string.clear();
	std::istringstream iss(line_in); //construct inputstream with the string line as input

	std::string tmp_string;
	while (!iss.eof()) {
		iss >> std::skipws >> tmp_string;

		if (!tmp_string.empty()) {
			vec_string.push_back(tmp_string);
		}
		tmp_string.clear();
	}

	return vec_string.size();
}

SMETWriter::SMETWriter(const std::string& in_filename, const SMETType& in_type, const bool& in_gzip)
           : other_header_keys(), ascii_precision(), ascii_width(), header(), mandatory_header_keys(), fout(),
             filename(in_filename), nodata_string(), smet_type(in_type), nodata_value(-999.), nr_of_fields(0),
             julian_field(0), timestamp_field(0), location_wgs84(0), location_epsg(0), gzip(in_gzip),
             location_in_header(false), location_in_data_wgs84(false), location_in_data_epsg(false),
             timestamp_present(false), julian_present(false), file_is_binary(false)
{

}

SMETWriter::~SMETWriter()
{
	cleanup();
}

void SMETWriter::cleanup() throw()
{
	//clear ios flags
	fout << resetiosflags(ios_base::fixed | ios_base::left);

	if (fout.is_open()) //close fout if open
		fout.close();
}

void SMETWriter::set_header_value(const std::string& key, const double& value)
{
	//check if key is decimal, transform to string and add to header
	if (SMETCommon::all_decimal_header_values.find(key) != SMETCommon::all_decimal_header_values.end()){
		stringstream ss;
		if ((key == "latitude") || (key == "longitude") || (key == "easting") || (key == "northing")){
			ss << fixed << setprecision(6) << value;
		} else if (key == "altitude"){
			ss << fixed << setprecision(1) << value;
		} else if ((key == "epsg") || (key == "tz")){
			ss << fixed << setprecision(0) << value;
		} else {
			ss << value; //for nodata
		}

		set_header_value(key, ss.str());
	} else {
		//It's a non-standard header value
		stringstream ss;
		ss << value; //for nodata
		set_header_value(key, ss.str());
	}
}

void SMETWriter::set_header_value(const std::string& key, const std::string& value)
{
	//check if header key/value pair is valid
	if (valid_header_pair(key, value)){
		header[key] = value;

		if ((SMETCommon::all_optional_header_keys.find(key) == SMETCommon::all_optional_header_keys.end())
		    && (SMETCommon::all_mandatory_header_keys.find(key) == SMETCommon::all_mandatory_header_keys.end()))
			other_header_keys.push_back(key);
	} else {
		throw SMETException("Invalid, inconsistent or unknown key/value pair: " + key + " = " + value, SMET_AT);
	}
}

bool SMETWriter::valid_header_pair(const std::string& key, const std::string& value)
{
	bool key_ok = true;

	if (SMETCommon::all_mandatory_header_keys.find(key) != SMETCommon::all_mandatory_header_keys.end()){
		mandatory_header_keys.insert(key);
		key_ok = true;
	}

	//nodata value needs extra treatment
	if (key == "nodata"){
		istringstream ss(value);
		if (!(ss >> nodata_value)) return false;
		nodata_string = value;
	}

	//Using WGS_84 coordinate system
	if (key == "latitude")  location_wgs84 |= 1;
	else if (key == "longitude") location_wgs84 |= 2;
	else if (key == "altitude")  location_wgs84 |= 4;

	//Using an EPSG coordinate system
	if (key == "easting")  location_epsg |= 1;
	else if (key == "northing") location_epsg |= 2;
	else if (key == "altitude") location_epsg |= 4;
	else if (key == "epsg")     location_epsg |= 8;

	//Now do some value checks
	if (key == "epsg"){
		istringstream ss(value);
		int intvalue;
		if (!(ss >> intvalue)) return false;
	}

	if (SMETCommon::all_decimal_header_values.find(key) != SMETCommon::all_decimal_header_values.end()){
		//check if value is a double value
		if (!SMETCommon::is_decimal(value)) return false;
	}

	if ((location_epsg == 15) || (location_wgs84 == 7))
		location_in_header = true;

	//Rudimentary check on keys: fields, units_offset, units_multiplier
	if ((key == "fields") || (key == "units_offset") || (key == "units_multiplier"))
		key_ok = check_fields(key, value);

	return key_ok;
}

bool SMETWriter::check_fields(const std::string& key, const std::string& value)
{
	vector<string> tmp_vec;
	const size_t counter = SMETCommon::readLineToVec(value, tmp_vec);

	//Firstly: Check if number of fields is consistent
	if (nr_of_fields != 0){
		if (counter != nr_of_fields) return false;
	} else {
		nr_of_fields = counter;
	}

	if (key == "fields"){
        size_t count_wgs84 = 0, count_epsg = 0;

		//check if location is in data and if timestamp is present
		for (size_t ii = 0; ii<tmp_vec.size(); ii++){
			if (tmp_vec[ii] == "latitude") count_wgs84++;
			else if (tmp_vec[ii] == "longitude") count_wgs84++;
			else if (tmp_vec[ii] == "easting") count_epsg++;
			else if (tmp_vec[ii] == "northing") count_epsg++;

			if (tmp_vec[ii] == "altitude") {
				count_wgs84++;
				count_epsg++;
			}

			if (tmp_vec[ii] == "timestamp"){
				if (timestamp_present) return false; //no duplicate timestamp field allowed
				timestamp_present = true;
				timestamp_field = ii;
			}

			if (tmp_vec[ii] == "julian"){
				if (julian_present) return false; //no duplicate julian field allowed
				julian_present = true;
				julian_field = ii;
			}
		}

		if (count_wgs84 == 3)
			location_in_data_wgs84 = true;

		if (count_epsg == 3)
			location_in_data_epsg = true;
	} else {
		//every value in units_offset and units_multiplier must be a decimal
		for (vector<string>::const_iterator it = tmp_vec.begin(); it != tmp_vec.end(); ++it)
			if (!SMETCommon::is_decimal(*it)) return false;
	}

	return true;
}

void SMETWriter::write_signature()
{
	fout << "SMET " << SMETCommon::smet_version << " ";
	if (smet_type == ASCII)
		fout << "ASCII" << "\n";
	else
		fout << "BINARY" << "\n";
}

bool SMETWriter::valid_header()
{
	if (mandatory_header_keys.size() != 3)
		return false;

	if (location_in_data_epsg){ //EPSG code must be in the header anyway
		const map<string,string>::const_iterator it = header.find("epsg");
		if (it == header.end()){
			return false;
		} else {
			return true;
		}
	}

	if (location_in_header || location_in_data_wgs84)
		return true;

	return false;
}

void SMETWriter::write(const std::vector<std::string>& vec_timestamp, const std::vector<double>& data)
{
	fout.open(filename.c_str(), ios::binary);
	if (fout.fail()) {
		stringstream ss;
		ss << "Error openning file \"" << filename << "\" for writing, possible reason: " << strerror(errno);
		throw SMETException(ss.str(), SMET_AT);
	}

	write_header(); //Write the header info, always in ASCII format

	if (nr_of_fields == 0){
		cleanup();
		return;
	}

	if (!timestamp_present)
		throw SMETException("No timestamp present when writing file \""+filename+"\", use write(const vector<double>& data)", SMET_AT);

	const size_t nr_of_data_fields = nr_of_fields - 1;
	if(nr_of_fields<=1) { //avoid division by zero in the next block
		cerr << "[W] Attempting to write a dataset that contains no fields to file \"" + filename + "\"!\n";
		return;
	}

	const size_t nr_of_lines = data.size() / (nr_of_fields-1);
	if ((nr_of_lines != vec_timestamp.size()) || ((data.size() % (nr_of_fields-1)) != 0))
		throw SMETException("Inconsistency between the number of timestamp and number of data points detected for file \""+filename+"\", recheck your data", SMET_AT);

	std::vector<double> current_data(nr_of_fields-1);
	check_formatting();

	if (smet_type == ASCII){
		for (size_t ii=0; ii<nr_of_lines; ii++){
			const size_t offset = ii*(nr_of_fields-1);
			if (!data.empty())
				copy(data.begin()+offset, data.begin()+offset+nr_of_data_fields, current_data.begin());
			write_data_line_ascii(vec_timestamp[ii], current_data);
		}
	} else {
		throw SMETException("Cannot write binary file \""+filename+"\" with a timestamp, use julian instead", SMET_AT);
	}

	cleanup();
}

void SMETWriter::write(const std::vector<double>& data)
{
	fout.open(filename.c_str(), ios::binary);
	if (fout.fail()) {
		stringstream ss;
		ss << "Error openning file \"" << filename << "\" for writing, possible reason: " << strerror(errno);
		throw SMETException(ss.str(), SMET_AT);
	}

	write_header(); //Write the header info, always in ASCII format

	if (nr_of_fields == 0){
		cleanup();
		return;
	}

	const size_t nr_of_lines = data.size() / nr_of_fields;
	if ((data.size() % nr_of_fields) != 0)
		throw SMETException("Inconsistency between data and header fields detected in file \""+filename+"\", recheck your data", SMET_AT);

	std::vector<double> current_data(nr_of_fields);
	check_formatting();

	if (smet_type == ASCII){
		for (size_t ii=0; ii<nr_of_lines; ii++){
			if (!data.empty())
				copy(data.begin()+ii*nr_of_fields, data.begin()+ii*nr_of_fields+nr_of_fields, current_data.begin());
			write_data_line_ascii("0000-01-01T00:00", current_data); //dummy time
		}
	} else {
		for (size_t ii=0; ii<nr_of_lines; ii++){
			if (!data.empty())
				copy(data.begin()+ii*nr_of_fields, data.begin()+ii*nr_of_fields+nr_of_fields, current_data.begin());
			write_data_line_binary(current_data);
		}

		file_is_binary = false;
	}

	cleanup();
}

void SMETWriter::write_header()
{
	if (!valid_header())
		throw SMETException("The header data you supplied is not valid, file \""+filename+"\" cannot be written", SMET_AT);

	write_signature();

	fout << "[HEADER]" << "\n";
	fout << "station_id       = " << header["station_id"] << "\n";

	map<string,string>::const_iterator it = header.find("station_name");
	if (it != header.end())
		fout << "station_name     = " << it->second << "\n";

	if (location_in_header){
		if (location_wgs84 == 7){
			fout << "latitude         = " << header["latitude"]  << "\n";
			fout << "longitude        = " << header["longitude"] << "\n";
			fout << "altitude         = " << header["altitude"]  << "\n";
		}

		if (location_epsg == 15){
			fout << "easting          = " << header["easting"]   << "\n";
			fout << "northing         = " << header["northing"]  << "\n";
			if (location_wgs84 != 7)
				fout << "altitude         = " << header["altitude"]  << "\n";
			fout << "epsg             = " << header["epsg"]  << "\n";
		}
	} else {
		if (location_in_data_epsg)
			fout << "epsg             = " << header["epsg"]      << "\n";
	}

	fout << "nodata           = " << header["nodata"] << "\n";

	//Optional header keys:
	it = header.find("tz");
	if (it != header.end())
		fout << "tz               = " << it->second << "\n";

	it = header.find("creation");
	if (it != header.end())
		fout << "creation         = " << it->second << "\n";

	it = header.find("source");
	if (it != header.end())
		fout << "source           = " << it->second << "\n";

	it = header.find("units_offset");
	if (it != header.end())
		fout << "units_offset     = " << it->second << "\n";

	it = header.find("units_multiplier");
	if (it != header.end())
		fout << "units_multiplier = " << it->second << "\n";

	it = header.find("comment");
	if (it != header.end())
		fout << "comment          = " << it->second << "\n";

	for (size_t ii=0; ii<other_header_keys.size(); ii++){
		fout << other_header_keys[ii] << " = " << header[other_header_keys[ii]] << "\n";
	}

	fout << "fields           = " << header["fields"] << "\n";
	fout << "[DATA]" << endl;
}

void SMETWriter::write_data_line_binary(const std::vector<double>& data)
{
	const char eoln = '\n';

	if (!file_is_binary){ //open it as binary file
		fout.close();
		fout.open(filename.c_str(), ios::out | ios::app | ios::binary); //reopen as binary file
		if (fout.fail()) {
			stringstream ss;
			ss << "Error writing to file \"" << filename << "\", possible reason: " << strerror(errno);
			throw SMETException(ss.str(), SMET_AT);
		}

		file_is_binary = true;
	}

	for (size_t ii = 0; ii < data.size(); ii++){
		if (julian_present && (julian_field == ii)){
			const double julian = data[ii];
			fout.write((char*)&julian, sizeof(double)); //the julian date is written in 64bit IEEE754 precision
		} else {
			const float val = (float)data[ii];
			fout.write((char*)&val, sizeof(float)); //normal data fields are written in 32bit IEEE754 precision
		}
	}

	fout.write((char*)&eoln, sizeof(char));
}

void SMETWriter::write_data_line_ascii(const std::string& timestamp, const std::vector<double>& data)
{
	fout.fill(' ');
	fout << right;
	fout << fixed;

	if ((data.empty()) && timestamp_present) fout << timestamp;

	for (size_t ii = 0; ii < data.size(); ii++){
		if (ii > 0) fout << " ";
		if (timestamp_present && (timestamp_field == ii))	fout << timestamp << " ";

		fout << setw(ascii_width[ii]) << setprecision(ascii_precision[ii]);
		if (data[ii] == nodata_value) fout << nodata_string; //to have a nicer representation
		else fout << data[ii];
	}
	fout << "\n";
}

void SMETWriter::check_formatting()
{
	const size_t nr_of_data_fields = (timestamp_present)? nr_of_fields-1 : nr_of_fields;

	if ((ascii_precision.size() != nr_of_data_fields) || (ascii_width.size() != nr_of_data_fields)){
		ascii_precision.resize(nr_of_data_fields, 3);
		ascii_width.resize(nr_of_data_fields, 8);
	}

	if (julian_present){
		ascii_precision[julian_field] = 8;
		ascii_width[julian_field] = 16;
	}
}

void SMETWriter::set_width(const std::vector<int>& vec_width)
{
	ascii_width = vec_width;
}

void SMETWriter::set_precision(const std::vector<int>& vec_precision)
{
	ascii_precision = vec_precision;
}

const size_t SMETReader::streampos_every_n_lines = 2000; //save streampos every 2000 lines of data

SMETReader::SMETReader(const std::string& in_fname)
            : data_start_fpointer(), vec_offset(), vec_multiplier(), vec_fieldnames(),
              header(), indexer(),
              filename(in_fname), timestamp_start("-4714-11-24T00:00"),
              timestamp_end("9999-12-31T00:00"), nodata_value(-999.),
              julian_start(0.), julian_end(5373483.5),
              nr_of_fields(0), timestamp_field(0), julian_field(0),
              location_wgs84(0), location_epsg(0), location_data_wgs84(0), location_data_epsg(0),
              eoln('\n'),
              timestamp_present(false), julian_present(false), isAscii(true), mksa(true),
              timestamp_interval(false), julian_interval(false)
{
	std::ifstream fin; //Input file streams
	fin.clear();
	fin.open (filename.c_str(), ios::in|ios::binary); //ascii does end of line translation, which messes up the pointer code
	if (fin.fail()) {
		stringstream ss;
		ss << "Error openning file \"" << filename << "\" for reading, possible reason: " << strerror(errno);
		ss << " Please check file existence and permissions!";
		throw SMETException(ss.str(), SMET_AT);
	}

	try {
		eoln = SMETCommon::getEoln(fin); //get the end of line character for the file
		read_header(fin);
		process_header();
	} catch(...){
		cleanup(fin); //closes file
		throw;
	}
	cleanup(fin); //closes file
}

SMETReader::~SMETReader() {}

void SMETReader::cleanup(std::ifstream& fin) throw()
{
	if (fin.is_open()) //close fin if open
		fin.close();
}

void SMETReader::convert_to_MKSA(const bool& in_mksa)
{
	mksa = in_mksa;
}

std::string SMETReader::get_field_name(const size_t& nr_of_field)
{
	if (nr_of_field < nr_of_fields){
		return vec_fieldnames[nr_of_field];
	} else {
		stringstream ss;
		ss << "Trying to access field #" << nr_of_field << " (starting from 0) of " << nr_of_fields << " fields in file \"" << filename << "\". ";
		ss << "This is out of bounds!";
		throw SMETException(ss.str(), SMET_AT);
	}
}

size_t SMETReader::get_nr_of_fields() const
{
	return nr_of_fields;
}

void SMETReader::get_units_conversion(std::vector<double>& offset, std::vector<double>& multiplier) const
{
	multiplier = vec_multiplier;
	offset = vec_offset;
}

bool SMETReader::location_in_header(const LocationType& type) const
{
	if ((location_epsg == 15) && (type == EPSG)){
		return true;
	} else if ((location_wgs84 == 7) && (type == WGS84)){
		return true;
	}

	return false;
}

bool SMETReader::location_in_data(const LocationType& type) const
{
	if ((location_epsg == 8) && (location_data_epsg == 7) && (type == EPSG)){
		return true;
	} else if ((location_data_wgs84 == 7) && (type == WGS84)){
		return true;
	}

	return false;
}

void SMETReader::process_header()
{
	vector<string> tmp_vec;
	set<string> obligatory_keys;
	for (map<string,string>::iterator it = header.begin(); it != header.end(); ++it){
		if (SMETCommon::all_mandatory_header_keys.find(it->first) != SMETCommon::all_mandatory_header_keys.end())
			obligatory_keys.insert(it->first);

		if (it->first == "fields"){
			SMETCommon::readLineToVec(it->second, tmp_vec);
			string newfields;
			if (!tmp_vec.empty()){
				for (size_t ii=0; ii<tmp_vec.size(); ii++){
					if (tmp_vec[ii] == "timestamp"){
						timestamp_present = true;
						timestamp_field = ii;
					} else {
						if (nr_of_fields > 0) newfields += " ";
						vec_fieldnames.push_back(tmp_vec[ii]);
						newfields += tmp_vec[ii];
						nr_of_fields++;
					}

					if (tmp_vec[ii] == "julian"){
						julian_present = true;
						julian_field = ii;
					}

					//Using WGS_84 coordinate system
					if (tmp_vec[ii] == "latitude")  location_data_wgs84 |= 1;
					else if (tmp_vec[ii] == "longitude") location_data_wgs84 |= 2;
					else if (tmp_vec[ii] == "altitude")  location_data_wgs84 |= 4;

					//Using an EPSG coordinate system
					if (tmp_vec[ii] == "easting")  location_data_epsg |= 1;
					else if (tmp_vec[ii] == "northing") location_data_epsg |= 2;
					else if (tmp_vec[ii] == "altitude") location_data_epsg |= 4;
				}
				it->second = newfields;
			}
		}

		if (it->first == "nodata")
			nodata_value = SMETCommon::convert_to_double(it->second);

		//Using WGS_84 coordinate system
		if (it->first == "latitude")  location_wgs84 |= 1;
		else if (it->first == "longitude") location_wgs84 |= 2;
		else if (it->first == "altitude")  location_wgs84 |= 4;

		//Using an EPSG coordinate system
		if (it->first == "easting")  location_epsg |= 1;
		else if (it->first == "northing") location_epsg |= 2;
		else if (it->first == "altitude") location_epsg |= 4;
		else if (it->first == "epsg")     location_epsg |= 8;

		//Now do a value check on EPSG
		if (it->first == "epsg")
			SMETCommon::convert_to_int(it->second);
	}

	if (get_header_value("units_offset") != ""){
		vector<string> tmp_offset;
		string offsetstring;
		SMETCommon::readLineToVec(get_header_value("units_offset"), tmp_offset);

		for (size_t ii=0; ii<tmp_offset.size(); ii++){
			if (!timestamp_present || (ii != timestamp_field)){
				if (ii>0) offsetstring += " ";
				offsetstring += tmp_offset[ii];
				vec_offset.push_back(SMETCommon::convert_to_double(tmp_offset[ii]));
			}
		}
		header["units_offset"] = offsetstring;
	} else {
		vec_offset.resize(nr_of_fields, 0.0);
	}

	if (get_header_value("units_multiplier") != ""){
		vector<string> tmp_multiplier;
		string multiplierstring;
		SMETCommon::readLineToVec(get_header_value("units_multiplier"), tmp_multiplier);

		for (size_t ii=0; ii<tmp_multiplier.size(); ii++){
			if (!timestamp_present || (ii != timestamp_field)){
				if (ii>0) multiplierstring += " ";
				multiplierstring += tmp_multiplier[ii];
				vec_multiplier.push_back(SMETCommon::convert_to_double(tmp_multiplier[ii]));
			}
		}
		header["units_multiplier"] = multiplierstring;
	} else {
		vec_multiplier.resize(nr_of_fields, 1.0);
	}

	if (obligatory_keys.size() != SMETCommon::all_mandatory_header_keys.size())
		throw SMETException("\""+filename+"\" is not a valid SMET file, mandatory information in header is missing", SMET_AT);

	if ((location_wgs84 == 7) || (location_epsg == 15) || (location_data_wgs84 == 7)
	    || ((location_epsg == 8) && (location_data_epsg == 7))){
		//location info present somewhere
	} else {
		throw SMETException("\""+filename+"\" is not a valid SMET file, mandatory location info is missing (header and data)", SMET_AT);
	}

	const size_t nr_offset = vec_offset.size();
	const size_t nr_multiplier = vec_multiplier.size();
	if (nr_offset!=nr_of_fields || nr_multiplier!=nr_of_fields) {
		const size_t Nfields = (timestamp_present)? nr_of_fields+1 : nr_of_fields;
		const size_t Noffset = (timestamp_present)? nr_offset+1 : nr_offset;
		const size_t Nmultiplier =(timestamp_present)? nr_multiplier+1 : nr_multiplier;
		std::stringstream ss;
		ss << "File \"" << filename << "\" has " << Nfields << " data fields, but specifies ";
		ss << Noffset << " offsets and " << Nmultiplier << " multipliers. ";
		ss << "The number of offsets and multipliers MUST fit the number of fields, including the (optional) timestamp column.";
		throw SMETException(ss.str(), SMET_AT);
	}
}

void SMETReader::read_header(std::ifstream& fin)
{
	//1. Read signature
	std::string line;
	vector<string> tmpvec;
	getline(fin, line, eoln); //read complete signature line
	SMETCommon::stripComments(line);
	SMETCommon::readLineToVec(line, tmpvec);
	checkSignature(tmpvec, isAscii);

	//2. Read Header
	while (!fin.eof() && (fin.peek() != '[')) //skip lines until '[' is found
		getline(fin, line, eoln);

	getline(fin, line, eoln);
	SMETCommon::stripComments(line);
	SMETCommon::trim(line);
	SMETCommon::toUpper(line);

	if (line != "[HEADER]")
		throw;// InvalidFormatException("Section " + line + " in "+ filename + " invalid", AT);

	while (!fin.eof() && (fin.peek() != '[')){ //Read until next section
		getline(fin, line, eoln);
		SMETCommon::stripComments(line);
		SMETCommon::trim(line);

		if (!line.empty()) {
			if (!SMETCommon::readKeyValuePair(line, "=", header))
				throw SMETException("Invalid key value pair in file \""+filename+"\", in [Header] section: " + line, SMET_AT);
		}

	}

	//Read [DATA] section tag
	getline(fin, line, eoln);
	SMETCommon::stripComments(line);
	SMETCommon::trim(line);
	SMETCommon::toUpper(line);

	if (line != "[DATA]")
		throw SMETException("Section " + line + " in "+ filename + " invalid, expected [DATA]", SMET_AT);

	data_start_fpointer = fin.tellg();
}

void SMETReader::checkSignature(const std::vector<std::string>& vecSignature, bool& o_isAscii)
{
	if ((vecSignature.size() != 3) || (vecSignature[0] != "SMET"))
		throw SMETException("The signature of file " + filename + " is invalid. Is it really a SMET file?", SMET_AT);

	const std::string version = vecSignature[1];
	if ((version != "0.9") && (version != "0.95") && (version != "0.99")
	    && (version != "1.0") && (version != SMETCommon::smet_version))
		throw SMETException("Unsupported file format version for file " + filename, SMET_AT);

	if(version=="0.9" || version=="0.95" || version=="0.99" || version=="1.0") {
		cerr << "[W] SMET specification 1.1 changes the priorities of units_multiplier and units_offset. "
			<< "Please check/update your files and bring them to 1.1!!" << endl;
	}

	const std::string type = vecSignature[2];
	if (type == "ASCII")
		o_isAscii = true;
	else if (type == "BINARY")
		o_isAscii = false;
	else
		throw SMETException("The 3rd column of file " + filename + " must be either ASCII or BINARY", SMET_AT);
}

void SMETReader::read(const std::string& i_timestamp_start, const std::string& i_timestamp_end,
                      std::vector<std::string>& vec_timestamp, std::vector<double>& vec_data)
{
	if (!timestamp_present){
		read(vec_timestamp, vec_data);
	} else {
		timestamp_interval = true;
		timestamp_start = i_timestamp_start;
		timestamp_end = i_timestamp_end;

		read(vec_timestamp, vec_data);

		timestamp_interval = false;
	}
}

void SMETReader::read(const double& i_julian_start, const double& i_julian_end, std::vector<double>& vec_data)
{
	if (!julian_present){
		read(vec_data);
	} else {
		julian_interval = true;
		julian_start = i_julian_start;
		julian_end = i_julian_end;

		read(vec_data);

		julian_interval = false;
	}
}

void SMETReader::read(std::vector<std::string>& vec_timestamp, std::vector<double>& vec_data)
{
	if (!timestamp_present)
		throw SMETException("Requesting to read timestamp when there is none present in \""+filename+"\"", SMET_AT);

	ifstream fin;
	fin.clear();
	fin.open (filename.c_str(), ios::in|ios::binary); //ascii mode messes up pointer code on windows (automatic eol translation)
	if (fin.fail()) {
		stringstream ss;
		ss << "Error openning file \"" << filename << "\" for reading, possible reason: " << strerror(errno);
		throw SMETException(ss.str(), SMET_AT);
	}

	try {
		streampos fpointer = static_cast<streampos>(-1);
		if (timestamp_interval && timestamp_present){
			fpointer = indexer.getIndex(timestamp_start);
		} else if (julian_interval && julian_present){
			fpointer = indexer.getIndex(julian_start);
		}

		if(fpointer!=static_cast<streampos>(-1))
			fin.seekg(fpointer); //a previous pointer was found, jump to it
		else {
			if(data_start_fpointer!=static_cast<streampos>(-1))
				fin.seekg(data_start_fpointer); //nothing was found, jump to data start position in the file
			else { //the data section was itself empty (not even containg \n)
				cleanup(fin);
				return;
			}
		}

		if (fin.fail() || fin.bad())
			fin.seekg(data_start_fpointer);

		if (isAscii)
			read_data_ascii(fin, vec_timestamp, vec_data);
		else
			throw SMETException("Binary SMET file \""+filename+"\" has no field timestamp, only julian date", SMET_AT);
	} catch(...) {
		cleanup(fin);
		throw;
	}