WSL/SLF GitLab Repository

IOUtils.cc 17.9 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
/***********************************************************************************/
/*  Copyright 2009 WSL Institute for Snow and Avalanche Research    SLF-DAVOS      */
/***********************************************************************************/
/* This file is part of MeteoIO.
    MeteoIO is free software: you can redistribute it and/or modify
    it under the terms of the GNU Lesser General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    MeteoIO is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public License
    along with MeteoIO.  If not, see <http://www.gnu.org/licenses/>.
*/
18
#include <cmath>
19

20
#include <meteoio/IOUtils.h>
21
22
#include <meteoio/Config.h>    // to avoid forward declaration hell
#include <meteoio/MeteoData.h> // to avoid forward declaration hell
23

24
25
#ifdef _WIN32
	#include <windows.h>
26
27
28
	//removing two macros defined in windows.h
	#undef max
	#undef min
29
30
31
32
33
#else
	#include <dirent.h>
	#include <sys/stat.h>
#endif

34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
namespace mio {

bool IOUtils::checkEpsilonEquality(const double& val1, const double& val2, const double& epsilon)
{
	if (((val1-epsilon) < val2) && ((val1+epsilon) > val2)) {
		return true;
	}

	return false;
}

double IOUtils::pow2(const double& val)
{
	return (val*val);
}

double IOUtils::pow3(const double& val)
{
	return (val*val*val);
}

double IOUtils::pow4(const double& val)
{
	return (val*val*val*val);
}

double IOUtils::bearing_to_angle(const double& bearing) {
	const double to_rad = M_PI/180.;
	return (fmod(360.-bearing+90., 360.)*to_rad);
}

double IOUtils::angle_to_bearing(const double& angle) {
	const double to_deg = 180.0 / M_PI;
	return (fmod( 90.-angle*to_deg+360. , 360. ));
}

70
71
72
73
74
75
76
77
void IOUtils::stripComments(std::string& str)
{
	size_t found = str.find_first_of("#;");
	if (found != std::string::npos){
		str.erase(found); //rest of line disregarded
	}
}

78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
void IOUtils::copy_file(const std::string& src, const std::string& dest)
{
	if (src == dest) return; //copying to the same file doesn't make sense, but is no crime either

	std::ifstream fin(src.c_str(), std::ios::binary);
	if (fin.fail()) throw FileAccessException(src, AT);

	std::ofstream fout(dest.c_str(), std::ios::binary);
	if (fout.fail()) {
		fin.close();
		throw FileAccessException(dest, AT);
	}

	fout << fin.rdbuf();

	fin.close();
	fout.close();
}

97
98
std::string IOUtils::cleanPath(const std::string& in_path) {
	std::string out_path(in_path);
Cyril Perot's avatar
Cyril Perot committed
99
100
101
102
103
104
105

	size_t curr = out_path.find('\\', 0);
	while(curr!=std::string::npos){
		out_path.replace(curr, 1, "/");
		curr = out_path.find('\\', curr);
	}
	//out_path.replace(out_path.begin(), out_path.end(), '\\', '/');
106
107
108
	return out_path;
}

109
110
void IOUtils::trim(std::string& str)
{
111
112
113
	const std::string whitespaces(" \t\f\v\n\r");
	const size_t startpos = str.find_first_not_of(whitespaces); // Find the first character position after excluding leading blank spaces
	const size_t endpos = str.find_last_not_of(whitespaces); // Find the first character position from reverse af
114
115
116
117
118
119
120
121
122
123

	// if all spaces or empty return an empty string
	if(( std::string::npos == startpos ) || ( std::string::npos == endpos)) {
		str = "";
	} else {
		str = str.substr( startpos, endpos-startpos+1 );
	}
}

void IOUtils::toUpper(std::string& str){
124
	for(size_t t=0; t<str.length(); t++) {
125
		str[t] = (char)toupper(str[t]);
126
127
128
129
	}
}

bool IOUtils::readKeyValuePair(const std::string& in_line, const std::string& delimiter,
130
                               std::map<std::string,std::string>& out_map, const std::string& keyprefix, const bool& setToUpperCase)
131
132
133
134
135
136
137
{
	//size_t pos = in_line.find(delimiter); //first occurence of '='

	size_t pos = std::string::npos;
	if ((delimiter==" ") || (delimiter=="\t")) {
		pos = in_line.find_first_of(" \t", 0);
	} else {
Cyril Perot's avatar
Cyril Perot committed
138
		pos = in_line.find(delimiter); //first occurence of '='
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
	}


	if(pos != std::string::npos) { //ignore in_lines that are empty or without '='
		std::string key = in_line.substr(0, pos);
		std::string value = in_line.substr(pos + 1);

		IOUtils::trim(key);
		IOUtils::trim(value);
		//cerr << "key:" << key << " val:" << value << endl;

		if ((key == "") || (value=="")) {
			return false;
		}

154
155
		if (setToUpperCase)
			IOUtils::toUpper(key);
Cyril Perot's avatar
Cyril Perot committed
156

157

158
159
160
161
162
163
164
165
166
		out_map[keyprefix + key] = value;
	} else {
		return false;
		//cerr << "line:" << in_line << "delimiter" << endl;
	}

	return true;
}

167
168
bool IOUtils::validFileName(const std::string& filename)
{
169
	const size_t startpos = filename.find_first_not_of(" \t\n"); // Find the first character position after excluding leading blank spaces
170
171
172
173
174
175
176
177
	if((startpos!=0) || (filename==".") || (filename=="..")) {
		return false;
	}

	return true;
}

#ifdef _WIN32
178
179
bool IOUtils::fileExists(const std::string& filename)
{
180
	return ( GetFileAttributes( filename.c_str() ) != INVALID_FILE_ATTRIBUTES );
181
}
182

183
184
185
186
187
void IOUtils::readDirectory(const std::string& path, std::list<std::string>& dirlist, const std::string& pattern)
{
	WIN32_FIND_DATA ffd;
	HANDLE hFind = INVALID_HANDLE_VALUE;

188
	const size_t path_length = path.length();
189
190
191
	if (path_length > (MAX_PATH - 3)) {
		std::cout << "Path " << path << "is too long (" << path_length << " characters)" << std::endl;
		throw FileAccessException("Error opening directory " + path, AT);
192
193
	}

194
195
	const std::string filepath = path+"\\"+pattern;
	hFind = FindFirstFile(filepath.c_str(), &ffd);
196
197
198
	if (INVALID_HANDLE_VALUE == hFind) {
		throw FileAccessException("Error opening directory " + path, AT);
	}
199

200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
	do {
		if (ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
			//this is a directory -> do nothing
		} else {
			std::string filename(ffd.cFileName);
			dirlist.push_back(filename);
		}
	}
	while (FindNextFile(hFind, &ffd) != 0);

	const DWORD dwError = GetLastError();
	if (dwError != ERROR_NO_MORE_FILES) {
		throw FileAccessException("Error listing files in directory " + path, AT);
	}

	FindClose(hFind);
}
#else
bool IOUtils::fileExists(const std::string& filename)
219
{
220
221
222
223
	struct stat buffer ;

	if ((stat( filename.c_str(), &buffer))==0) {//File exists if stat returns 0
		return true ;
224
225
	}

226
	return false;
227
228
}

229
230
231
232
233
234
235
236
237
238
void IOUtils::readDirectory(const std::string& path, std::list<std::string>& dirlist, const std::string& pattern)
{
	DIR *dp;
	struct dirent *dirp;

	if((dp  = opendir(path.c_str())) == NULL) {
		throw FileAccessException("Error opening directory " + path, AT);
	}

	while ((dirp = readdir(dp)) != NULL) {
239
		const std::string tmp = std::string(dirp->d_name);
240
241
242
243
		if( tmp.compare(".")!=0 && tmp.compare("..")!=0 ) { //skip "." and ".."
			if (pattern=="") {
				dirlist.push_back(tmp);
			} else {
244
				const size_t pos = tmp.find(pattern);
245
246
247
248
249
250
251
252
				if (pos!=std::string::npos) {
					dirlist.push_back(tmp);
				}
			}
		}
	}
	closedir(dp);
}
253
#endif
254

Cyril Perot's avatar
Cyril Perot committed
255
void IOUtils::readKeyValueHeader(std::map<std::string,std::string>& headermap,
256
                                 std::istream& fin,
257
                                 const size_t& linecount,
258
                                 const std::string& delimiter)
259
{
260
	size_t linenr = 0;
261
262
263
264
265
	std::string line="";

	//make a test for end of line encoding:
	char eol = IOUtils::getEoln(fin);

266
	for (size_t ii=0; ii< linecount; ii++){
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
		if (std::getline(fin, line, eol)) {
			//cout << line <<endl;
			linenr++;

			bool result = IOUtils::readKeyValuePair(line, delimiter, headermap);

			if (!result) { //  means if ((key == "") || (value==""))
				std::stringstream out;
				out << "Invalid key value pair in line: " << linenr << " of header";
				throw IOException(out.str(), AT);
			}
		} else {
			throw InvalidFormatException("Premature EOF while reading Header", AT);
		}
	}
}

char IOUtils::getEoln(std::istream& fin)
{
	std::streambuf* pbuf;
	char tmp = '0';
	int chars = 0;

290
	const std::streampos position = fin.tellg();
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305

	do {
		fin.get(tmp);
		chars++;

		if ((tmp == '\r') || (tmp == '\n')) {
			char peekc = tmp;
			//cout << (int)tmp << endl;
			while ((!fin.eof() && ((peekc=='\r') || (peekc=='\n')))) {
				tmp = peekc;
				fin.get(peekc);
				chars++;
			}
			pbuf = fin.rdbuf();
			pbuf->pubseekpos(position); //rewind
306
			fin.clear(); //reset eof flag, etc
307
308
309
			return tmp;
		}
	} while ((chars < 3000) && (!fin.eof()));
Cyril Perot's avatar
Cyril Perot committed
310

311
312
	pbuf = fin.rdbuf();
	pbuf->pubseekpos(position); //rewind
313
	fin.clear(); //reset eof flag, etc
314
315
316
317

	return '\n';
}

318
void IOUtils::skipLines(std::istream& fin, const size_t& nbLines, const char& eoln)
319
320
{
	std::string dummy;
321
	for (size_t ii=0; ii<nbLines; ii++) {
322
323
324
325
326
327
		if(!getline(fin, dummy, eoln)) {
			throw InvalidFormatException("Premature EOF while skipping lines", AT);
		}
	}
}

328
size_t IOUtils::readLineToVec(const std::string& line_in, std::vector<std::string>& vecString)
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
{
	vecString.clear();
	std::istringstream iss(line_in); //construct inputstream with the string line as input

	std::string tmp_string;
	while (!iss.eof()) {
		iss >> std::skipws >> tmp_string;

		if (tmp_string != "") {
			vecString.push_back(tmp_string);
		}
		tmp_string="";
	}

	return vecString.size();
}

346
size_t IOUtils::readLineToVec(const std::string& line_in, std::vector<std::string>& vecString, const char& delim)
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
{
	vecString.clear();
	std::string tmp_string;
	std::istringstream iss(line_in);

	while (getline(iss, tmp_string, delim)){
		vecString.push_back(tmp_string);
	}

	return vecString.size();
}

// generic template function convertString must be defined in the header

const char ALPHANUM[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
362
const char NUM[] = "0123456789";
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404

template<> bool IOUtils::convertString<std::string>(std::string& t, const std::string& str, std::ios_base& (*f)(std::ios_base&))
{
	(void)f;
	std::string s = str;
	trim(s); //delete trailing and leading whitespaces and tabs

	t = s;
	return true;
}

template<> bool IOUtils::convertString<bool>(bool& t, const std::string& str, std::ios_base& (*f)(std::ios_base&))
{
	std::string s = str;
	trim(s); //delete trailing and leading whitespaces and tabs

	if (toupper(s[0])=='T' || toupper(s[0])=='Y' ) {
		t = true;
	} else if (toupper(s[0])=='F' || toupper(s[0])=='N' ) {
		t = false;
	} else {
		std::istringstream iss(s);
		int i;
		iss >> f >> i; //Convert first part of stream with the formatter (e.g. std::dec, std::oct)
		if (iss.fail()) {//Conversion failed
			return false;
		}
		t = (i != 0);
	}

	std::string::size_type pos = s.find_first_not_of(ALPHANUM);
	if (pos != std::string::npos) {
		std::string tmp = s.substr(pos);
		trim(tmp);
		if ((tmp.length() > 0) && tmp[0] != '#' && tmp[0] != ';') {//if line holds more than one value it's invalid
			return false;
		}
	}

	return true;
}

405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
template<> bool IOUtils::convertString<unsigned int>(unsigned int& t, const std::string& str, std::ios_base& (*f)(std::ios_base&))
{
	std::string s = str;
	trim(s); //delete trailing and leading whitespaces and tabs
	if (s.size() == 0) {
		t = unodata;
		return true;
	} else {
		std::istringstream iss(s);
		iss.setf(std::ios::fixed);
		iss.precision(std::numeric_limits<double>::digits10); //try to read values with maximum precision
		iss >> f >> t; //Convert first part of stream with the formatter (e.g. std::dec, std::oct)
		if (iss.fail()) {
			//Conversion failed
			return false;
		}
		std::string tmp="";
		getline(iss,  tmp); //get rest of line, if any
		trim(tmp);
		if ((tmp.length() > 0) && tmp[0] != '#' && tmp[0] != ';') {
			//if line holds more than one value it's invalid
			return false;
		}
		return true;
	}
}

432
bool IOUtils::convertString(Date& t, const std::string& str, const double& time_zone, std::ios_base& (*f)(std::ios_base&))
433
434
435
436
437
{
	std::string s = str;
	trim(s); //delete trailing and leading whitespaces and tabs

	(void)f;
438
439
	int year;
	unsigned int month, day, hour, minute, second;
440
	char rest[32] = "";
441
442

	//HACK: we read the seconds, but we ignore them...
443
	if (sscanf(s.c_str(), "%d-%u-%u %u:%u:%u%31s", &year, &month, &day, &hour, &minute, &second, rest) >= 6) {
444
		t.setDate(year, month, day, hour, minute, time_zone);
445
	} else if (sscanf(s.c_str(), "%d-%u-%uT%u:%u:%u%31s", &year, &month, &day, &hour, &minute, &second, rest) >= 6) {
446
		t.setDate(year, month, day, hour, minute, time_zone);
447
	} else if (sscanf(s.c_str(), "%d-%u-%u %u:%u%31s", &year, &month, &day, &hour, &minute, rest) >= 5) {
448
		t.setDate(year, month, day, hour, minute, time_zone);
449
	} else if (sscanf(s.c_str(), "%d-%u-%uT%u:%u%31s", &year, &month, &day, &hour, &minute, rest) >= 5) {
450
		t.setDate(year, month, day, hour, minute, time_zone);
451
452
	} else if (sscanf(s.c_str(), "%d-%u-%u%31s", &year, &month, &day, rest) >= 3) {
		t.setDate(year, month, day, (unsigned)0, (unsigned)0, time_zone);
453
	} else if (sscanf(s.c_str(), "%u:%u%31s", &hour, &minute, rest) >= 2) {
454
		t.setDate( ((double)hour)/24. + ((double)minute)/24./60. , time_zone);
455
	} else {
456
		//try to read purely numerical date, potentially surrounded by other chars
457
		const size_t in_len = str.length();
458
		const size_t beg = str.find_first_of(NUM);
459
460
461
462
463
		if(beg==npos || beg==in_len) return false;
		size_t end = str.find_first_not_of(NUM, beg+1);
		if(end==npos) end = in_len;

		const std::string datum = str.substr(beg, end-beg);
464
		const size_t d_len = datum.length();
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
		if(d_len<8 || d_len>14) return false;
		if( convertString(year,datum.substr(0,4))==false ) return false;
		if( convertString(month,datum.substr(4,2))==false ) return false;
		if( convertString(day,datum.substr(6,2))==false ) return false;
		if( convertString(hour,datum.substr(8,2))==false ) return false;
		if(d_len==10)
			minute=0;
		else {
			if(d_len>=12) {
				if( convertString(minute,datum.substr(10,2))==false ) return false;
			} else
				return false;
			if(d_len==12)
				second=0;
			else {
				if(d_len==14) {
					if( convertString(second,datum.substr(12,2))==false ) return false;
				} else
					return false;
			}
		}
486

487
		t.setDate( year, month, day, hour, minute, time_zone );
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
	}

	std::string tmp(rest);
	trim(tmp);
	if ((tmp.length() > 0) && tmp[0] != '#' && tmp[0] != ';') {//if line holds more than one value it's invalid
		return false;
	}

	return true;
}

template<> bool IOUtils::convertString<Coords>(Coords& t, const std::string& str, std::ios_base& (*f)(std::ios_base&))
{
	std::string s = str;
	trim(s); //delete trailing and leading whitespaces and tabs

	(void)f;
	double lat, lon;
	try {
		Coords::parseLatLon(s,lat, lon);
	} catch(IOException &e) {
		return false;
	}
	t.setLatLon(lat, lon, nodata);

	return true;
}


Cyril Perot's avatar
Cyril Perot committed
517
void IOUtils::getProjectionParameters(const Config& cfg, std::string& coordin, std::string& coordinparam,
518
                                      std::string& coordout, std::string& coordoutparam) {
519
	cfg.getValue("COORDSYS", "Input", coordin);
520
521
522
	cfg.getValue("COORDPARAM", "Input", coordinparam, Config::nothrow);
	cfg.getValue("COORDSYS", "Output", coordout, Config::nothrow);
	cfg.getValue("COORDPARAM", "Output", coordoutparam, Config::nothrow);
523
524
}

525
void IOUtils::getTimeZoneParameters(const Config& cfg, double& tz_in, double& tz_out) {
526
527
	cfg.getValue("TIME_ZONE", "Input", tz_in, Config::nothrow);
	cfg.getValue("TIME_ZONE", "Output", tz_out, Config::nothrow);
528
}
Cyril Perot's avatar
Cyril Perot committed
529

530
size_t IOUtils::seek(const Date& soughtdate, const std::vector<MeteoData>& vecM, const bool& exactmatch){
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
	//returns index of element, if element does not exist it returns closest index after soughtdate
	//the element needs to be an exact hit or embedded between two measurments

	if (vecM.size() <= 0) {//no elements in buffer
		return IOUtils::npos;
	}

	//if we reach this point: at least one element in buffer
	if (vecM[0].date > soughtdate) {
		return IOUtils::npos;
	}

	if (vecM[vecM.size()-1].date < soughtdate) {//last element is earlier, return npos
		return IOUtils::npos;
	}

	if (vecM[0].date == soughtdate) {//closest element
		return 0;
	}

	//if we reach this point: the date is spanned by the buffer and there are at least two elements
552
	//HACK: would it be better to create a timeseries object and call vector's binary search on it?
553
	if (exactmatch){
554
		size_t first = 1, last = vecM.size()-1;
555
556
557

		//perform binary search
		while (first <= last) {
558
			size_t mid = (first + last) / 2;  // compute mid point
Cyril Perot's avatar
Cyril Perot committed
559
			if (soughtdate > vecM[mid].date)
560
				first = mid + 1;                   // repeat search in top half
Cyril Perot's avatar
Cyril Perot committed
561
			else if (soughtdate < vecM[mid].date)
562
				last = mid - 1;                    // repeat search in bottom half
Cyril Perot's avatar
Cyril Perot committed
563
564
			else
				return mid;                        // found it. return position
565
566
		}
	} else {
567
		size_t first = 0, last = vecM.size()-1;
568
569
570

		//perform binary search
		while (first <= last) {
571
			size_t mid = (first + last) / 2;  // compute mid point
Cyril Perot's avatar
Cyril Perot committed
572

573
574
575
576
			if (mid < (vecM.size()-1))
				if ((soughtdate > vecM[mid].date) && (soughtdate < vecM[mid+1].date))
					return mid+1;

Cyril Perot's avatar
Cyril Perot committed
577
			if (soughtdate > vecM[mid].date)
578
				first = mid + 1;                   // repeat search in top half
Cyril Perot's avatar
Cyril Perot committed
579
			else if (soughtdate < vecM[mid].date)
580
581
				last = mid - 1;                    // repeat search in bottom half
			else
Cyril Perot's avatar
Cyril Perot committed
582
583
				return mid;                        // found it. return position

584
585
586
587
588
589
		}
	}

	return IOUtils::npos;
}

590
591
592
593
594
595
596
597
598
599
600
601
602
603
std::string IOUtils::printFractionalDay(const double& fractional) {
	const double hours=floor(fractional*24.);
	const double minutes=floor((fractional*24.-hours)*60.);
	const double seconds=fractional*24.*3600.-hours*3600.-minutes*60.;

	std::stringstream tmp;
	tmp << std::fixed << std::setfill('0') << std::setprecision(0);
	tmp << std::setw(2) << hours << ":";
	tmp << std::setw(2) << minutes << ":";
	tmp << std::setw(2) << seconds;

	return tmp.str();
}

604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
void IOUtils::getArraySliceParams(const unsigned int& dimx, const unsigned int& nbworkers, const unsigned int &wk, unsigned int& startx, unsigned int& nx)
{
	if(nbworkers>dimx) {
		std::stringstream ss;
		ss << "Can not split " << dimx << " columns in " << nbworkers << " bands!";
		throw InvalidArgumentException(ss.str(), AT);
	}

	const unsigned int nx_avg = dimx / nbworkers;
	const unsigned int remainder = dimx % nbworkers;

	if(wk<=remainder) { //distribute remainder, 1 extra column per worker, on first workers
		nx = nx_avg+1;
		startx = (wk-1)*nx;
	} else { //all remainder has been distributed, we now attribute a normal number of columns
		nx = nx_avg;
		startx = remainder*(nx+1) + (wk-1-remainder)*nx;
	}
}

624
} //namespace