MeteoIODoc 2.11.0
IOUtils.h
Go to the documentation of this file.
1// SPDX-License-Identifier: LGPL-3.0-or-later
2/***********************************************************************************/
3/* Copyright 2009 WSL Institute for Snow and Avalanche Research SLF-DAVOS */
4/***********************************************************************************/
5/* This file is part of MeteoIO.
6 MeteoIO is free software: you can redistribute it and/or modify
7 it under the terms of the GNU Lesser General Public License as published by
8 the Free Software Foundation, either version 3 of the License, or
9 (at your option) any later version.
10
11 MeteoIO is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU Lesser General Public License for more details.
15
16 You should have received a copy of the GNU Lesser General Public License
17 along with MeteoIO. If not, see <http://www.gnu.org/licenses/>.
18*/
19#ifndef IOUTILS_H
20#define IOUTILS_H
21
22#include <iostream>
23#include <string>
24#include <map>
25#include <vector>
26#include <set>
27#include <cstdlib>
28#include <limits>
29#include <cmath>
30
35
36namespace mio {
37
38#ifdef _MSC_VER
39double round(const double& x);
40#endif
41
42class MeteoData;
43class Coords;
44class Config;
45
51std::string getLibVersion(const bool& short_version=false);
52
53namespace IOUtils {
55 raw = 1,
56 filtered = 1 << 1,
57 resampled = 1 << 2,
58 generated = 1 << 3,
59 num_of_levels = 1 << 4
60 };
61
72 };
73
75 const double nodata = -999.0;
76 const unsigned int unodata = static_cast<unsigned int>(-1);
77 const int inodata = -999;
78 const short int snodata = -999;
79 const char cnodata = std::numeric_limits<char>::max();
80 const size_t npos = static_cast<size_t>(-1);
81
82 const double grid_epsilon = 5.;
84 const double lat_epsilon = lon_epsilon;
85
86 inline double C_TO_K(const double& T) {return ((T==nodata)? T : T + Cst::t_water_freezing_pt);}
87 inline double K_TO_C(const double& T) {return ((T==nodata)? T : T - Cst::t_water_freezing_pt);}
88
96 inline double UV_TO_DW(const double& U, const double& V) {return ((U==nodata || V==nodata)? nodata : fmod(atan2(U, V) * Cst::to_deg + 180., 360.));}
97
104 inline double VWDW_TO_U(const double& VW, const double& DW) {return ((VW==nodata || DW==nodata)? nodata : (- VW * sin(DW*Cst::to_rad)));}
105
112 inline double VWDW_TO_V(const double& VW, const double& DW) {return ((VW==nodata || DW==nodata)? nodata : (- VW * cos(DW*Cst::to_rad)));}
113
121 inline bool checkEpsilonEquality(const double& val1, const double& val2, const double& epsilon) {return (std::abs(val1-val2) < epsilon);}
122
132 size_t seek(const Date& soughtdate, const std::vector<MeteoData>& vecM, const bool& exactmatch=true);
133
139 double bearing_to_angle(const double& bearing);
145 double angle_to_bearing(const double& angle);
151 double bearing(std::string bearing_str);
157 std::string bearing(double bearing);
158
164 std::string getLogName();
165
170 std::string getHostName();
171
176 std::string getDomainName();
177
182 void trim(std::string &s);
183
189 std::string trim(const std::string &s);
190
191 void stripComments(std::string& str);
192
193 void stripComments(std::string& str, const char& comment_mk);
194
195 void cleanEscapedCharacters(std::string& str, const std::vector<char>& escaped_chars);
196
204 void replace_all(std::string &input, const std::string& search, const std::string& format);
205
210 void removeDuplicateWhitespaces(std::string& line);
211
217 void replaceWhitespaces(std::string& line, const char& rep = '\0');
218
224 void replaceInvalidChars(std::string& line, const char& rep = '\0');
225
230 void removeQuotes(std::string& line);
231
237 void removeChars(std::string& line, const std::set<char>& specialChars);
238
247 void cleanFieldName(std::string& field, const bool& clean_whitespaces = true, const char& rep = '-');
248
256 size_t count(const std::string &input, const std::string& search);
257
266 size_t FNV_hash(const std::string& text);
267
277 bool readKeyValuePair(const std::string& in_line, const std::string& delimiter,
278 std::string &key, std::string &value, const bool& setToUpperCase=false);
279
280 void toUpper(std::string& str);
281 std::string strToUpper(std::string str);
282 void toLower(std::string& str);
283 std::string strToLower(std::string str);
284 bool isNumeric(std::string input, const unsigned int& nBase=10);
285 size_t readLineToVec(const std::string& line_in, std::vector<double>& vec_data);
286 size_t readLineToSet(const std::string& line_in, std::set<std::string>& setString);
287 size_t readLineToVec(const std::string& line_in, std::vector<std::string>& vecString);
288 size_t readLineToVec(const std::string& line_in, std::vector<std::string>& vecString, const char& delim);
289 size_t readLineToVec(const std::string& line_in, std::vector<double>& vecRet, const char& delim);
290
291 std::vector<std::string> split(const std::string& str, char delim);
292 std::vector<std::string> split(const std::string& str, std::string delim);
293
294 template <class T> std::string toString(const T& t) {
295 std::ostringstream os;
296 os << t;
297 return os.str();
298 }
299
309 template <class T> bool convertString(T& t, std::string str, std::ios_base& (*f)(std::ios_base&) = std::dec) {
310 trim(str); //delete trailing and leading whitespaces and tabs
311 if (str.empty()) {
312 t = static_cast<T>(nodata);
313 return true;
314 } else {
315 std::istringstream iss(str);
316 iss.setf(std::ios::fixed);
317 iss.precision(std::numeric_limits<double>::digits10); //try to read values with maximum precision
318 iss >> f >> t; //Convert first part of stream with the formatter (e.g. std::dec, std::oct)
319 if (iss.fail()) {
320 //Conversion failed
321 return false;
322 }
323 std::string tmp;
324 getline(iss, tmp); //get rest of line, if any
325 trim(tmp);
326 if (!tmp.empty() && tmp[0] != '#' && tmp[0] != ';') {
327 //if line holds more than one value it's invalid
328 return false;
329 }
330 return true;
331 }
332 }
333 // fully specialized template functions (implementation must not be in header)
334 template<> bool convertString<double>(double& t, std::string str, std::ios_base& (*f)(std::ios_base&));
335 template<> bool convertString<std::string>(std::string& t, std::string str, std::ios_base& (*f)(std::ios_base&));
336 template<> bool convertString<bool>(bool& t, std::string str, std::ios_base& (*f)(std::ios_base&));
337 template<> bool convertString<char>(char& t, std::string str, std::ios_base& (*f)(std::ios_base&));
338 template<> bool convertString<unsigned int>(unsigned int& t, std::string str, std::ios_base& (*f)(std::ios_base&));
339 template<> bool convertString<Coords>(Coords& t, std::string str, std::ios_base& (*f)(std::ios_base&));
340
341 bool convertString(Date& t, std::string str, const double& time_zone, std::ios_base& (*f)(std::ios_base&) = std::dec);
342
351 template <class T> void getValueForKey(const std::map<std::string,std::string>& properties,
352 const std::string& key, T& t, const ThrowOptions& options=IOUtils::dothrow) {
353 if (key.empty() && options!=IOUtils::nothrow)
354 throw InvalidArgumentException("Empty key", AT);
355
356 const std::map<std::string, std::string>::const_iterator it( properties.find(key) );
357 if (it == properties.end()){
358 if (options == IOUtils::nothrow)
359 return;
360 else
361 throw UnknownValueException("No value for key " + key, AT);
362 }
363 const std::string& value = it->second;
364
365 if (!convertString<T>(t, value, std::dec) && options!=IOUtils::nothrow) {
366 std::cerr << "[E] When reading \"" << key << "\" = \"" << t << "\"\n";
367 throw ConversionFailedException(value, AT);
368 }
369 }
370
379 template <class T> void getValueForKey(const std::map<std::string,std::string>& properties,
380 const std::string& key, std::vector<T>& vecT, const ThrowOptions& options=IOUtils::dothrow)
381 {
382 if (key.empty() && options!=IOUtils::nothrow)
383 throw InvalidArgumentException("Empty key", AT);
384
385 const std::map<std::string, std::string>::const_iterator it( properties.find(key) );
386 if (it == properties.end()) {
387 if (options == IOUtils::nothrow) {
388 return;
389 } else {
390 throw UnknownValueException("No value for key " + key, AT);
391 }
392 }
393 const std::string& value = it->second;
394
395 //split value string
396 std::vector<std::string> vecUnconvertedValues;
397 const size_t counter = readLineToVec(value, vecUnconvertedValues);
398 vecT.resize( counter );
399 for (size_t ii=0; ii<counter; ii++){
400 T myvar;
401 if (!convertString<T>(myvar, vecUnconvertedValues.at(ii), std::dec) && options!=IOUtils::nothrow){
402 std::cerr << "[E] When reading \"" << key << "\" = \"" << myvar << "\"\n";
403 throw ConversionFailedException(vecUnconvertedValues.at(ii), AT);
404 }
405 vecT[ii] = myvar;
406 }
407 }
408
416 template <class T> T standardizeNodata(const T& value, const double& plugin_nodata) {
417 if (value==plugin_nodata) return static_cast<T> (nodata);
418 else return value;
419 }
420
428 template <class T> static void parseArg(const std::pair< std::string, std::string>& arg, const std::string& algo, T& val) {
429 if (!IOUtils::convertString(val, arg.second))
430 throw InvalidArgumentException("Can not parse argument '"+arg.first+"::"+arg.second+"' for " + algo, AT);
431 }
432
442 void getProjectionParameters(const Config& cfg, std::string& coordin, std::string& coordinparam,
443 std::string& coordout, std::string& coordoutparam);
444
452 void getProjectionParameters(const Config& cfg, std::string& coordin, std::string& coordinparam);
453
460 void getTimeZoneParameters(const Config& cfg, double& tz_in, double& tz_out);
461
467 double unitsPrefix(const char& prefix);
468
477 double unitsConversion(const double& val, std::string unitIn, std::string unitOut);
478} //end namespace IOUtils
479
480} //end namespace mio
481
482#endif
#define AT
Definition: IOExceptions.h:28
A class that reads a key/value file. These files (typically named *.ini) follow the INI file format s...
Definition: Config.h:79
thrown when an unsuccessful attempt to convert data types/classes is made (e.g. attempt to convert a ...
Definition: IOExceptions.h:118
A class to handle geographic coordinate systems. This class offers an easy way to transparently conve...
Definition: Coords.h:83
A class to handle timestamps. This class handles conversion between different time display formats (I...
Definition: Date.h:87
thrown when encountered an unexpected function's argument (e.g. bad index, bad or missing parameter n...
Definition: IOExceptions.h:130
thrown when encountered an unexpected value (e.g. unknown name or key)
Definition: IOExceptions.h:142
static const double to_rad
Definition: Meteoconst.h:81
static const double t_water_freezing_pt
Definition: Meteoconst.h:50
static const double earth_R0
Definition: Meteoconst.h:62
static const double to_deg
Definition: Meteoconst.h:82
bool convertString< char >(char &t, std::string str, std::ios_base &(*f)(std::ios_base &))
Definition: IOUtils.cc:596
double bearing_to_angle(const double &bearing)
Converts a compass bearing to a trigonometric angle.
Definition: IOUtils.cc:68
size_t readLineToVec(const std::string &line_in, std::vector< double > &vec_data)
Definition: IOUtils.cc:395
const double lat_epsilon
in degrees. Small angle for latitudes.
Definition: IOUtils.h:84
bool readKeyValuePair(const std::string &in_line, const std::string &delimiter, std::string &key, std::string &value, const bool &setToUpperCase)
read a string line, parse it and save it into a map object, that is passed by reference
Definition: IOUtils.cc:326
bool convertString(Date &t, std::string str, const double &time_zone, std::ios_base &(*f)(std::ios_base &))
Convert a string to a date (template specialization of convertString)
Definition: IOUtils.cc:654
static void parseArg(const std::pair< std::string, std::string > &arg, const std::string &algo, T &val)
Parse a given named argument.
Definition: IOUtils.h:428
std::string getLogName()
Retrieve the user name This checks various environment variables (USERNAME, USER, LOGNAME).
Definition: IOUtils.cc:348
T standardizeNodata(const T &value, const double &plugin_nodata)
Standardize a given value to use MeteoIO's internal nodata value (if applicable)
Definition: IOUtils.h:416
double bearing(std::string bearing_str)
Converts a string bearing to a compass bearing.
Definition: IOUtils.cc:76
void stripComments(std::string &str)
Definition: IOUtils.cc:129
double VWDW_TO_V(const double &VW, const double &DW)
From wind speed and direction to v component (south-to-north)
Definition: IOUtils.h:112
void removeQuotes(std::string &line)
Removes single and double quotation marks.
Definition: IOUtils.cc:236
void removeChars(std::string &line, const std::set< char > &specialChars)
Removes any character present in the provided set from the given line.
Definition: IOUtils.cc:242
void toLower(std::string &str)
Definition: IOUtils.cc:290
double C_TO_K(const double &T)
Definition: IOUtils.h:86
const size_t npos
npos is the out-of-range value
Definition: IOUtils.h:80
void removeDuplicateWhitespaces(std::string &line)
Removes consecutive occurrences of spaces and tabs.
Definition: IOUtils.cc:218
bool convertString< unsigned int >(unsigned int &t, std::string str, std::ios_base &(*f)(std::ios_base &))
Definition: IOUtils.cc:620
const int inodata
Definition: IOUtils.h:77
double unitsConversion(const double &val, std::string unitIn, std::string unitOut)
Performs simple unit conversion (supports temperature, prefixes and exponents) NOTE "composite" units...
Definition: IOUtils.cc:910
void cleanEscapedCharacters(std::string &str, const std::vector< char > &escaped_chars)
Definition: IOUtils.cc:157
double unitsPrefix(const char &prefix)
Convert a textual representation of a unit prefix (like 'm' or 'G') to multiplying factor.
Definition: IOUtils.cc:885
size_t FNV_hash(const std::string &text)
Fowler/Noll/Vo hash function (FNV-1a)
Definition: IOUtils.cc:273
bool convertString< double >(double &t, std::string str, std::ios_base &(*f)(std::ios_base &))
Definition: IOUtils.cc:545
bool isNumeric(std::string str, const unsigned int &nBase)
Definition: IOUtils.cc:306
std::string strToUpper(std::string str)
Definition: IOUtils.cc:294
size_t readLineToSet(const std::string &line_in, std::set< std::string > &setString)
Definition: IOUtils.cc:417
std::string getDomainName()
Retrieve the domain name of the computer running the binary.
Definition: IOUtils.cc:383
OperationMode
Keywords for mode of operation. Please keep all the GRID_xxx last!
Definition: IOUtils.h:63
@ GRID_EXTRACT_PTS
as GRID_EXTRACT, but queries plugin only for virtual stations points, instead of full grids
Definition: IOUtils.h:68
@ GRID_SMART
extract all relevant grid points from a provided grid
Definition: IOUtils.h:69
@ GRID_ALL
extract all grid points from a provided grid
Definition: IOUtils.h:70
@ GRID_RESAMPLE
generate a grid at a different resolution
Definition: IOUtils.h:71
@ VSTATIONS
extract virtual stations as specified in the ini file
Definition: IOUtils.h:65
@ GRID_1DINTERPOLATE
temporally interpolate existing grids (must be enumerated before GRID_EXTRACT)
Definition: IOUtils.h:66
@ GRID_EXTRACT
extract data from grids at locations provided in the ini file
Definition: IOUtils.h:67
@ STD
default: extract timeseries from timeseries or grids from grids or spatially interpolate timeseries
Definition: IOUtils.h:64
const double nodata
This is the internal nodata value.
Definition: IOUtils.h:75
double UV_TO_DW(const double &U, const double &V)
From wind speed components (u,v) to wind direction, following standard meteorological definitions: U ...
Definition: IOUtils.h:96
void getValueForKey(const std::map< std::string, std::string > &properties, const std::string &key, T &t, const ThrowOptions &options=IOUtils::dothrow)
Returns, with the requested type, the value associated to a key (template function).
Definition: IOUtils.h:351
bool convertString< Coords >(Coords &t, std::string str, std::ios_base &(*f)(std::ios_base &))
Definition: IOUtils.cc:789
std::string toString(const T &t)
Definition: IOUtils.h:294
const char cnodata
Definition: IOUtils.h:79
const unsigned int unodata
Definition: IOUtils.h:76
void replace_all(std::string &input, const std::string &search, const std::string &format)
Replace a substring within a given string by another one.
Definition: IOUtils.cc:198
void getProjectionParameters(const Config &cfg, std::string &coordin, std::string &coordinparam, std::string &coordout, std::string &coordoutparam)
A function that parses a Config object for COORSYS, COORDPARAM keywords in [Input] and [Output] secti...
Definition: IOUtils.cc:806
void trim(std::string &str)
Removes trailing and leading whitespaces, tabs and newlines from a string.
Definition: IOUtils.cc:176
double VWDW_TO_U(const double &VW, const double &DW)
From wind speed and direction to u component (west-to-east)
Definition: IOUtils.h:104
void toUpper(std::string &str)
Definition: IOUtils.cc:286
const short int snodata
Definition: IOUtils.h:78
size_t seek(const Date &soughtdate, const std::vector< MeteoData > &vecM, const bool &exactmatch)
Search for an element at a given date in a vector of MeteoData. The position of the matching date is ...
Definition: IOUtils.cc:827
std::string getHostName()
Retrieve the name of the computer running the binary.
Definition: IOUtils.cc:362
const double lon_epsilon
in degrees. Small angle for longitudes, so sin(x)=x
Definition: IOUtils.h:83
void replaceWhitespaces(std::string &line, const char &rep)
Replaces spaces and tabs with a single character or removes them.
Definition: IOUtils.cc:226
ProcessingLevel
Definition: IOUtils.h:54
@ resampled
Definition: IOUtils.h:57
@ num_of_levels
Definition: IOUtils.h:59
@ raw
Definition: IOUtils.h:55
@ filtered
Definition: IOUtils.h:56
@ generated
Definition: IOUtils.h:58
void getTimeZoneParameters(const Config &cfg, double &tz_in, double &tz_out)
A function that parses a Config object for the time_zone keyword and returns the timezone.
Definition: IOUtils.cc:821
void replaceInvalidChars(std::string &line, const char &rep)
Replaces invalid characters with a single character or removes them.
Definition: IOUtils.cc:231
double K_TO_C(const double &T)
Definition: IOUtils.h:87
bool convertString< bool >(bool &t, std::string str, std::ios_base &(*f)(std::ios_base &))
Definition: IOUtils.cc:516
std::vector< std::string > split(const std::string &s, char delimiter)
Definition: IOUtils.cc:478
void cleanFieldName(std::string &field, const bool &clean_whitespaces, const char &rep)
Cleans up a string to be usable as, for example, a parameter name.
Definition: IOUtils.cc:248
double angle_to_bearing(const double &angle)
Converts a trigonometric angle to a compass bearing.
Definition: IOUtils.cc:72
size_t count(const std::string &input, const std::string &search)
count how many times a substring appears in a string
Definition: IOUtils.cc:258
const double grid_epsilon
What is an acceptable small distance on a grid, in meters.
Definition: IOUtils.h:82
bool checkEpsilonEquality(const double &val1, const double &val2, const double &epsilon)
Check whether two values are equal regarding a certain epsilon environment (within certain radius of ...
Definition: IOUtils.h:121
std::string strToLower(std::string str)
Definition: IOUtils.cc:301
ThrowOptions
Definition: IOUtils.h:74
@ nothrow
Definition: IOUtils.h:74
@ dothrow
Definition: IOUtils.h:74
long int round(const double &x)
Optimized version of c++ round() This version works with positive and negative numbers but does not c...
Definition: MathOptim.h:45
Definition: Config.cc:31
std::string getLibVersion(const bool &short_version)
Return the library version.
Definition: IOUtils.cc:59