WSL/SLF GitLab Repository

views_helpers.py 10.5 KB
Newer Older
1
# ==========================================  VIEWS HELPERS ===========================================================
2

3
import os
4
from datetime import datetime, timedelta
5
import importlib
6
7

from django.core.exceptions import FieldDoesNotExist
8
from django.db.models import Min, Max, Avg, Func
9
from django.http import HttpResponseNotFound
10
from datetime import timezone
11
12
13
from pathlib import Path
import configparser

14
15
from gcnet.util.geometry import convert_string_to_list

16
import django
17

18
19
20
21
django.setup()

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'project.settings')

22
23
24
# =========================================== CONSTANTS ===============================================================

# String passed in kwargs['parameters'] that is used to return returned_parameters
25
ALL_DISPLAY_VALUES_STRING = 'multiple'
26
27

# Specifies which fields to return from database table
28
ALL_DISPLAY_VALUES = ['swin',
29
30
                      'swin_maximum',
                      'swout',
31
                      'swin_stdev',
32
                      'netrad',
33
                      'netrad_stdev',
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
                      'airtemp1',
                      'airtemp1_maximum',
                      'airtemp1_minimum',
                      'airtemp2',
                      'airtemp2_maximum',
                      'airtemp2_minimum',
                      'airtemp_cs500air1',
                      'airtemp_cs500air2',
                      'rh1',
                      'rh2',
                      'windspeed1',
                      'windspeed_u1_maximum',
                      'windspeed_u1_stdev',
                      'windspeed2',
                      'windspeed_u2_maximum',
                      'windspeed_u2_stdev',
                      'winddir1',
                      'winddir2',
                      'pressure',
                      'sh1',
                      'sh2',
                      'battvolt',
                      'reftemp']
57
58
59
60
61
62


# =========================================== FUNCTIONS ===============================================================

# ------------------------------------------- Read Config ------------------------------------------------------------

63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
def read_config(config_path: str):
    config_file = Path(config_path)

    # Load gcnet configuration file
    gc_config = configparser.RawConfigParser(inline_comment_prefixes='#', allow_no_value=True)
    gc_config.read(config_file)

    # print("Read config params file: {0}, sections: {1}".format(config_path, ', '.join(gc_config.sections())))

    if len(gc_config.sections()) < 1:
        print("Invalid config file, missing sections")
        return None

    return gc_config


79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
# ------------------------------------------- Documentation Context ---------------------------------------------------

# Get documentation context with field attributes of model_class
def get_documentation_context(model_class):
    params_dict = {}
    for field in model_class._meta.get_fields():
        params_dict[field.name] = {'param': field.name, 'long_name': field.verbose_name, 'units': field.help_text}

    keys_to_remove = ['id', 'timestamp_iso', 'timestamp', 'year', 'julianday', 'quarterday', 'halfday', 'day', 'week']
    for key in keys_to_remove:
        params_dict.pop(key)

    context = {'parameters': params_dict}

    return context


96
# -------------------------------------- Date Validators --------------------------------------------------------------
97

98
def validate_date_gcnet(start, end):
99
100
101
102
103
104
105
106
107
    # Check if start and end are both only in day format (and do not include times),
    # add additional day to end date of dict_ts
    if validate_day_only_format(start) and validate_day_only_format(end):
        end_plus1 = get_date(end)
        dict_ts = {'timestamp_iso__gte': start,
                   'timestamp_iso__lt': end_plus1}
        return dict_ts

    elif validate_iso_format_gcnet(start) and validate_iso_format_gcnet(end):
108
        dict_ts = {'timestamp_iso__range': (format_timestamp(start), format_timestamp(end))}
109
110
111
112
113
114
115
116
117
118
        return dict_ts

    elif validate_unix_timestamp(int(start)) and validate_unix_timestamp(int(end)):
        dict_ts = {'timestamp__range': (start, end)}
        return dict_ts

    else:
        raise ValueError("Incorrect date formats, start and end dates should both be in ISO format or unix timestamp")


119
120
121
122
123
def get_date(input_date, date_format="%Y-%m-%d", add_day=1):
    date_plus_1day = datetime.strptime(input_date, date_format) + timedelta(days=add_day)
    return date_plus_1day.strftime(date_format)


124
125
126
127
128
129
130
131
132
133
# Returns timestamp string with UTC timezone if it matches designated format, for example '2021-12-04 17:00:00+00:00'
# Else returns timestamp string unaltered
def format_timestamp(timestamp):
    try:
        dt_object = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=timezone.utc)
        return dt_object
    except ValueError:
        return timestamp


134
135
136
137
138
139
140
def validate_day_only_format(date_text):
    try:
        day_only_format = "%Y-%m-%d"
        datetime.strptime(date_text, day_only_format)
        return True
    except ValueError:
        return False
141
142


143
144
145
146
def validate_iso_format_gcnet(date_text):
    try:
        datetime.fromisoformat(date_text)
        return True
147
    except ValueError:
148
149
150
151
152
153
154
        return False


def validate_unix_timestamp(date_text):
    try:
        datetime.fromtimestamp(date_text)
        return True
155
    except ValueError:
156
157
158
159
        return False


# ----------------------------------------  Get Model Functions -------------------------------------------------------
160
161
def get_model(app, **kwargs):
    model = kwargs['model']
162
163
    model_url = model.rsplit('.', 1)[-1]
    class_name = get_model_from_config(model_url)
Rebecca Kurup Buchholz's avatar
Rebecca Kurup Buchholz committed
164
    package = importlib.import_module(f'{app}.models')
165
166
167
    return getattr(package, class_name)


168
169
170
171
172
def get_model_class(class_name):
    package = importlib.import_module("gcnet.models")
    return getattr(package, class_name)


173
174
def get_model_url_dict():
    # Read the stations config file
Rebecca Kurup Buchholz's avatar
Rebecca Kurup Buchholz committed
175
    stations_path = Path('gcnet/config/stations.ini')
176
177
178
179
180
181
182
183
184
185
    stations_config = read_config(stations_path)

    # Check if stations_config exists
    if not stations_config:
        return HttpResponseNotFound("<h1>Not found: station config doesn't exist</h1>")

    # Assign variables to stations_config values and loop through each station in stations_config, create dictionary of
    # model_url:model key:value pairs
    model_dict = {}
    for section in stations_config.sections():
186
        if stations_config.get(section, 'active') == 'True':
187
188
189
190
191
192
193
194
195
196
197
198
            model_id = stations_config.get(section, 'model')
            model_url = stations_config.get(section, 'model_url')
            model_dict[model_url] = model_id
    return model_dict


def get_model_from_config(model_url):
    model_dict = get_model_url_dict()
    model = model_url
    if model_url in model_dict:
        model = model_dict[model_url]
    return model
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237


# ----------------------------------------  Streaming Helpers ---------------------------------------------------------

# Assign null_value
def get_null_value(nodata_kwargs):
    if nodata_kwargs == 'empty':
        null_value = ''
    else:
        null_value = nodata_kwargs
    return null_value


# Fill hash_lines with config_buffer lines prepended with '# '
def get_hashed_lines(config_buffer):
    hash_lines = []
    for line in config_buffer.replace('\r\n', '\n').split('\n'):
        line = '# ' + line + '\n'
        hash_lines.append(line)
    return hash_lines


# --------------------------------------- Aggregate View Helpers ------------------------------------------------------

class Round2(Func):
    function = "ROUND"
    template = "%(function)s(%(expressions)s::numeric, 2)"


# Get 'dict_fields' for aggregate views
def get_dict_fields(display_values):
    dict_fields = {'timestamp_first': Min('timestamp_iso'),
                   'timestamp_last': Max('timestamp_iso')}

    for parameter in display_values:
        dict_fields[parameter + '_min'] = Min(parameter)
        dict_fields[parameter + '_max'] = Max(parameter)
        dict_fields[parameter + '_avg'] = Round2(Avg(parameter))

238
239
240
    return dict_fields


241
242
# Get dict_timestamps for metadata view
def get_dict_timestamps():
243
244
245
246
    dict_timestamps = {'timestamp_iso_earliest': Min('timestamp_iso'),
                       'timestamp_earliest': (Min('timestamp')),
                       'timestamp_iso_latest': Max('timestamp_iso'),
                       'timestamp_latest': Max('timestamp')}
247
248
249
250

    return dict_timestamps


251
# --------------------------------------- Dynamic Parameters Validators -----------------------------------------------
252
253
254
255

# Validate parameters and return them as display_values list
# parameters  - comma separated string from kwargs['parameters']
# model_class  - validated model as a class
256
def validate_display_values(parameters, model_class):
257
258
259
260
261
262
263
264
265
266
267
268
269
    # Split parameters comma separated string into parameter_list
    parameters_list = convert_string_to_list(parameters)

    # Validate parameters in parameters_list and add to display_values
    display_values = []
    for parameter in parameters_list:
        try:
            model_class._meta.get_field(parameter)
            display_values = display_values + [parameter]
        except FieldDoesNotExist:
            pass

    return display_values
270
271
272


# Get display_values by validating passed parameters
273
# If parameters == ALL_DISPLAY_VALUES_STRING assign display_values to values in returned_parameters
274
275
# Else validate parameter(s) passed in URL
def get_display_values(parameters, model_class):
276

277
278
279
280
    if parameters == ALL_DISPLAY_VALUES_STRING:
        return ALL_DISPLAY_VALUES

    return validate_display_values(parameters, model_class)
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310


def multiprocessing_timestamp_dict(manager_dict, param, model_class, timestamps_dict):
    filter_dict = {f'{param}__isnull': False}

    qs = (model_class.objects
          .values(param)
          .filter(**filter_dict)
          .aggregate(**timestamps_dict))

    # TODO remove the following block that converts unix timestamps
    #  from whole seconds into milliseconds after data re-imported
    timestamp_latest = qs.get('timestamp_latest')
    timestamp_earliest = qs.get('timestamp_earliest')
    if timestamp_latest is not None and timestamp_earliest is not None:
        timestamp_latest_dict = {'timestamp_latest': timestamp_latest * 1000}
        qs.update(timestamp_latest_dict)
        timestamp_earliest_dict = {'timestamp_earliest': timestamp_earliest * 1000}
        qs.update(timestamp_earliest_dict)

    manager_dict[param] = qs


def get_multiprocessing_arguments(queryset, parameters, model_class, dict_timestamps):
    arguments = []

    for param in parameters:
        arguments.append((queryset, param, model_class, dict_timestamps))

    return arguments