1 |
|
---|
2 |
|
---|
3 |
"""Utilities to help data processing |
---|
4 |
|
---|
5 |
Mostly time functions right now |
---|
6 |
|
---|
7 |
""" |
---|
8 |
|
---|
9 |
__version__ = "v0.1" |
---|
10 |
__author__ = "Sara Haines <sara_haines@unc.edu>" |
---|
11 |
|
---|
12 |
import os.path |
---|
13 |
from datetime import datetime, timedelta, tzinfo |
---|
14 |
from dateutil.tz import tzlocal, tzutc |
---|
15 |
from dateutil.parser import parse |
---|
16 |
import time |
---|
17 |
import math |
---|
18 |
|
---|
19 |
from ncutil import * |
---|
20 |
|
---|
21 |
def check_configs(): |
---|
22 |
"""Test config files for comformnity |
---|
23 |
|
---|
24 |
check either one or all for a platform |
---|
25 |
|
---|
26 |
id in filename == platform.id |
---|
27 |
datetime in filename <= platform.config_start_date |
---|
28 |
(close in time usually the same day |
---|
29 |
also platform.config_start_date < platform.config_end_date |
---|
30 |
(there needs to be some time that the platform was operational) |
---|
31 |
test existence of specific structural elements (platform info and sensor info) |
---|
32 |
and specific fields for both platform and sensor |
---|
33 |
verify that for each platform_info['packages'] there is sensor_info and same id |
---|
34 |
for pi['packages'][0] in si.keys() |
---|
35 |
pi['packages'][0] == si['adcp']['id'] |
---|
36 |
bounds on data in fields |
---|
37 |
show difference between two consecutive configs? |
---|
38 |
pretty print to screen of dictionary info for platform and sensor info |
---|
39 |
|
---|
40 |
cn = os.path.splitext(os.path.basename(config))[0] |
---|
41 |
cndt = filt_datetime(os.path.basename(config)) |
---|
42 |
pi = get_config(cn+'.platform_info') |
---|
43 |
if pi['config_start_date']: |
---|
44 |
config_start_dt = filt_datetime(pi['config_start_date']) |
---|
45 |
elif pi['config_start_date'] == None: |
---|
46 |
config_start_dt = now_dt |
---|
47 |
if pi['config_end_date']: |
---|
48 |
config_end_dt = filt_datetime(pi['config_end_date']) |
---|
49 |
elif pi['config_end_date'] == None: |
---|
50 |
config_end_dt = now_dt |
---|
51 |
|
---|
52 |
print cn + ' -----------------' |
---|
53 |
print cndt |
---|
54 |
print config_start_dt |
---|
55 |
print config_end_dt |
---|
56 |
print now_dt |
---|
57 |
print 'file date ok? ' + str(cndt <= config_start_dt) |
---|
58 |
print 'operation date ok? ' + str(config_start_dt < config_end_dt) |
---|
59 |
""" |
---|
60 |
|
---|
61 |
def dt2es(dt): |
---|
62 |
"""Convert datetime object to epoch seconds (es) as seconds since Jan-01-1970 """ |
---|
63 |
|
---|
64 |
delta = dt - datetime(1970,1,1,0,0,0) |
---|
65 |
es = delta.days*24*60*60 + delta.seconds |
---|
66 |
return es |
---|
67 |
|
---|
68 |
def es2dt(es): |
---|
69 |
""" Convert epoch seconds (es) to datetime object""" |
---|
70 |
dt = datetime(*time.gmtime(es)[0:6]) |
---|
71 |
return dt |
---|
72 |
|
---|
73 |
def find_months(year, month=1): |
---|
74 |
"""Find which months to process |
---|
75 |
|
---|
76 |
Since data are in subdirectories based on months determine |
---|
77 |
previous, current, and next month to look in directories for data |
---|
78 |
of the current month or month to process. |
---|
79 |
|
---|
80 |
:Parameters: |
---|
81 |
year : int value or str 'yyyy_mm' |
---|
82 |
month : int value |
---|
83 |
|
---|
84 |
:Returns: |
---|
85 |
which_months : tuple of 3 datetime objects |
---|
86 |
(prev_month, current_month, next_month) |
---|
87 |
|
---|
88 |
Examples |
---|
89 |
-------- |
---|
90 |
>>> find_months(2007, 2) |
---|
91 |
>>> find_months('2007_02') |
---|
92 |
|
---|
93 |
""" |
---|
94 |
if type(year) == int and type(month) == int : |
---|
95 |
dt = datetime(year, month, day=1) |
---|
96 |
this_month = dt |
---|
97 |
elif type(year) == str : |
---|
98 |
dt = filt_datetime(year) |
---|
99 |
this_month = dt |
---|
100 |
|
---|
101 |
if dt.month == 1: |
---|
102 |
prev_month = datetime(dt.year-1, month=12, day=1) |
---|
103 |
next_month = datetime(dt.year, dt.month+1, day=1) |
---|
104 |
elif dt.month == 12: |
---|
105 |
prev_month = datetime(dt.year, dt.month-1, day=1) |
---|
106 |
next_month = datetime(dt.year+1, month=1, day=1) |
---|
107 |
else: |
---|
108 |
prev_month = datetime(dt.year, dt.month-1, day=1) |
---|
109 |
next_month = datetime(dt.year, dt.month+1, day=1) |
---|
110 |
|
---|
111 |
return (prev_month, this_month, next_month) |
---|
112 |
|
---|
113 |
def this_month(): |
---|
114 |
"""Return this month (GMT) as formatted string (yyyy_mm) """ |
---|
115 |
this_month_str = "%4d_%02d" % time.gmtime()[0:2] |
---|
116 |
return this_month_str |
---|
117 |
|
---|
118 |
def scanf_datetime(ts, fmt='%Y-%m-%dT%H:%M:%S'): |
---|
119 |
"""Convert string representing date and time to datetime object""" |
---|
120 |
|
---|
121 |
|
---|
122 |
try: |
---|
123 |
t = time.strptime(ts, fmt) |
---|
124 |
|
---|
125 |
dt = datetime(*t[0:6]) |
---|
126 |
except ValueError, e: |
---|
127 |
|
---|
128 |
|
---|
129 |
dt = None |
---|
130 |
else: |
---|
131 |
|
---|
132 |
z = dt - datetime.utcnow() |
---|
133 |
daysdiff = abs(z.days) |
---|
134 |
|
---|
135 |
|
---|
136 |
if daysdiff > 3650: |
---|
137 |
dt = None |
---|
138 |
|
---|
139 |
return dt |
---|
140 |
|
---|
141 |
def filt_datetime_test(input_string, remove_ext=True): |
---|
142 |
""" |
---|
143 |
Following the template, (YY)YYMMDDhhmmss |
---|
144 |
and versions with of this with decreasing time precision, |
---|
145 |
find the most precise, reasonable string match and |
---|
146 |
return its datetime object. |
---|
147 |
""" |
---|
148 |
|
---|
149 |
from dateutil.parser import parse |
---|
150 |
|
---|
151 |
|
---|
152 |
from os.path import splitext |
---|
153 |
import re |
---|
154 |
if remove_ext: |
---|
155 |
(s, e) = splitext(input_string) |
---|
156 |
input_string = s |
---|
157 |
|
---|
158 |
try: |
---|
159 |
dt = parse(input_string, fuzzy=True) |
---|
160 |
except ValueError, e: |
---|
161 |
print 'filt_datetime: Could not parse date. No date found in ', input_string |
---|
162 |
dt = None |
---|
163 |
else: |
---|
164 |
return dt |
---|
165 |
|
---|
166 |
|
---|
167 |
|
---|
168 |
|
---|
169 |
def filt_datetime(input_string, gran=False, remove_ext=True): |
---|
170 |
""" |
---|
171 |
Following the template, (YY)YYMMDDhhmmss |
---|
172 |
and versions with of this with decreasing time precision, |
---|
173 |
find the most precise, reasonable string match and |
---|
174 |
return its datetime object. |
---|
175 |
|
---|
176 |
gran=False don't return granularity number |
---|
177 |
|
---|
178 |
|
---|
179 |
""" |
---|
180 |
|
---|
181 |
|
---|
182 |
from os.path import splitext |
---|
183 |
import re |
---|
184 |
if remove_ext: |
---|
185 |
(s, e) = splitext(input_string) |
---|
186 |
input_string = s |
---|
187 |
|
---|
188 |
|
---|
189 |
|
---|
190 |
|
---|
191 |
|
---|
192 |
|
---|
193 |
|
---|
194 |
|
---|
195 |
case1_regex = r""" |
---|
196 |
# case 1: YYYYMMDDhhmmss |
---|
197 |
(\d{4}) # 2- or 4-digit YEAR (e.g. '07' or '2007') |
---|
198 |
\D? # optional 1 character non-digit separator (e.g. ' ' or '-') |
---|
199 |
(\d{2}) # 2-digit MONTH (e.g. '12') |
---|
200 |
\D? # optional 1 character non-digit separator |
---|
201 |
(\d{2}) # 2-digit DAY of month (e.g. '10') |
---|
202 |
\D? # optional 1 character non-digit separator (e.g. ' ' or 'T') |
---|
203 |
(\d{2}) # 2-digit HOUR (e.g. '10') |
---|
204 |
\D? # optional 1 character non-digit separator (e.g. ' ' or ':') |
---|
205 |
(\d{2}) # 2-digit MINUTE (e.g. '10') |
---|
206 |
\D? # optional 1 character non-digit separator (e.g. ' ' or ':') |
---|
207 |
(\d{2}) # 2-digit SECOND (e.g. '10') |
---|
208 |
""" |
---|
209 |
|
---|
210 |
case2_regex = r""" |
---|
211 |
# case 2: YYYYMMDDhhmm (no seconds) |
---|
212 |
(\d{4}) # 2- or 4-digit YEAR |
---|
213 |
\D? # optional 1 character non-digit separator (e.g. ' ' or '-') |
---|
214 |
(\d{2}) # 2-digit MONTH |
---|
215 |
\D? # optional 1 character non-digit separator |
---|
216 |
(\d{2}) # 2-digit DAY |
---|
217 |
\D? # optional 1 character non-digit separator (e.g. ' ' or 'T') |
---|
218 |
(\d{2}) # 2-digit HOUR |
---|
219 |
\D? # optional 1 character non-digit separator (e.g. ' ' or ':') |
---|
220 |
(\d{2}) # 2-digit MINUTE |
---|
221 |
""" |
---|
222 |
|
---|
223 |
case3_regex = r""" |
---|
224 |
# case 3: YYYYMMDDhh (no seconds, no minutes) |
---|
225 |
(\d{4}) # 2- or 4-digit YEAR |
---|
226 |
\D? # optional 1 character non-digit separator (e.g. ' ' or '-') |
---|
227 |
(\d{2}) # 2-digit MONTH |
---|
228 |
\D? # optional 1 character non-digit separator |
---|
229 |
(\d{2}) # 2-digit DAY |
---|
230 |
\D? # optional 1 character non-digit separator (e.g. ' ' or 'T') |
---|
231 |
(\d{2}) # 2-digit HOUR |
---|
232 |
""" |
---|
233 |
|
---|
234 |
case4_regex = r""" |
---|
235 |
# case 4: YYYYMMDD (no time values, just date) |
---|
236 |
(\d{4}) # 2- or 4-digit YEAR |
---|
237 |
\D? # optional 1 character non-digit separator (e.g. ' ' or '-') |
---|
238 |
(\d{2}) # 2-digit MONTH |
---|
239 |
\D? # optional 1 character non-digit separator |
---|
240 |
(\d{2}) # 2-digit DAY |
---|
241 |
""" |
---|
242 |
|
---|
243 |
case5_regex = r""" |
---|
244 |
# case 5: YYYYMM (no time values, just month year) |
---|
245 |
(\d{4}) # 2- or 4-digit YEAR |
---|
246 |
\D? # optional 1 character non-digit separator (e.g. ' ' or '-') |
---|
247 |
(\d{2}) # 2-digit MONTH |
---|
248 |
""" |
---|
249 |
|
---|
250 |
case6_regex = r""" |
---|
251 |
# case 6: YYMMDDhhmmss |
---|
252 |
(\d{2}) # 2- or 4-digit YEAR (e.g. '07' or '2007') |
---|
253 |
\D? # optional 1 character non-digit separator (e.g. ' ' or '-') |
---|
254 |
(\d{2}) # 2-digit MONTH (e.g. '12') |
---|
255 |
\D? # optional 1 character non-digit separator |
---|
256 |
(\d{2}) # 2-digit DAY of month (e.g. '10') |
---|
257 |
\D? # optional 1 character non-digit separator (e.g. ' ' or 'T') |
---|
258 |
(\d{2}) # 2-digit HOUR (e.g. '10') |
---|
259 |
\D? # optional 1 character non-digit separator (e.g. ' ' or ':') |
---|
260 |
(\d{2}) # 2-digit MINUTE (e.g. '10') |
---|
261 |
\D? # optional 1 character non-digit separator (e.g. ' ' or ':') |
---|
262 |
(\d{2}) # 2-digit SECOND (e.g. '10') |
---|
263 |
""" |
---|
264 |
|
---|
265 |
case7_regex = r""" |
---|
266 |
# case 7: YYMMDDhhmm (no seconds) |
---|
267 |
(\d{2}) # 2- or 4-digit YEAR |
---|
268 |
\D? # optional 1 character non-digit separator (e.g. ' ' or '-') |
---|
269 |
(\d{2}) # 2-digit MONTH |
---|
270 |
\D? # optional 1 character non-digit separator |
---|
271 |
(\d{2}) # 2-digit DAY |
---|
272 |
\D? # optional 1 character non-digit separator (e.g. ' ' or 'T') |
---|
273 |
(\d{2}) # 2-digit HOUR |
---|
274 |
\D? # optional 1 character non-digit separator (e.g. ' ' or ':') |
---|
275 |
(\d{2}) # 2-digit MINUTE |
---|
276 |
""" |
---|
277 |
|
---|
278 |
case8_regex = r""" |
---|
279 |
# case 8: YYMMDDhh (no seconds, no minutes) |
---|
280 |
(\d{2}) # 2- or 4-digit YEAR |
---|
281 |
\D? # optional 1 character non-digit separator (e.g. ' ' or '-') |
---|
282 |
(\d{2}) # 2-digit MONTH |
---|
283 |
\D? # optional 1 character non-digit separator |
---|
284 |
(\d{2}) # 2-digit DAY |
---|
285 |
\D? # optional 1 character non-digit separator (e.g. ' ' or 'T') |
---|
286 |
(\d{2}) # 2-digit HOUR |
---|
287 |
""" |
---|
288 |
|
---|
289 |
case9_regex = r""" |
---|
290 |
# case 9: YYMMDD (no time values, just date) |
---|
291 |
(\d{2}) # 2- or 4-digit YEAR |
---|
292 |
\D? # optional 1 character non-digit separator (e.g. ' ' or '-') |
---|
293 |
(\d{2}) # 2-digit MONTH |
---|
294 |
\D? # optional 1 character non-digit separator |
---|
295 |
(\d{2}) # 2-digit DAY |
---|
296 |
""" |
---|
297 |
|
---|
298 |
case10_regex = r""" |
---|
299 |
# case 10: YYMM (no time values, just month year) |
---|
300 |
(\d{2}) # 2- or 4-digit YEAR |
---|
301 |
\D? # optional 1 character non-digit separator (e.g. ' ' or '-') |
---|
302 |
(\d{2}) # 2-digit MONTH |
---|
303 |
""" |
---|
304 |
|
---|
305 |
|
---|
306 |
|
---|
307 |
|
---|
308 |
|
---|
309 |
cases = [case1_regex, case2_regex, case3_regex, case4_regex, case5_regex, case6_regex, case7_regex, case8_regex, case9_regex] |
---|
310 |
patterns = [re.compile(c, re.VERBOSE) for c in cases] |
---|
311 |
matches = [p.search(input_string) for p in patterns] |
---|
312 |
|
---|
313 |
|
---|
314 |
|
---|
315 |
for ind in range(len(matches)): |
---|
316 |
if bool(matches[ind]): |
---|
317 |
|
---|
318 |
bits = matches[ind].groups() |
---|
319 |
values = [int(yi) for yi in bits] |
---|
320 |
|
---|
321 |
if values[0] < 50: |
---|
322 |
values[0] += 2000 |
---|
323 |
elif values[0]>=50 and values[0]<100: |
---|
324 |
values[0] += 1900 |
---|
325 |
|
---|
326 |
|
---|
327 |
if len(values)==1: |
---|
328 |
values.extend([1,1]) |
---|
329 |
elif len(values)==2: |
---|
330 |
values.extend([1]) |
---|
331 |
|
---|
332 |
|
---|
333 |
|
---|
334 |
try: |
---|
335 |
dt = datetime(*values) |
---|
336 |
except ValueError, e: |
---|
337 |
|
---|
338 |
|
---|
339 |
dt = None |
---|
340 |
else: |
---|
341 |
|
---|
342 |
z = dt - datetime.utcnow() |
---|
343 |
daysdiff = abs(z.days) |
---|
344 |
|
---|
345 |
|
---|
346 |
if daysdiff > 3600: |
---|
347 |
dt = None |
---|
348 |
else: |
---|
349 |
dt = None |
---|
350 |
|
---|
351 |
matches[ind] = dt |
---|
352 |
|
---|
353 |
|
---|
354 |
|
---|
355 |
|
---|
356 |
b = [bool(x) for x in matches] |
---|
357 |
try: |
---|
358 |
ind = b.index(True) |
---|
359 |
except ValueError, e: |
---|
360 |
print 'filt_datetime: No date found in ', input_string |
---|
361 |
dt = None |
---|
362 |
else: |
---|
363 |
dt = matches[ind] |
---|
364 |
if gran: |
---|
365 |
return dt,ind |
---|
366 |
else: |
---|
367 |
return dt |
---|
368 |
|
---|
369 |
def display_time_diff(diff): |
---|
370 |
"""Display time difference in HH:MM:DD using number weeks (W) |
---|
371 |
and days (D) if necessary""" |
---|
372 |
|
---|
373 |
days = diff.days |
---|
374 |
minutes, seconds = divmod(diff.seconds, 60) |
---|
375 |
hours, minutes = divmod(minutes, 60) |
---|
376 |
|
---|
377 |
|
---|
378 |
if (days==1): |
---|
379 |
str = "%02d:%02d" % (24+hours, minutes) |
---|
380 |
elif (days>1): |
---|
381 |
str = "%d Days %02d:%02d" % (days, hours, minutes) |
---|
382 |
else: |
---|
383 |
str = "%02d:%02d" % (hours, minutes) |
---|
384 |
return str |
---|
385 |
|
---|
386 |
def copy_loop_sequence(src, dst, fn_glob, numFiles=24): |
---|
387 |
""" """ |
---|
388 |
|
---|
389 |
|
---|
390 |
|
---|
391 |
|
---|
392 |
def addnan(dt, data, maxdelta=None): |
---|
393 |
""" |
---|
394 |
insert NaN for time gaps |
---|
395 |
|
---|
396 |
:Parameters: |
---|
397 |
dt : numpy.array of datetime |
---|
398 |
data : numpy.array of data |
---|
399 |
maxdelta : size of time gap (fraction or number of days) to insert |
---|
400 |
[default is two times its own sample interval] |
---|
401 |
|
---|
402 |
:Returns: |
---|
403 |
new_dt : numpy.array of datetime |
---|
404 |
new_data : numpy.array of data |
---|
405 |
|
---|
406 |
|
---|
407 |
""" |
---|
408 |
|
---|
409 |
|
---|
410 |
from matplotlib.dates import date2num, num2date |
---|
411 |
|
---|
412 |
|
---|
413 |
|
---|
414 |
|
---|
415 |
dn = date2num(dt) |
---|
416 |
delta = numpy.diff(dn) |
---|
417 |
sample_interval = numpy.median(delta) |
---|
418 |
if maxdelta==None: |
---|
419 |
maxdelta = 2.*sample_interval |
---|
420 |
|
---|
421 |
igap = (delta > maxdelta).nonzero()[0] |
---|
422 |
ngap = len(igap) |
---|
423 |
if not ngap: |
---|
424 |
return (dt, data) |
---|
425 |
else: |
---|
426 |
|
---|
427 |
sample_interval = timedelta(0.5*sample_interval) |
---|
428 |
|
---|
429 |
dt_insert = [dt[gap]+sample_interval for gap in igap] |
---|
430 |
|
---|
431 |
new_dt = numpy.insert(numpy.array(dt), igap+1, dt_insert) |
---|
432 |
|
---|
433 |
new_data = numpy.insert(numpy.array(data, dtype=float), igap+1, numpy.nan, axis=0) |
---|
434 |
|
---|
435 |
|
---|
436 |
if numpy.isnan(new_data).all(): |
---|
437 |
new_data[-1]=0. |
---|
438 |
return (new_dt, new_data) |
---|
439 |
|
---|
440 |
|
---|
441 |
|
---|
442 |
|
---|
443 |
def udconvert(val, units_from, units_to): |
---|
444 |
"""Convert units using NCAR UDUNITS-2 |
---|
445 |
|
---|
446 |
Convert data to another unit using UDUNITS-2 API. |
---|
447 |
|
---|
448 |
:Parameters: |
---|
449 |
val : scalar or list of scalars, numpy.array |
---|
450 |
Data to be converted |
---|
451 |
units_from : string |
---|
452 |
Units from which the values to be converted |
---|
453 |
units_to : string |
---|
454 |
Units to which the values will be converted |
---|
455 |
|
---|
456 |
:Returns: |
---|
457 |
val_to : float scalar, list, or numpy.array |
---|
458 |
Data that is converted to new units |
---|
459 |
units_to : string |
---|
460 |
Units to which the data are now converted |
---|
461 |
|
---|
462 |
Files |
---|
463 |
----- |
---|
464 |
XML file that can be edited to change and add new conversions |
---|
465 |
/usr/local/share/udunits/udunits-common.xml |
---|
466 |
|
---|
467 |
Not recommended to edit but useful info on UDUNITS-2 |
---|
468 |
udunits2-accepted.xml |
---|
469 |
udunits2-base.xml |
---|
470 |
udunits2-derived.xml |
---|
471 |
udunits2-prefixes.xml |
---|
472 |
udunits2.xml |
---|
473 |
|
---|
474 |
""" |
---|
475 |
import udunits |
---|
476 |
cnv = udunits.udunits(units_from, units_to) |
---|
477 |
|
---|
478 |
if cnv[0]==0: |
---|
479 |
val_to = val*cnv[1] + cnv[2] |
---|
480 |
|
---|
481 |
|
---|
482 |
|
---|
483 |
|
---|
484 |
else: |
---|
485 |
print cnv |
---|
486 |
return (None, None) |
---|
487 |
|
---|
488 |
|
---|
489 |
|
---|
490 |
|
---|
491 |
|
---|
492 |
return (val_to, units_to) |
---|
493 |
|
---|
494 |
|
---|
495 |
def meters2feet(meters): |
---|
496 |
"""Convert meters to feet: <feet> = <meters>*3.28084 """ |
---|
497 |
return meters*3.28084 |
---|
498 |
|
---|
499 |
def feet2meters(feet): |
---|
500 |
"""Convert feet to meters: <meters> = <feet>*0.3048 """ |
---|
501 |
return feet*0.3048 |
---|
502 |
|
---|
503 |
def millibar2inches_Hg(millibar): |
---|
504 |
"""Convert millibars to inches Hg: <inches_Hg> = <millibar>*0.0295301 """ |
---|
505 |
return millibar*0.0295301 |
---|
506 |
|
---|
507 |
def celsius2fahrenheit(celsius): |
---|
508 |
"""Convert deg Celsius to deg Fahrenheit: <fahrenheit> = ((1.8*<celsius>)+32) """ |
---|
509 |
return (1.8*celsius)+32 |
---|
510 |
|
---|
511 |
def millimeters2inches(millimeters): |
---|
512 |
""" Convert millimeter to inches: <inches> = <millimeters>*0.0393700787) """ |
---|
513 |
return millimeters*0.0393700787 |
---|
514 |
|
---|
515 |
def inches2millimeters(inches): |
---|
516 |
""" Convert <mm> = <inches>*25.4 """ |
---|
517 |
return inches*25.4 |
---|
518 |
|
---|
519 |
def meters_sec2knots(meters_sec): |
---|
520 |
""" Convert m/s to knots: <knots> = <meters_sec>*1.94384449) """ |
---|
521 |
return meters_sec*1.94384449 |
---|
522 |
|
---|
523 |
def wind_vector2u(wind_speed, wind_from_direction): |
---|
524 |
""" Convert wind vector to U (east) component: <u> = <wind_speed>*sine(<wind_from_direction>*pi/180) """ |
---|
525 |
return wind_speed*math.sin(wind_from_direction*math.pi/180) |
---|
526 |
|
---|
527 |
def wind_vector2v(wind_speed, wind_from_direction): |
---|
528 |
""" Convert wind vector to V (north) component: <v> = <wind_speed>*cosine(<wind_from_direction>*pi/180) """ |
---|
529 |
return wind_speed*math.cos(wind_from_direction*math.pi/180) |
---|
530 |
|
---|
531 |
def proc2latest(pi, si, yyyy_mm): |
---|
532 |
"""Select specific variables and times from current monthly netCDF |
---|
533 |
and post as latest data. TEST MODE. |
---|
534 |
|
---|
535 |
For each active config file, load specific variables from NCCOOS |
---|
536 |
monthly netCDF, make any necessary changes to data or attributes |
---|
537 |
conform to SEACOOS Data Model, subset data (last 48 hours), and |
---|
538 |
create new netCDF file in latest netCDF directory. |
---|
539 |
|
---|
540 |
NOTE: In test mode right now. See auto() function for similar action. |
---|
541 |
|
---|
542 |
""" |
---|
543 |
|
---|
544 |
platform = pi['id'] |
---|
545 |
package = si['id'] |
---|
546 |
|
---|
547 |
si['proc_filename'] = '%s_%s_%s.nc' % (platform, package, yyyy_mm) |
---|
548 |
ifn = os.path.join(si['proc_dir'], si['proc_filename']) |
---|
549 |
|
---|
550 |
si['latest_filename'] = 'nccoos-%s-%s-latest.nc' % (platform, package) |
---|
551 |
ofn = os.path.join(si['latest_dir'], si['latest_filename']) |
---|
552 |
if os.path.exists(ifn): |
---|
553 |
print ' ... ... latest : %s ' % (ofn,) |
---|
554 |
|
---|
555 |
(es, units) = nc_get_time(ifn) |
---|
556 |
dt = [es2dt(e) for e in es] |
---|
557 |
last_dt = dt[-1] |
---|
558 |
else: |
---|
559 |
|
---|
560 |
print " ... ... latest: NO latest file created" |
---|
561 |
if os.path.exists(ofn): |
---|
562 |
os.remove(ofn) |
---|
563 |
return |
---|
564 |
|
---|
565 |
|
---|
566 |
n = len(dt) |
---|
567 |
idx = numpy.array([False for i in range(n)]) |
---|
568 |
for i, val in enumerate(dt): |
---|
569 |
if val>last_dt-timedelta(days=2) and val<=last_dt+timedelta(seconds=360): |
---|
570 |
idx[i] = True |
---|
571 |
dt = numpy.array(dt) |
---|
572 |
dt = dt[idx] |
---|
573 |
|
---|
574 |
|
---|
575 |
d = nc_load(ifn, si['latest_vars']) |
---|
576 |
global_atts, var_atts, dim_inits, var_inits, var_data = d |
---|
577 |
list_of_record_vars = nc_find_record_vars(ifn) |
---|
578 |
|
---|
579 |
|
---|
580 |
|
---|
581 |
|
---|
582 |
|
---|
583 |
dim_inits = list(dim_inits) |
---|
584 |
for i in range(len(dim_inits)): |
---|
585 |
if dim_inits[i][1]==0: |
---|
586 |
dim_inits[i] = ('ntime', len(dt)) |
---|
587 |
dim_inits = tuple(dim_inits) |
---|
588 |
|
---|
589 |
|
---|
590 |
varNames = [vn for vn, vt, vd in var_inits] |
---|
591 |
var_data = list(var_data) |
---|
592 |
for i in range(len(varNames)): |
---|
593 |
vn, vd = var_data[i] |
---|
594 |
|
---|
595 |
if vn in list_of_record_vars: |
---|
596 |
var_data[i]=(vn, vd[idx]) |
---|
597 |
var_data = tuple(var_data) |
---|
598 |
|
---|
599 |
global_atts['start_date'] = dt[0].strftime('%Y-%m-%d %H:%M:%S') |
---|
600 |
d = (global_atts, var_atts, dim_inits, var_inits, var_data) |
---|
601 |
|
---|
602 |
|
---|
603 |
nc_create(ofn, d) |
---|
604 |
|
---|
605 |
|
---|
606 |
nc_rename_dimension(ofn, 'ntime', 'time') |
---|
607 |
nc_rename_dimension(ofn, 'nlat', 'lat') |
---|
608 |
nc_rename_dimension(ofn, 'nlon', 'lon') |
---|
609 |
nc_rename_dimension(ofn, 'nz', 'z') |
---|
610 |
|
---|
611 |
|
---|
612 |
nc_replace_fillvalue(ofn, -99999.0) |
---|
613 |
|
---|
614 |
def proc2csv(pi, si, yyyy_mm): |
---|
615 |
"""Select specific variables and times from current monthly netCDF |
---|
616 |
and post file of csv data. TEST MODE. |
---|
617 |
|
---|
618 |
For each active config file, load specific variables from NCCOOS |
---|
619 |
monthly netCDF, make any necessary changes to data or attributes |
---|
620 |
conform to CSV output, subset data, and |
---|
621 |
create new file in csv directory. |
---|
622 |
|
---|
623 |
NOTE: See auto() function for similar action. |
---|
624 |
|
---|
625 |
""" |
---|
626 |
|
---|
627 |
platform = pi['id'] |
---|
628 |
package = si['id'] |
---|
629 |
|
---|
630 |
si['proc_filename'] = '%s_%s_%s.nc' % (platform, package, yyyy_mm) |
---|
631 |
ifn = os.path.join(si['proc_dir'], si['proc_filename']) |
---|
632 |
|
---|
633 |
si['csv_filename'] = 'nccoos_%s_%s_latest.csv' % (platform, package) |
---|
634 |
ofn = os.path.join(si['csv_dir'], si['csv_filename']) |
---|
635 |
f = open(ofn, 'w') |
---|
636 |
|
---|
637 |
if os.path.exists(ifn): |
---|
638 |
print ' ... ... csv : %s ' % (ofn,) |
---|
639 |
|
---|
640 |
(es, units) = nc_get_time(ifn) |
---|
641 |
dt = [es2dt(e) for e in es] |
---|
642 |
last_dt = dt[-1] |
---|
643 |
else: |
---|
644 |
|
---|
645 |
print ' ... ... csv: NO csv data reported ' |
---|
646 |
f.write('"No DATA REPORTED", " \\- ", " \\- "\n') |
---|
647 |
f.close() |
---|
648 |
return |
---|
649 |
|
---|
650 |
|
---|
651 |
n = len(dt) |
---|
652 |
idx = numpy.array([False for i in range(n)]) |
---|
653 |
for i, val in enumerate(dt): |
---|
654 |
if val>last_dt-timedelta(days=1) and val<=last_dt+timedelta(seconds=360): |
---|
655 |
idx[i] = True |
---|
656 |
dt = numpy.array(dt) |
---|
657 |
dt = dt[idx] |
---|
658 |
|
---|
659 |
|
---|
660 |
d = nc_load(ifn, si['csv_vars']) |
---|
661 |
global_atts, var_atts, dim_inits, var_inits, var_data = d |
---|
662 |
|
---|
663 |
|
---|
664 |
|
---|
665 |
last_dt = last_dt.replace(tzinfo=tzutc()) |
---|
666 |
|
---|
667 |
last_dt_local = last_dt.astimezone(tzlocal()) |
---|
668 |
|
---|
669 |
diff = abs(last_dt - last_dt_local) |
---|
670 |
if diff.days>0: |
---|
671 |
last_dt_str = last_dt.strftime("%H:%M %Z on %b %d, %Y") + \ |
---|
672 |
' (' + last_dt_local.strftime("%H:%M %Z, %b %d") + ')' |
---|
673 |
else: |
---|
674 |
last_dt_str = last_dt.strftime("%H:%M %Z") + \ |
---|
675 |
' (' + last_dt_local.strftime("%H:%M %Z") + ')' \ |
---|
676 |
+ last_dt.strftime(" on %b %d, %Y") |
---|
677 |
|
---|
678 |
|
---|
679 |
now_utc_dt = datetime.now(tzutc()) |
---|
680 |
now_utc_dt = now_utc_dt.replace(second=0, microsecond=0) |
---|
681 |
|
---|
682 |
now_local_dt = datetime.now(tzlocal()) |
---|
683 |
now_local_dt = now_local_dt.replace(second=0, microsecond=0) |
---|
684 |
|
---|
685 |
|
---|
686 |
diff = abs(now_local_dt - now_utc_dt) |
---|
687 |
if diff.days>0: |
---|
688 |
now_str = now_utc_dt.strftime("%H:%M %Z on %b %d, %Y") + \ |
---|
689 |
' (' + now_local_dt.strftime("%H:%M %Z, %b %d") + ')' |
---|
690 |
else: |
---|
691 |
now_str = now_utc_dt.strftime("%H:%M %Z") + \ |
---|
692 |
' (' + now_local_dt.strftime("%H:%M %Z") + ')' \ |
---|
693 |
+ now_utc_dt.strftime(" on %b %d, %Y") |
---|
694 |
|
---|
695 |
|
---|
696 |
stale_diff = abs(now_utc_dt - last_dt) |
---|
697 |
if stale_diff.days>0 or stale_diff.seconds>=8*60*60: |
---|
698 |
stale_str = display_time_diff(stale_diff) |
---|
699 |
else: |
---|
700 |
stale_str = '' |
---|
701 |
|
---|
702 |
varNames = [vn for vn, vt, vd in var_inits] |
---|
703 |
var_data = list(var_data) |
---|
704 |
for i in range(len(varNames)): |
---|
705 |
vn, vd = var_data[i] |
---|
706 |
vd = vd[idx] |
---|
707 |
|
---|
708 |
|
---|
709 |
var_name_str = '%s (%s)' % (var_atts[vn]['long_name'], var_atts[vn]['short_name']) |
---|
710 |
valunits = var_atts[vn]['units'] |
---|
711 |
if vn=='rain': |
---|
712 |
val = vd.sum() |
---|
713 |
var_name_str = 'Rain Total (24 hrs)' |
---|
714 |
else: |
---|
715 |
val = vd[-1] |
---|
716 |
|
---|
717 |
|
---|
718 |
|
---|
719 |
if bool('__len__' in dir(val)): |
---|
720 |
val = numpy.mean(numpy.ma.masked_where(numpy.isnan(val), val)) |
---|
721 |
var_name_str = 'Depth Averaged '+var_name_str |
---|
722 |
|
---|
723 |
|
---|
724 |
import udunits |
---|
725 |
|
---|
726 |
sn = var_atts[vn]['standard_name'] |
---|
727 |
valunits_from = valunits |
---|
728 |
if 'temperature' in sn or sn in ('wind_chill', 'dew_point'): |
---|
729 |
if valunits_from == 'degrees Celsius': |
---|
730 |
valunits_from = 'degC' |
---|
731 |
valunits_to = 'degC' |
---|
732 |
elif 'velocity' in sn or 'speed' in sn or 'current' in sn: |
---|
733 |
valunits_to = 'm s-1' |
---|
734 |
elif 'flux' in sn or sn in ('discharge',): |
---|
735 |
if valunits_from == 'cfs': |
---|
736 |
valunits_from = 'ft^3/sec' |
---|
737 |
valunits_to = 'm^3/s' |
---|
738 |
elif 'rain' in sn: |
---|
739 |
valunits_to = 'mm' |
---|
740 |
elif 'level' in sn or 'height' in sn or 'depth' in sn: |
---|
741 |
valunits_to = 'm' |
---|
742 |
else: |
---|
743 |
|
---|
744 |
valunits_to = valunits_from |
---|
745 |
|
---|
746 |
cnv = udunits.udunits(valunits_from, valunits_to) |
---|
747 |
|
---|
748 |
if cnv[0]==0: |
---|
749 |
val_to = val*cnv[1] + cnv[2] |
---|
750 |
if val_to > 99: |
---|
751 |
metric_str = '%.4g (%s)' % (val_to, valunits_to) |
---|
752 |
else: |
---|
753 |
metric_str = '%.2g (%s)' % (val_to, valunits_to) |
---|
754 |
|
---|
755 |
|
---|
756 |
|
---|
757 |
|
---|
758 |
elif cnv[0]==-1 or cnv[0]==-2: |
---|
759 |
if val > 99: |
---|
760 |
metric_str = '%.4g (%s)' % (val, valunits) |
---|
761 |
else: |
---|
762 |
metric_str = '%.2g (%s)' % (val, valunits) |
---|
763 |
else: |
---|
764 |
metric_str = '\-' |
---|
765 |
|
---|
766 |
|
---|
767 |
if 'temperature' in sn or sn in ('wind_chill', 'dew_point'): |
---|
768 |
if valunits_from == 'degrees Celsius': |
---|
769 |
valunits_from = 'degC' |
---|
770 |
valunits_to = 'degF' |
---|
771 |
elif 'velocity' in sn or 'speed' in sn or 'current' in sn: |
---|
772 |
valunits_to = 'knots' |
---|
773 |
elif 'flux' in sn or sn in ('discharge',): |
---|
774 |
if valunits_from == 'cfs': |
---|
775 |
valunits_from = 'ft^3/sec' |
---|
776 |
valunits_to = 'ft^3/s' |
---|
777 |
elif 'rain' in sn: |
---|
778 |
valunits_to = 'in' |
---|
779 |
elif 'level' in sn or 'height' in sn or 'depth' in sn: |
---|
780 |
valunits_to = 'ft' |
---|
781 |
else: |
---|
782 |
valunits_to = valunits_from |
---|
783 |
|
---|
784 |
cnv = udunits.udunits(valunits_from, valunits_to) |
---|
785 |
if cnv[0]==0: |
---|
786 |
val_to = val*cnv[1] + cnv[2] |
---|
787 |
if val > 99: |
---|
788 |
english_str ='%.4g (%s)' % (val_to, valunits_to) |
---|
789 |
else: |
---|
790 |
english_str = '%.2g (%s)' % (val_to, valunits_to) |
---|
791 |
|
---|
792 |
|
---|
793 |
|
---|
794 |
|
---|
795 |
elif cnv[0]==-1 or cnv[0]==-2: |
---|
796 |
if val > 99: |
---|
797 |
english_str = '%.4g (%s)' % (val, valunits) |
---|
798 |
else: |
---|
799 |
english_str = '%.2g (%s)' % (val, valunits) |
---|
800 |
else: |
---|
801 |
english_str = '\-' |
---|
802 |
|
---|
803 |
if metric_str == english_str: |
---|
804 |
english_str = '\-' |
---|
805 |
|
---|
806 |
if vn=='time': |
---|
807 |
f.write('"**%s:** %s", ""\n' % ('Sample Time', last_dt_str)) |
---|
808 |
elif vn=='blank': |
---|
809 |
f.write('"%s", "%s", "%s"\n' % (' ', ' ', ' ')) |
---|
810 |
else: |
---|
811 |
f.write('"%s", "%s", "%s"\n' % (var_name_str, metric_str, english_str)) |
---|
812 |
|
---|
813 |
f.close() |
---|
814 |
|
---|
815 |
|
---|
816 |
|
---|
817 |
|
---|