1 |
|
---|
2 |
|
---|
3 |
"""Utilities to help data processing |
---|
4 |
|
---|
5 |
Mostly time functions right now |
---|
6 |
|
---|
7 |
TO DO: |
---|
8 |
check_configs() |
---|
9 |
unit conversions (udunits?) |
---|
10 |
""" |
---|
11 |
|
---|
12 |
__version__ = "v0.1" |
---|
13 |
__author__ = "Sara Haines <sara_haines@unc.edu>" |
---|
14 |
|
---|
15 |
import os.path |
---|
16 |
from datetime import datetime, timedelta, tzinfo |
---|
17 |
from dateutil.tz import tzlocal, tzutc |
---|
18 |
import time |
---|
19 |
import math |
---|
20 |
|
---|
21 |
from ncutil import * |
---|
22 |
|
---|
23 |
def check_configs(): |
---|
24 |
"""Test config files for comformnity |
---|
25 |
|
---|
26 |
check either one or all for a platform |
---|
27 |
|
---|
28 |
id in filename == platform.id |
---|
29 |
datetime in filename <= platform.config_start_date |
---|
30 |
(close in time usually the same day |
---|
31 |
also platform.config_start_date < platform.config_end_date |
---|
32 |
(there needs to be some time that the platform was operational) |
---|
33 |
test existence of specific structural elements (platform info and sensor info) |
---|
34 |
and specific fields for both platform and sensor |
---|
35 |
verify that for each platform_info['packages'] there is sensor_info and same id |
---|
36 |
for pi['packages'][0] in si.keys() |
---|
37 |
pi['packages'][0] == si['adcp']['id'] |
---|
38 |
bounds on data in fields |
---|
39 |
show difference between two consecutive configs? |
---|
40 |
pretty print to screen of dictionary info for platform and sensor info |
---|
41 |
|
---|
42 |
cn = os.path.splitext(os.path.basename(config))[0] |
---|
43 |
cndt = filt_datetime(os.path.basename(config))[0] |
---|
44 |
pi = get_config(cn+'.platform_info') |
---|
45 |
if pi['config_start_date']: |
---|
46 |
config_start_dt = filt_datetime(pi['config_start_date'])[0] |
---|
47 |
elif pi['config_start_date'] == None: |
---|
48 |
config_start_dt = now_dt |
---|
49 |
if pi['config_end_date']: |
---|
50 |
config_end_dt = filt_datetime(pi['config_end_date'])[0] |
---|
51 |
elif pi['config_end_date'] == None: |
---|
52 |
config_end_dt = now_dt |
---|
53 |
|
---|
54 |
print cn + ' -----------------' |
---|
55 |
print cndt |
---|
56 |
print config_start_dt |
---|
57 |
print config_end_dt |
---|
58 |
print now_dt |
---|
59 |
print 'file date ok? ' + str(cndt <= config_start_dt) |
---|
60 |
print 'operation date ok? ' + str(config_start_dt < config_end_dt) |
---|
61 |
""" |
---|
62 |
|
---|
63 |
def dt2es(dt): |
---|
64 |
"""Convert datetime object to epoch seconds (es) as seconds since Jan-01-1970 """ |
---|
65 |
|
---|
66 |
delta = dt - datetime(1970,1,1,0,0,0) |
---|
67 |
es = delta.days*24*60*60 + delta.seconds |
---|
68 |
return es |
---|
69 |
|
---|
70 |
def es2dt(es): |
---|
71 |
""" Convert epoch seconds (es) to datetime object""" |
---|
72 |
dt = datetime(*time.gmtime(es)[0:6]) |
---|
73 |
return dt |
---|
74 |
|
---|
75 |
def find_months(year, month=1): |
---|
76 |
"""Find which months to process |
---|
77 |
|
---|
78 |
Since data are in subdirectories based on months determine |
---|
79 |
previous, current, and next month to look in directories for data |
---|
80 |
of the current month or month to process. |
---|
81 |
|
---|
82 |
:Parameters: |
---|
83 |
year : int value or str 'yyyy_mm' |
---|
84 |
month : int value |
---|
85 |
|
---|
86 |
:Returns: |
---|
87 |
which_months : tuple of 3 datetime objects |
---|
88 |
(prev_month, current_month, next_month) |
---|
89 |
|
---|
90 |
Examples |
---|
91 |
-------- |
---|
92 |
>>> find_months(2007, 2) |
---|
93 |
>>> find_months('2007_02') |
---|
94 |
|
---|
95 |
""" |
---|
96 |
if type(year) == int and type(month) == int : |
---|
97 |
dt = datetime(year, month, day=1) |
---|
98 |
this_month = dt |
---|
99 |
elif type(year) == str : |
---|
100 |
dt = filt_datetime(year)[0] |
---|
101 |
this_month = dt |
---|
102 |
|
---|
103 |
if dt.month == 1: |
---|
104 |
prev_month = datetime(dt.year-1, month=12, day=1) |
---|
105 |
next_month = datetime(dt.year, dt.month+1, day=1) |
---|
106 |
elif dt.month == 12: |
---|
107 |
prev_month = datetime(dt.year, dt.month-1, day=1) |
---|
108 |
next_month = datetime(dt.year+1, month=1, day=1) |
---|
109 |
else: |
---|
110 |
prev_month = datetime(dt.year, dt.month-1, day=1) |
---|
111 |
next_month = datetime(dt.year, dt.month+1, day=1) |
---|
112 |
|
---|
113 |
return (prev_month, this_month, next_month) |
---|
114 |
|
---|
115 |
def this_month(): |
---|
116 |
"""Return this month (GMT) as formatted string (yyyy_mm) """ |
---|
117 |
this_month_str = "%4d_%02d" % time.gmtime()[0:2] |
---|
118 |
return this_month_str |
---|
119 |
|
---|
120 |
def scanf_datetime(ts, fmt='%Y-%m-%dT%H:%M:%S'): |
---|
121 |
"""Convert string representing date and time to datetime object""" |
---|
122 |
|
---|
123 |
|
---|
124 |
t = time.strptime(ts, fmt) |
---|
125 |
|
---|
126 |
dt = datetime(*t[0:6]) |
---|
127 |
return dt |
---|
128 |
|
---|
129 |
def filt_datetime(input_string, remove_ext=True): |
---|
130 |
""" |
---|
131 |
Following the template, (YY)YYMMDDhhmmss |
---|
132 |
and versions with of this with decreasing time precision, |
---|
133 |
find the most precise, reasonable string match and |
---|
134 |
return its datetime object. |
---|
135 |
""" |
---|
136 |
|
---|
137 |
|
---|
138 |
from os.path import splitext |
---|
139 |
import re |
---|
140 |
if remove_ext: |
---|
141 |
(s, e) = splitext(input_string) |
---|
142 |
input_string = s |
---|
143 |
|
---|
144 |
|
---|
145 |
|
---|
146 |
|
---|
147 |
|
---|
148 |
|
---|
149 |
|
---|
150 |
|
---|
151 |
case1_regex = r""" |
---|
152 |
# case 1: (YY)YYMMDDhhmmss |
---|
153 |
(\d{4}|\d{2}) # 2- or 4-digit YEAR (e.g. '07' or '2007') |
---|
154 |
\D? # optional 1 character non-digit separator (e.g. ' ' or '-') |
---|
155 |
(\d{2}) # 2-digit MONTH (e.g. '12') |
---|
156 |
\D? # optional 1 character non-digit separator |
---|
157 |
(\d{2}) # 2-digit DAY of month (e.g. '10') |
---|
158 |
\D? # optional 1 character non-digit separator (e.g. ' ' or 'T') |
---|
159 |
(\d{2}) # 2-digit HOUR (e.g. '10') |
---|
160 |
\D? # optional 1 character non-digit separator (e.g. ' ' or ':') |
---|
161 |
(\d{2}) # 2-digit MINUTE (e.g. '10') |
---|
162 |
\D? # optional 1 character non-digit separator (e.g. ' ' or ':') |
---|
163 |
(\d{2}) # 2-digit SECOND (e.g. '10') |
---|
164 |
""" |
---|
165 |
|
---|
166 |
case2_regex = r""" |
---|
167 |
# case 2: (YY)YYMMDDhhmm (no seconds) |
---|
168 |
(\d{4}|\d{2}) # 2- or 4-digit YEAR |
---|
169 |
\D? # optional 1 character non-digit separator (e.g. ' ' or '-') |
---|
170 |
(\d{2}) # 2-digit MONTH |
---|
171 |
\D? # optional 1 character non-digit separator |
---|
172 |
(\d{2}) # 2-digit DAY |
---|
173 |
\D? # optional 1 character non-digit separator (e.g. ' ' or 'T') |
---|
174 |
(\d{2}) # 2-digit HOUR |
---|
175 |
\D? # optional 1 character non-digit separator (e.g. ' ' or ':') |
---|
176 |
(\d{2}) # 2-digit MINUTE |
---|
177 |
""" |
---|
178 |
|
---|
179 |
case3_regex = r""" |
---|
180 |
# case 3: (YY)YYMMDDhh (no seconds, no minutes) |
---|
181 |
(\d{4}|\d{2}) # 2- or 4-digit YEAR |
---|
182 |
\D? # optional 1 character non-digit separator (e.g. ' ' or '-') |
---|
183 |
(\d{2}) # 2-digit MONTH |
---|
184 |
\D? # optional 1 character non-digit separator |
---|
185 |
(\d{2}) # 2-digit DAY |
---|
186 |
\D? # optional 1 character non-digit separator (e.g. ' ' or 'T') |
---|
187 |
(\d{2}) # 2-digit HOUR |
---|
188 |
""" |
---|
189 |
|
---|
190 |
case4_regex = r""" |
---|
191 |
# case 4: (YY)YYMMDD (no time values, just date) |
---|
192 |
(\d{4}|\d{2}) # 2- or 4-digit YEAR |
---|
193 |
\D? # optional 1 character non-digit separator (e.g. ' ' or '-') |
---|
194 |
(\d{2}) # 2-digit MONTH |
---|
195 |
\D? # optional 1 character non-digit separator |
---|
196 |
(\d{2}) # 2-digit DAY |
---|
197 |
""" |
---|
198 |
|
---|
199 |
case5_regex = r""" |
---|
200 |
# case 5: (YY)YYMM (no time values, just month year) |
---|
201 |
(\d{4}|\d{2}) # 2- or 4-digit YEAR |
---|
202 |
\D? # optional 1 character non-digit separator (e.g. ' ' or '-') |
---|
203 |
(\d{2}) # 2-digit MONTH |
---|
204 |
""" |
---|
205 |
|
---|
206 |
|
---|
207 |
|
---|
208 |
|
---|
209 |
|
---|
210 |
cases = [case1_regex, case2_regex, case3_regex, case4_regex, case5_regex] |
---|
211 |
patterns = [re.compile(c, re.VERBOSE) for c in cases] |
---|
212 |
matches = [p.search(input_string) for p in patterns] |
---|
213 |
|
---|
214 |
|
---|
215 |
|
---|
216 |
for ind in range(len(matches)): |
---|
217 |
if bool(matches[ind]): |
---|
218 |
|
---|
219 |
bits = matches[ind].groups() |
---|
220 |
values = [int(yi) for yi in bits] |
---|
221 |
|
---|
222 |
if values[0] < 50: |
---|
223 |
values[0] += 2000 |
---|
224 |
elif values[0]>=50 and values[0]<100: |
---|
225 |
values[0] += 1900 |
---|
226 |
|
---|
227 |
|
---|
228 |
if len(values)==1: |
---|
229 |
values.extend([1,1]) |
---|
230 |
elif len(values)==2: |
---|
231 |
values.extend([1]) |
---|
232 |
|
---|
233 |
|
---|
234 |
|
---|
235 |
try: |
---|
236 |
dt = datetime(*values) |
---|
237 |
except ValueError, e: |
---|
238 |
|
---|
239 |
|
---|
240 |
dt = None |
---|
241 |
else: |
---|
242 |
|
---|
243 |
z = dt - datetime.utcnow() |
---|
244 |
daysdiff = abs(z.days) |
---|
245 |
|
---|
246 |
|
---|
247 |
if daysdiff > 3650: |
---|
248 |
dt = None |
---|
249 |
else: |
---|
250 |
dt = None |
---|
251 |
|
---|
252 |
|
---|
253 |
matches[ind] = dt |
---|
254 |
|
---|
255 |
|
---|
256 |
|
---|
257 |
|
---|
258 |
b = [bool(x) for x in matches] |
---|
259 |
try: |
---|
260 |
ind = b.index(True) |
---|
261 |
except ValueError, e: |
---|
262 |
print 'filt_datetime: No date found in ', input_string |
---|
263 |
dt = None |
---|
264 |
else: |
---|
265 |
dt = matches[ind] |
---|
266 |
return dt,ind |
---|
267 |
|
---|
268 |
def display_time_diff(diff): |
---|
269 |
"""Display time difference in HH:MM:DD using number weeks (W) |
---|
270 |
and days (D) if necessary""" |
---|
271 |
|
---|
272 |
days = diff.days |
---|
273 |
minutes, seconds = divmod(diff.seconds, 60) |
---|
274 |
hours, minutes = divmod(minutes, 60) |
---|
275 |
|
---|
276 |
|
---|
277 |
if (days==1): |
---|
278 |
str = "%02d:%02d" % (24+hours, minutes) |
---|
279 |
elif (days>1): |
---|
280 |
str = "%d Days %02d:%02d" % (days, hours, minutes) |
---|
281 |
else: |
---|
282 |
str = "%02d:%02d" % (hours, minutes) |
---|
283 |
return str |
---|
284 |
|
---|
285 |
def copy_loop_sequence(src, dst, fn_glob, numFiles=24): |
---|
286 |
""" """ |
---|
287 |
|
---|
288 |
|
---|
289 |
|
---|
290 |
|
---|
291 |
def addnan(dt, data, maxdelta=None): |
---|
292 |
""" |
---|
293 |
insert NaN for time gaps |
---|
294 |
|
---|
295 |
:Parameters: |
---|
296 |
dt : numpy.array of datetime |
---|
297 |
data : numpy.array of data |
---|
298 |
maxdelta : size of time gap (fraction or number of days) to insert |
---|
299 |
[default is two times its own sample interval] |
---|
300 |
|
---|
301 |
:Returns: |
---|
302 |
new_dt : numpy.array of datetime |
---|
303 |
new_data : numpy.array of data |
---|
304 |
|
---|
305 |
|
---|
306 |
""" |
---|
307 |
|
---|
308 |
|
---|
309 |
from matplotlib.dates import date2num, num2date |
---|
310 |
|
---|
311 |
|
---|
312 |
|
---|
313 |
|
---|
314 |
dn = date2num(dt) |
---|
315 |
delta = numpy.diff(dn) |
---|
316 |
sample_interval = numpy.median(delta) |
---|
317 |
if maxdelta==None: |
---|
318 |
maxdelta = 2.*sample_interval |
---|
319 |
|
---|
320 |
igap = (delta > maxdelta).nonzero()[0] |
---|
321 |
ngap = len(igap) |
---|
322 |
if not ngap: |
---|
323 |
return (dt, data) |
---|
324 |
else: |
---|
325 |
|
---|
326 |
sample_interval = timedelta(0.5*sample_interval) |
---|
327 |
|
---|
328 |
dt_insert = [dt[gap]+sample_interval for gap in igap] |
---|
329 |
|
---|
330 |
new_dt = numpy.insert(numpy.array(dt), igap+1, dt_insert) |
---|
331 |
|
---|
332 |
new_data = numpy.insert(numpy.array(data), igap+1, numpy.nan, axis=0) |
---|
333 |
|
---|
334 |
|
---|
335 |
if numpy.isnan(new_data).all(): |
---|
336 |
new_data[-1]=0. |
---|
337 |
return (new_dt, new_data) |
---|
338 |
|
---|
339 |
|
---|
340 |
|
---|
341 |
|
---|
342 |
def udconvert(val, units_from, units_to): |
---|
343 |
"""Convert units using NCAR UDUNITS-2 |
---|
344 |
|
---|
345 |
Convert data to another unit using UDUNITS-2 API. |
---|
346 |
|
---|
347 |
:Parameters: |
---|
348 |
val : scalar or list of scalars, numpy.array |
---|
349 |
Data to be converted |
---|
350 |
units_from : string |
---|
351 |
Units from which the values to be converted |
---|
352 |
units_to : string |
---|
353 |
Units to which the values will be converted |
---|
354 |
|
---|
355 |
:Returns: |
---|
356 |
val_to : float scalar, list, or numpy.array |
---|
357 |
Data that is converted to new units |
---|
358 |
units_to : string |
---|
359 |
Units to which the data are now converted |
---|
360 |
|
---|
361 |
Files |
---|
362 |
----- |
---|
363 |
XML file that can be edited to change and add new conversions |
---|
364 |
/usr/local/share/udunits/udunits-common.xml |
---|
365 |
|
---|
366 |
Not recommended to edit but useful info on UDUNITS-2 |
---|
367 |
udunits2-accepted.xml |
---|
368 |
udunits2-base.xml |
---|
369 |
udunits2-derived.xml |
---|
370 |
udunits2-prefixes.xml |
---|
371 |
udunits2.xml |
---|
372 |
|
---|
373 |
""" |
---|
374 |
import udunits |
---|
375 |
cnv = udunits.udunits(units_from, units_to) |
---|
376 |
|
---|
377 |
if cnv[0]==0: |
---|
378 |
val_to = val*cnv[1] + cnv[2] |
---|
379 |
|
---|
380 |
|
---|
381 |
|
---|
382 |
|
---|
383 |
else: |
---|
384 |
print cnv |
---|
385 |
return (None, None) |
---|
386 |
|
---|
387 |
|
---|
388 |
|
---|
389 |
|
---|
390 |
|
---|
391 |
return (val_to, units_to) |
---|
392 |
|
---|
393 |
|
---|
394 |
def meters2feet(meters): |
---|
395 |
"""Convert meters to feet: <feet> = <meters>*3.28084 """ |
---|
396 |
return meters*3.28084 |
---|
397 |
|
---|
398 |
def feet2meters(feet): |
---|
399 |
"""Convert feet to meters: <meters> = <feet>*0.3048 """ |
---|
400 |
return feet*0.3048 |
---|
401 |
|
---|
402 |
def millibar2inches_Hg(millibar): |
---|
403 |
"""Convert millibars to inches Hg: <inches_Hg> = <millibar>*0.0295301 """ |
---|
404 |
return millibar*0.0295301 |
---|
405 |
|
---|
406 |
def celsius2fahrenheit(celsius): |
---|
407 |
"""Convert deg Celsius to deg Fahrenheit: <fahrenheit> = ((1.8*<celsius>)+32) """ |
---|
408 |
return (1.8*celsius)+32 |
---|
409 |
|
---|
410 |
def millimeters2inches(millimeters): |
---|
411 |
""" Convert millimeter to inches: <inches> = <millimeters>*0.0393700787) """ |
---|
412 |
return millimeters*0.0393700787 |
---|
413 |
|
---|
414 |
def inches2millimeters(inches): |
---|
415 |
""" Convert <mm> = <inches>*25.4 """ |
---|
416 |
return inches*25.4 |
---|
417 |
|
---|
418 |
def meters_sec2knots(meters_sec): |
---|
419 |
""" Convert m/s to knots: <knots> = <meters_sec>*1.94384449) """ |
---|
420 |
return meters_sec*1.94384449 |
---|
421 |
|
---|
422 |
def wind_vector2u(wind_speed, wind_from_direction): |
---|
423 |
""" Convert wind vector to U (east) component: <u> = <wind_speed>*sine(<wind_from_direction>*pi/180) """ |
---|
424 |
return wind_speed*math.sin(wind_from_direction*math.pi/180) |
---|
425 |
|
---|
426 |
def wind_vector2v(wind_speed, wind_from_direction): |
---|
427 |
""" Convert wind vector to V (north) component: <v> = <wind_speed>*cosine(<wind_from_direction>*pi/180) """ |
---|
428 |
return wind_speed*math.cos(wind_from_direction*math.pi/180) |
---|
429 |
|
---|
430 |
def proc2latest(pi, si, yyyy_mm): |
---|
431 |
"""Select specific variables and times from current monthly netCDF |
---|
432 |
and post as latest data. TEST MODE. |
---|
433 |
|
---|
434 |
For each active config file, load specific variables from NCCOOS |
---|
435 |
monthly netCDF, make any necessary changes to data or attributes |
---|
436 |
conform to SEACOOS Data Model, subset data (last 48 hours), and |
---|
437 |
create new netCDF file in latest netCDF directory. |
---|
438 |
|
---|
439 |
NOTE: In test mode right now. See auto() function for similar action. |
---|
440 |
|
---|
441 |
""" |
---|
442 |
|
---|
443 |
platform = pi['id'] |
---|
444 |
package = si['id'] |
---|
445 |
|
---|
446 |
si['proc_filename'] = '%s_%s_%s.nc' % (platform, package, yyyy_mm) |
---|
447 |
ifn = os.path.join(si['proc_dir'], si['proc_filename']) |
---|
448 |
|
---|
449 |
si['latest_filename'] = 'nccoos_%s_%s_latest.nc' % (platform, package) |
---|
450 |
ofn = os.path.join(si['latest_dir'], si['latest_filename']) |
---|
451 |
if os.path.exists(ifn): |
---|
452 |
print ' ... ... latest : %s ' % (ofn,) |
---|
453 |
|
---|
454 |
(es, units) = nc_get_time(ifn) |
---|
455 |
dt = [es2dt(e) for e in es] |
---|
456 |
last_dt = dt[-1] |
---|
457 |
else: |
---|
458 |
|
---|
459 |
print " ... ... latest: NO latest file created" |
---|
460 |
if os.path.exists(ofn): |
---|
461 |
os.remove(ofn) |
---|
462 |
return |
---|
463 |
|
---|
464 |
|
---|
465 |
n = len(dt) |
---|
466 |
idx = numpy.array([False for i in range(n)]) |
---|
467 |
for i, val in enumerate(dt): |
---|
468 |
if val>last_dt-timedelta(days=2) and val<=last_dt+timedelta(seconds=360): |
---|
469 |
idx[i] = True |
---|
470 |
dt = numpy.array(dt) |
---|
471 |
dt = dt[idx] |
---|
472 |
|
---|
473 |
|
---|
474 |
d = nc_load(ifn, si['latest_vars']) |
---|
475 |
global_atts, var_atts, dim_inits, var_inits, var_data = d |
---|
476 |
list_of_record_vars = nc_find_record_vars(ifn) |
---|
477 |
|
---|
478 |
|
---|
479 |
|
---|
480 |
|
---|
481 |
|
---|
482 |
dim_inits = list(dim_inits) |
---|
483 |
for i in range(len(dim_inits)): |
---|
484 |
if dim_inits[i][1]==0: |
---|
485 |
dim_inits[i] = ('ntime', len(dt)) |
---|
486 |
dim_inits = tuple(dim_inits) |
---|
487 |
|
---|
488 |
|
---|
489 |
varNames = [vn for vn, vt, vd in var_inits] |
---|
490 |
var_data = list(var_data) |
---|
491 |
for i in range(len(varNames)): |
---|
492 |
vn, vd = var_data[i] |
---|
493 |
|
---|
494 |
if vn in list_of_record_vars: |
---|
495 |
var_data[i]=(vn, vd[idx]) |
---|
496 |
var_data = tuple(var_data) |
---|
497 |
|
---|
498 |
global_atts['start_date'] = dt[0].strftime('%Y-%m-%d %H:%M:%S') |
---|
499 |
d = (global_atts, var_atts, dim_inits, var_inits, var_data) |
---|
500 |
|
---|
501 |
|
---|
502 |
nc_create(ofn, d) |
---|
503 |
|
---|
504 |
|
---|
505 |
nc_rename_dimension(ofn, 'ntime', 'time') |
---|
506 |
nc_rename_dimension(ofn, 'nlat', 'lat') |
---|
507 |
nc_rename_dimension(ofn, 'nlon', 'lon') |
---|
508 |
nc_rename_dimension(ofn, 'nz', 'z') |
---|
509 |
|
---|
510 |
|
---|
511 |
nc_replace_fillvalue(ofn, -99999.0) |
---|
512 |
|
---|
513 |
def proc2csv(pi, si, yyyy_mm): |
---|
514 |
"""Select specific variables and times from current monthly netCDF |
---|
515 |
and post file of csv data. TEST MODE. |
---|
516 |
|
---|
517 |
For each active config file, load specific variables from NCCOOS |
---|
518 |
monthly netCDF, make any necessary changes to data or attributes |
---|
519 |
conform to CSV output, subset data, and |
---|
520 |
create new file in csv directory. |
---|
521 |
|
---|
522 |
NOTE: See auto() function for similar action. |
---|
523 |
|
---|
524 |
""" |
---|
525 |
|
---|
526 |
platform = pi['id'] |
---|
527 |
package = si['id'] |
---|
528 |
|
---|
529 |
si['proc_filename'] = '%s_%s_%s.nc' % (platform, package, yyyy_mm) |
---|
530 |
ifn = os.path.join(si['proc_dir'], si['proc_filename']) |
---|
531 |
|
---|
532 |
si['csv_filename'] = 'nccoos_%s_%s_latest.csv' % (platform, package) |
---|
533 |
ofn = os.path.join(si['csv_dir'], si['csv_filename']) |
---|
534 |
f = open(ofn, 'w') |
---|
535 |
|
---|
536 |
if os.path.exists(ifn): |
---|
537 |
print ' ... ... csv : %s ' % (ofn,) |
---|
538 |
|
---|
539 |
(es, units) = nc_get_time(ifn) |
---|
540 |
dt = [es2dt(e) for e in es] |
---|
541 |
last_dt = dt[-1] |
---|
542 |
else: |
---|
543 |
|
---|
544 |
print ' ... ... csv: NO csv data reported ' |
---|
545 |
f.write('"No DATA REPORTED", " \\- ", " \\- "\n') |
---|
546 |
f.close() |
---|
547 |
return |
---|
548 |
|
---|
549 |
|
---|
550 |
n = len(dt) |
---|
551 |
idx = numpy.array([False for i in range(n)]) |
---|
552 |
for i, val in enumerate(dt): |
---|
553 |
if val>last_dt-timedelta(days=1) and val<=last_dt+timedelta(seconds=360): |
---|
554 |
idx[i] = True |
---|
555 |
dt = numpy.array(dt) |
---|
556 |
dt = dt[idx] |
---|
557 |
|
---|
558 |
|
---|
559 |
d = nc_load(ifn, si['csv_vars']) |
---|
560 |
global_atts, var_atts, dim_inits, var_inits, var_data = d |
---|
561 |
|
---|
562 |
|
---|
563 |
|
---|
564 |
last_dt = last_dt.replace(tzinfo=tzutc()) |
---|
565 |
|
---|
566 |
last_dt_local = last_dt.astimezone(tzlocal()) |
---|
567 |
|
---|
568 |
diff = abs(last_dt - last_dt_local) |
---|
569 |
if diff.days>0: |
---|
570 |
last_dt_str = last_dt.strftime("%H:%M %Z on %b %d, %Y") + \ |
---|
571 |
' (' + last_dt_local.strftime("%H:%M %Z, %b %d") + ')' |
---|
572 |
else: |
---|
573 |
last_dt_str = last_dt.strftime("%H:%M %Z") + \ |
---|
574 |
' (' + last_dt_local.strftime("%H:%M %Z") + ')' \ |
---|
575 |
+ last_dt.strftime(" on %b %d, %Y") |
---|
576 |
|
---|
577 |
|
---|
578 |
now_utc_dt = datetime.now(tzutc()) |
---|
579 |
now_utc_dt = now_utc_dt.replace(second=0, microsecond=0) |
---|
580 |
|
---|
581 |
now_local_dt = datetime.now(tzlocal()) |
---|
582 |
now_local_dt = now_local_dt.replace(second=0, microsecond=0) |
---|
583 |
|
---|
584 |
|
---|
585 |
diff = abs(now_local_dt - now_utc_dt) |
---|
586 |
if diff.days>0: |
---|
587 |
now_str = now_utc_dt.strftime("%H:%M %Z on %b %d, %Y") + \ |
---|
588 |
' (' + now_local_dt.strftime("%H:%M %Z, %b %d") + ')' |
---|
589 |
else: |
---|
590 |
now_str = now_utc_dt.strftime("%H:%M %Z") + \ |
---|
591 |
' (' + now_local_dt.strftime("%H:%M %Z") + ')' \ |
---|
592 |
+ now_utc_dt.strftime(" on %b %d, %Y") |
---|
593 |
|
---|
594 |
|
---|
595 |
stale_diff = abs(now_utc_dt - last_dt) |
---|
596 |
if stale_diff.days>0 or stale_diff.seconds>=8*60*60: |
---|
597 |
stale_str = display_time_diff(stale_diff) |
---|
598 |
else: |
---|
599 |
stale_str = '' |
---|
600 |
|
---|
601 |
varNames = [vn for vn, vt, vd in var_inits] |
---|
602 |
var_data = list(var_data) |
---|
603 |
for i in range(len(varNames)): |
---|
604 |
vn, vd = var_data[i] |
---|
605 |
vd = vd[idx] |
---|
606 |
|
---|
607 |
|
---|
608 |
var_name_str = '%s (%s)' % (var_atts[vn]['long_name'], var_atts[vn]['short_name']) |
---|
609 |
valunits = var_atts[vn]['units'] |
---|
610 |
if vn=='rain': |
---|
611 |
val = vd.sum() |
---|
612 |
var_name_str = 'Rain Total (24 hrs)' |
---|
613 |
else: |
---|
614 |
val = vd[-1] |
---|
615 |
|
---|
616 |
|
---|
617 |
|
---|
618 |
|
---|
619 |
if bool('__len__' in dir(val)): |
---|
620 |
val = numpy.mean(numpy.ma.masked_where(numpy.isnan(val), val)) |
---|
621 |
|
---|
622 |
|
---|
623 |
import udunits |
---|
624 |
|
---|
625 |
sn = var_atts[vn]['standard_name'] |
---|
626 |
valunits_from = valunits |
---|
627 |
if 'temperature' in sn or sn in ('wind_chill', 'dew_point'): |
---|
628 |
if valunits_from == 'degrees Celsius': |
---|
629 |
valunits_from = 'degC' |
---|
630 |
valunits_to = 'degC' |
---|
631 |
elif 'velocity' in sn: |
---|
632 |
valunits_to = 'm s-1' |
---|
633 |
elif 'flux' in sn or sn in ('discharge',): |
---|
634 |
if valunits_from == 'cfs': |
---|
635 |
valunits_from = 'ft^3/sec' |
---|
636 |
valunits_to = 'm^3/s' |
---|
637 |
elif 'rain' in sn: |
---|
638 |
valunits_to = 'mm' |
---|
639 |
elif 'level' in sn or 'height' in sn or 'depth' in sn: |
---|
640 |
valunits_to = 'm' |
---|
641 |
else: |
---|
642 |
|
---|
643 |
valunits_to = valunits_from |
---|
644 |
|
---|
645 |
cnv = udunits.udunits(valunits_from, valunits_to) |
---|
646 |
|
---|
647 |
if cnv[0]==0: |
---|
648 |
val_to = val*cnv[1] + cnv[2] |
---|
649 |
if val_to > 99: |
---|
650 |
metric_str = '%.4g (%s)' % (val_to, valunits_to) |
---|
651 |
else: |
---|
652 |
metric_str = '%.2g (%s)' % (val_to, valunits_to) |
---|
653 |
|
---|
654 |
|
---|
655 |
|
---|
656 |
|
---|
657 |
elif cnv[0]==-1 or cnv[0]==-2: |
---|
658 |
if val > 99: |
---|
659 |
metric_str = '%.4g (%s)' % (val, valunits) |
---|
660 |
else: |
---|
661 |
metric_str = '%.2g (%s)' % (val, valunits) |
---|
662 |
else: |
---|
663 |
metric_str = '\-' |
---|
664 |
|
---|
665 |
|
---|
666 |
if 'temperature' in sn or sn in ('wind_chill', 'dew_point'): |
---|
667 |
if valunits_from == 'degrees Celsius': |
---|
668 |
valunits_from = 'degC' |
---|
669 |
valunits_to = 'degF' |
---|
670 |
elif 'velocity' in sn: |
---|
671 |
valunits_to = 'knots' |
---|
672 |
elif 'flux' in sn or sn in ('discharge',): |
---|
673 |
if valunits_from == 'cfs': |
---|
674 |
valunits_from = 'ft^3/sec' |
---|
675 |
valunits_to = 'ft^3/s' |
---|
676 |
elif 'rain' in sn: |
---|
677 |
valunits_to = 'in' |
---|
678 |
elif 'level' in sn or 'height' in sn or 'depth' in sn: |
---|
679 |
valunits_to = 'ft' |
---|
680 |
else: |
---|
681 |
valunits_to = valunits_from |
---|
682 |
|
---|
683 |
cnv = udunits.udunits(valunits_from, valunits_to) |
---|
684 |
if cnv[0]==0: |
---|
685 |
val_to = val*cnv[1] + cnv[2] |
---|
686 |
if val > 99: |
---|
687 |
english_str ='%.4g (%s)' % (val_to, valunits_to) |
---|
688 |
else: |
---|
689 |
english_str = '%.2g (%s)' % (val_to, valunits_to) |
---|
690 |
|
---|
691 |
|
---|
692 |
|
---|
693 |
|
---|
694 |
elif cnv[0]==-1 or cnv[0]==-2: |
---|
695 |
if val > 99: |
---|
696 |
english_str = '%.4g (%s)' % (val, valunits) |
---|
697 |
else: |
---|
698 |
english_str = '%.2g (%s)' % (val, valunits) |
---|
699 |
else: |
---|
700 |
english_str = '\-' |
---|
701 |
|
---|
702 |
if metric_str == english_str: |
---|
703 |
english_str = '\-' |
---|
704 |
|
---|
705 |
if vn=='time': |
---|
706 |
f.write('"**%s:** %s", ""\n' % ('Sample Time', last_dt_str)) |
---|
707 |
else: |
---|
708 |
f.write('"%s", "%s", "%s"\n' % (var_name_str, metric_str, english_str)) |
---|
709 |
|
---|
710 |
f.close() |
---|
711 |
|
---|
712 |
|
---|
713 |
|
---|
714 |
|
---|