1 |
#!/usr/bin/env python |
---|
2 |
# Last modified: Time-stamp: <2009-08-19 10:52:46 haines> |
---|
3 |
"""spin_crow_csv""" |
---|
4 |
|
---|
5 |
# create csv files from netcdf |
---|
6 |
|
---|
7 |
import os, sys, glob, re |
---|
8 |
import datetime, time, dateutil, dateutil.tz |
---|
9 |
import pycdf |
---|
10 |
import numpy |
---|
11 |
import csv |
---|
12 |
|
---|
13 |
sys.path.append('/opt/env/haines/dataproc/raw2proc') |
---|
14 |
import procutil |
---|
15 |
del(sys) |
---|
16 |
|
---|
17 |
print 'spin_crow_csv ...' |
---|
18 |
|
---|
19 |
proc_dir = '/seacoos/data/nccoos/level1/crow/wq/' |
---|
20 |
fns = glob.glob((os.path.join(proc_dir, '*.nc'))) |
---|
21 |
# fns = glob.glob((os.path.join(proc_dir, '*2009*.nc'))) |
---|
22 |
fns.sort() |
---|
23 |
|
---|
24 |
for fn in fns: |
---|
25 |
m=re.search('\d{4}_\d{2}', fn) |
---|
26 |
yyyy_mm = m.group() |
---|
27 |
prev_month, this_month, next_month = procutil.find_months(yyyy_mm) |
---|
28 |
yyyy_mm_str = this_month.strftime('%Y_%m') |
---|
29 |
|
---|
30 |
# load data |
---|
31 |
print ' ... ... read: ' + fn |
---|
32 |
nc = pycdf.CDFMF((fn,)) |
---|
33 |
ncvars = nc.variables() |
---|
34 |
# print ncvars |
---|
35 |
es = nc.var('time')[:] |
---|
36 |
units = nc.var('time').units |
---|
37 |
dt = [procutil.es2dt(e) for e in es] |
---|
38 |
# set timezone info to UTC (since data from level1 should be in UTC!!) |
---|
39 |
dt = [e.replace(tzinfo=dateutil.tz.tzutc()) for e in dt] |
---|
40 |
# return new datetime based on computer local |
---|
41 |
dt_local = [e.astimezone(dateutil.tz.tzlocal()) for e in dt] |
---|
42 |
|
---|
43 |
z = nc.var('z')[:] |
---|
44 |
wtemp = nc.var('wtemp')[:] |
---|
45 |
cond = nc.var('cond')[:] |
---|
46 |
turb = nc.var('turb')[:] |
---|
47 |
ph = nc.var('ph')[:] |
---|
48 |
do_mg = nc.var('do_mg')[:] |
---|
49 |
do_sat = nc.var('do_sat')[:] |
---|
50 |
batt = nc.var('battvolts')[:] |
---|
51 |
nc.close() |
---|
52 |
|
---|
53 |
# save csv |
---|
54 |
ofn = proc_dir + '/crow_wq_' + yyyy_mm_str + '.csv' |
---|
55 |
print '... ... write: %s' % (ofn,) |
---|
56 |
|
---|
57 |
ofp = open(ofn, 'w') |
---|
58 |
csvwriter = csv.writer(ofp, dialect='excel', quoting=csv.QUOTE_NONNUMERIC) |
---|
59 |
csvwriter.writerow(['TOA5','CBC','CPU:UNC_CrowBranch','DataHourly']) |
---|
60 |
csvwriter.writerow(['TIMESTAMP','SondeTempC','SpCond','DOSat','DOmg','pH','Turb','BattVolt_Min']) |
---|
61 |
csvwriter.writerow(['TS','','','','','','','']) |
---|
62 |
csvwriter.writerow(['','Smp','Smp','Smp','Smp','Smp','Smp','Min']) |
---|
63 |
|
---|
64 |
fmt = '%Y-%m-%d %H:%M:%S' |
---|
65 |
for i in range(len(dt)): |
---|
66 |
row = [dt[i].strftime(fmt), wtemp[i],cond[i],do_sat[i],do_mg[i],ph[i],turb[i],batt[i]] |
---|
67 |
csvwriter.writerow(row) |
---|
68 |
ofp.close() |
---|
69 |
|
---|
70 |
|
---|
71 |
|
---|
72 |
proc_dir = '/seacoos/data/nccoos/level1/crow/flow/' |
---|
73 |
fns = glob.glob((os.path.join(proc_dir, '*2009*.nc'))) |
---|
74 |
# fns = glob.glob((os.path.join(proc_dir, '*2009*.nc'))) |
---|
75 |
fns.sort() |
---|
76 |
for fn in fns: |
---|
77 |
m=re.search('\d{4}_\d{2}', fn) |
---|
78 |
yyyy_mm = m.group() |
---|
79 |
prev_month, this_month, next_month = procutil.find_months(yyyy_mm) |
---|
80 |
yyyy_mm_str = this_month.strftime('%Y_%m') |
---|
81 |
|
---|
82 |
print ' ... ... read: ' + fn |
---|
83 |
nc = pycdf.CDFMF((fn,)) |
---|
84 |
ncvars = nc.variables() |
---|
85 |
# print ncvars |
---|
86 |
es = nc.var('time')[:] |
---|
87 |
units = nc.var('time').units |
---|
88 |
dt = [procutil.es2dt(e) for e in es] |
---|
89 |
# set timezone info to UTC (since data from level1 should be in UTC!!) |
---|
90 |
dt = [e.replace(tzinfo=dateutil.tz.tzutc()) for e in dt] |
---|
91 |
|
---|
92 |
rain = nc.var('rain')[:] # inches of rain in past 15 min |
---|
93 |
have_sontek = 'sontek_wl' in ncvars.keys() or 'sontek_flow' in ncvars.keys() |
---|
94 |
pwl = nc.var('press_wl')[:] # feet |
---|
95 |
pfl = nc.var('press_flow')[:] # cfs |
---|
96 |
|
---|
97 |
if have_sontek: |
---|
98 |
swl = nc.var('sontek_wl')[:] # feet |
---|
99 |
sfl = nc.var('sontek_flow')[:] # cfs |
---|
100 |
|
---|
101 |
nc.close() |
---|
102 |
|
---|
103 |
# save csv |
---|
104 |
ofn = proc_dir + '/crow_flow_' + yyyy_mm_str + '.csv' |
---|
105 |
print '... ... write: %s' % (ofn,) |
---|
106 |
|
---|
107 |
ofp = open(ofn, 'w') |
---|
108 |
csvwriter = csv.writer(ofp, dialect='excel', quoting=csv.QUOTE_NONNUMERIC) |
---|
109 |
csvwriter.writerow(['TOA5','CBC','CPU:UNC_CrowBranch','Data15Min']) |
---|
110 |
|
---|
111 |
if have_sontek: |
---|
112 |
csvwriter.writerow(['TIMESTAMP','Rain INCHES', |
---|
113 |
'Press Water Level UNITS?','Press Flow CFS', |
---|
114 |
'SONTEK Water Level FEET','SONTEK Flow CFS']) |
---|
115 |
csvwriter.writerow(['TS','','','','','','']) |
---|
116 |
csvwriter.writerow(['','Smp','Smp','Smp','Smp','Smp']) |
---|
117 |
|
---|
118 |
fmt = '%Y-%m-%d %H:%M:%S' |
---|
119 |
for i in range(len(dt)): |
---|
120 |
row = [dt[i].strftime(fmt), rain[i], pwl[i], pfl[i], swl[i], sfl[i]] |
---|
121 |
csvwriter.writerow(row) |
---|
122 |
|
---|
123 |
else: |
---|
124 |
csvwriter.writerow(['TIMESTAMP','Rain INCHES', |
---|
125 |
'Press Water Level UNITS?','Press Flow CFS']) |
---|
126 |
csvwriter.writerow(['TS','','','','']) |
---|
127 |
csvwriter.writerow(['','Smp','Smp','Smp']) |
---|
128 |
|
---|
129 |
fmt = '%Y-%m-%d %H:%M:%S' |
---|
130 |
for i in range(len(dt)): |
---|
131 |
row = [dt[i].strftime(fmt), rain[i], pwl[i], pfl[i]] |
---|
132 |
csvwriter.writerow(row) |
---|
133 |
|
---|
134 |
ofp.close() |
---|