| 1 |
#!/usr/bin/env python |
|---|
| 2 |
# Last modified: Time-stamp: <2009-12-07 16:43:48 haines> |
|---|
| 3 |
"""spin_meet_csv""" |
|---|
| 4 |
|
|---|
| 5 |
# create csv files from netcdf |
|---|
| 6 |
|
|---|
| 7 |
import os, sys, glob, re |
|---|
| 8 |
import datetime, time, dateutil, dateutil.tz |
|---|
| 9 |
import pycdf |
|---|
| 10 |
import numpy |
|---|
| 11 |
import csv |
|---|
| 12 |
|
|---|
| 13 |
sys.path.append('/opt/env/haines/dataproc/raw2proc') |
|---|
| 14 |
import procutil |
|---|
| 15 |
del(sys) |
|---|
| 16 |
|
|---|
| 17 |
print 'spin_meet_csv ...' |
|---|
| 18 |
|
|---|
| 19 |
proc_dir = '/seacoos/data/nccoos/level1/meet/wq' |
|---|
| 20 |
fns = glob.glob((os.path.join(proc_dir, '*.nc'))) |
|---|
| 21 |
# fns = glob.glob((os.path.join(proc_dir, '*2009*.nc'))) |
|---|
| 22 |
fns.sort() |
|---|
| 23 |
|
|---|
| 24 |
for fn in fns: |
|---|
| 25 |
m=re.search('\d{4}_\d{2}', fn) |
|---|
| 26 |
yyyy_mm = m.group() |
|---|
| 27 |
prev_month, this_month, next_month = procutil.find_months(yyyy_mm) |
|---|
| 28 |
yyyy_mm_str = this_month.strftime('%Y_%m') |
|---|
| 29 |
|
|---|
| 30 |
# load data |
|---|
| 31 |
print ' ... ... read: ' + fn |
|---|
| 32 |
nc = pycdf.CDFMF((fn,)) |
|---|
| 33 |
ncvars = nc.variables() |
|---|
| 34 |
# print ncvars |
|---|
| 35 |
es = nc.var('time')[:] |
|---|
| 36 |
units = nc.var('time').units |
|---|
| 37 |
dt = [procutil.es2dt(e) for e in es] |
|---|
| 38 |
# set timezone info to UTC (since data from level1 should be in UTC!!) |
|---|
| 39 |
dt = [e.replace(tzinfo=dateutil.tz.tzutc()) for e in dt] |
|---|
| 40 |
# return new datetime based on computer local |
|---|
| 41 |
dt_local = [e.astimezone(dateutil.tz.tzlocal()) for e in dt] |
|---|
| 42 |
|
|---|
| 43 |
z = nc.var('z')[:] |
|---|
| 44 |
wtemp = nc.var('wtemp')[:] |
|---|
| 45 |
cond = nc.var('cond')[:] |
|---|
| 46 |
turb = nc.var('turb')[:] |
|---|
| 47 |
ph = nc.var('ph')[:] |
|---|
| 48 |
do_mg = nc.var('do_mg')[:] |
|---|
| 49 |
do_sat = nc.var('do_sat')[:] |
|---|
| 50 |
batt = nc.var('battvolts')[:] |
|---|
| 51 |
nc.close() |
|---|
| 52 |
|
|---|
| 53 |
# save csv |
|---|
| 54 |
ofn = proc_dir + '_csv/meet_wq_' + yyyy_mm_str + '.csv' |
|---|
| 55 |
print '... ... write: %s' % (ofn,) |
|---|
| 56 |
|
|---|
| 57 |
ofp = open(ofn, 'w') |
|---|
| 58 |
csvwriter = csv.writer(ofp, dialect='excel', quoting=csv.QUOTE_NONNUMERIC) |
|---|
| 59 |
csvwriter.writerow(['TOA5','MOW','CPU:UNC_MeetingWaters','DataHourly']) |
|---|
| 60 |
csvwriter.writerow(['TIMESTAMP','SondeTempC','SpCond','DOSat','DOmg','pH','Turb','BattVolt_Min']) |
|---|
| 61 |
csvwriter.writerow(['TS','','','','','','','']) |
|---|
| 62 |
csvwriter.writerow(['','Smp','Smp','Smp','Smp','Smp','Smp','Min']) |
|---|
| 63 |
|
|---|
| 64 |
fmt = '%Y-%m-%d %H:%M:%S' |
|---|
| 65 |
for i in range(len(dt)): |
|---|
| 66 |
row = [dt[i].strftime(fmt), wtemp[i],cond[i],do_sat[i],do_mg[i],ph[i],turb[i],batt[i]] |
|---|
| 67 |
csvwriter.writerow(row) |
|---|
| 68 |
ofp.close() |
|---|
| 69 |
|
|---|
| 70 |
|
|---|
| 71 |
|
|---|
| 72 |
proc_dir = '/seacoos/data/nccoos/level1/meet/flow' |
|---|
| 73 |
fns = glob.glob((os.path.join(proc_dir, '*.nc'))) |
|---|
| 74 |
# fns = glob.glob((os.path.join(proc_dir, '*2009*.nc'))) |
|---|
| 75 |
fns.sort() |
|---|
| 76 |
for fn in fns: |
|---|
| 77 |
m=re.search('\d{4}_\d{2}', fn) |
|---|
| 78 |
yyyy_mm = m.group() |
|---|
| 79 |
prev_month, this_month, next_month = procutil.find_months(yyyy_mm) |
|---|
| 80 |
yyyy_mm_str = this_month.strftime('%Y_%m') |
|---|
| 81 |
|
|---|
| 82 |
print ' ... ... read: ' + fn |
|---|
| 83 |
nc = pycdf.CDFMF((fn,)) |
|---|
| 84 |
ncvars = nc.variables() |
|---|
| 85 |
# print ncvars |
|---|
| 86 |
es = nc.var('time')[:] |
|---|
| 87 |
units = nc.var('time').units |
|---|
| 88 |
dt = [procutil.es2dt(e) for e in es] |
|---|
| 89 |
# set timezone info to UTC (since data from level1 should be in UTC!!) |
|---|
| 90 |
dt = [e.replace(tzinfo=dateutil.tz.tzutc()) for e in dt] |
|---|
| 91 |
|
|---|
| 92 |
rain = nc.var('rain')[:] # inches of rain in past 15 min |
|---|
| 93 |
# all sontek data at Meeting of the Waters is suspicious and not to be used. |
|---|
| 94 |
have_sontek = False |
|---|
| 95 |
# have_sontek = 'sontek_wl' in ncvars.keys() or 'sontek_flow' in ncvars.keys() |
|---|
| 96 |
pwl = nc.var('press_wl')[:] # feet |
|---|
| 97 |
pfl = nc.var('press_flow')[:] # cfs |
|---|
| 98 |
|
|---|
| 99 |
if have_sontek: |
|---|
| 100 |
swl = nc.var('sontek_wl')[:] # feet |
|---|
| 101 |
sfl = nc.var('sontek_flow')[:] # cfs |
|---|
| 102 |
|
|---|
| 103 |
nc.close() |
|---|
| 104 |
|
|---|
| 105 |
# save csv |
|---|
| 106 |
ofn = proc_dir + '_csv/meet_flow_' + yyyy_mm_str + '.csv' |
|---|
| 107 |
print '... ... write: %s' % (ofn,) |
|---|
| 108 |
|
|---|
| 109 |
ofp = open(ofn, 'w') |
|---|
| 110 |
csvwriter = csv.writer(ofp, dialect='excel', quoting=csv.QUOTE_NONNUMERIC) |
|---|
| 111 |
csvwriter.writerow(['TOA5','MOW','CPU:UNC_MeetingWaters','Data15Min']) |
|---|
| 112 |
|
|---|
| 113 |
if have_sontek: |
|---|
| 114 |
csvwriter.writerow(['TIMESTAMP','Rain INCHES', |
|---|
| 115 |
'Press Water Level UNITS?','Press Flow CFS', |
|---|
| 116 |
'SONTEK Water Level FEET','SONTEK Flow CFS']) |
|---|
| 117 |
csvwriter.writerow(['TS','','','','','','']) |
|---|
| 118 |
csvwriter.writerow(['','Smp','Smp','Smp','Smp','Smp']) |
|---|
| 119 |
|
|---|
| 120 |
fmt = '%Y-%m-%d %H:%M:%S' |
|---|
| 121 |
for i in range(len(dt)): |
|---|
| 122 |
row = [dt[i].strftime(fmt), rain[i], pwl[i], pfl[i], swl[i], sfl[i]] |
|---|
| 123 |
csvwriter.writerow(row) |
|---|
| 124 |
|
|---|
| 125 |
else: |
|---|
| 126 |
csvwriter.writerow(['TIMESTAMP','Rain INCHES', |
|---|
| 127 |
'Press Water Level FEET','Press Flow CFS']) |
|---|
| 128 |
csvwriter.writerow(['TS','','','','']) |
|---|
| 129 |
csvwriter.writerow(['','Smp','Smp','Smp']) |
|---|
| 130 |
|
|---|
| 131 |
fmt = '%Y-%m-%d %H:%M:%S' |
|---|
| 132 |
for i in range(len(dt)): |
|---|
| 133 |
row = [dt[i].strftime(fmt), rain[i], pwl[i], pfl[i]] |
|---|
| 134 |
csvwriter.writerow(row) |
|---|
| 135 |
|
|---|
| 136 |
ofp.close() |
|---|
