1 |
|
---|
2 |
|
---|
3 |
""" |
---|
4 |
parse yr, yrday, time from csi loggernet and create monthly files |
---|
5 |
|
---|
6 |
input file |
---|
7 |
/seacoos/data/nccoos/level0/crow/crow_csi_loggernet_yyyymmdd-yyyymmdd.dat |
---|
8 |
|
---|
9 |
Output form |
---|
10 |
/seacoos/data/nccoos/level0/crow/yyyy_mm/wq/csi_wq_yyyy_mm.dat |
---|
11 |
/seacoos/data/nccoos/level0/crow/yyyy_mm/flow/csi_flow_yyyy_mm.dat |
---|
12 |
|
---|
13 |
|
---|
14 |
load data file |
---|
15 |
parse lines for time YYYY, jjj, HHMM |
---|
16 |
what year and month? |
---|
17 |
|
---|
18 |
create YYYY_MM directory and output file if does not exist. |
---|
19 |
write line to YYYY_MM/csi_loggernet_yyyy_mm.dat output file |
---|
20 |
|
---|
21 |
""" |
---|
22 |
|
---|
23 |
REAL_RE_STR = '\\s*(-?\\d(\\.\\d+|)[Ee][+\\-]\\d\\d?|-?(\\d+\\.\\d*|\\d*\\.\\d+)|-?\\d+)\\s*' |
---|
24 |
|
---|
25 |
import sys |
---|
26 |
import os |
---|
27 |
import re |
---|
28 |
|
---|
29 |
def parse_csi_loggernet(fn, lines): |
---|
30 |
""" |
---|
31 |
|
---|
32 |
From FSL (CSI datalogger program files): |
---|
33 |
|
---|
34 |
15 Output_Table 15.00 Min |
---|
35 |
1 15 L |
---|
36 |
2 Year_RTM L |
---|
37 |
3 Day_RTM L |
---|
38 |
4 Hour_Minute_RTM L |
---|
39 |
5 Rain15sec_TOT L |
---|
40 |
6 SonLevlft L |
---|
41 |
7 SonFlow L |
---|
42 |
8 PrDepthft L |
---|
43 |
9 PrFlowcfs L |
---|
44 |
|
---|
45 |
1 Output_Table 60.00 Min |
---|
46 |
1 1 L |
---|
47 |
2 Year_RTM L |
---|
48 |
3 Day_RTM L |
---|
49 |
4 Hour_Minute_RTM L |
---|
50 |
5 H2OTempC L |
---|
51 |
6 SpCond L |
---|
52 |
7 DOSat L |
---|
53 |
8 DOmg L |
---|
54 |
9 PH L |
---|
55 |
10 Turb L |
---|
56 |
11 BattVolts L |
---|
57 |
|
---|
58 |
Example data: |
---|
59 |
|
---|
60 |
1,2005,83,1600,16.47,0,.4,.04,8.14,115.5,14.25 |
---|
61 |
15,2005,83,1615,0,4.551,-.547,.897,.885 |
---|
62 |
15,2005,83,1630,0,4.541,.727,.908,1.005 |
---|
63 |
15,2005,83,1645,0,4.537,6.731,.878,.676 |
---|
64 |
15,2005,83,1700,0,4.537,6.731,.83,.167 |
---|
65 |
1,2005,83,1700,16.57,0,.4,.03,8.03,145.7,13.08 |
---|
66 |
15,2005,83,1715,0,4.547,5.29,.847,.347 |
---|
67 |
15,2005,83,1730,0,4.541,.908,.842,.287 |
---|
68 |
15,2005,83,1745,0,4.547,7.3,.853,.407 |
---|
69 |
15,2005,83,1800,0,4.551,6.939,.855,.437 |
---|
70 |
1,2005,83,1800,15.65,0,.2,.02,7.91,111.3,12.98 |
---|
71 |
|
---|
72 |
""" |
---|
73 |
|
---|
74 |
import numpy |
---|
75 |
from datetime import datetime |
---|
76 |
from time import strptime |
---|
77 |
import math |
---|
78 |
|
---|
79 |
p = os.path.split(fn) |
---|
80 |
id = p[1].split('_')[0] |
---|
81 |
|
---|
82 |
for line in lines: |
---|
83 |
csi = [] |
---|
84 |
|
---|
85 |
sw = re.split(',', line) |
---|
86 |
for s in sw: |
---|
87 |
m = re.search(REAL_RE_STR, s) |
---|
88 |
if m: |
---|
89 |
csi.append(float(m.groups()[0])) |
---|
90 |
|
---|
91 |
if len(csi)>=4 and (re.search('^1,',line) or re.search('^15,',line)): |
---|
92 |
|
---|
93 |
|
---|
94 |
|
---|
95 |
yyyy = csi[1] |
---|
96 |
yday = csi[2] |
---|
97 |
(MM, HH) = math.modf(csi[3]/100.) |
---|
98 |
MM = math.ceil(MM*100.) |
---|
99 |
if (HH == 24): |
---|
100 |
yday=yday+1 |
---|
101 |
HH = 0. |
---|
102 |
|
---|
103 |
sample_str = '%04d-%03d %02d:%02d' % (yyyy, yday, HH, MM) |
---|
104 |
sample_dt = scanf_datetime(sample_str, fmt='%Y-%j %H:%M') |
---|
105 |
month_str = '%4d_%02d' % sample_dt.timetuple()[0:2] |
---|
106 |
else: |
---|
107 |
|
---|
108 |
print 'ill-formed time, line not to be copied: ' + line |
---|
109 |
continue |
---|
110 |
|
---|
111 |
if re.search('^1,',line) and len(csi)>=4: |
---|
112 |
|
---|
113 |
data_dir = os.path.join(p[0],'wq',month_str) |
---|
114 |
|
---|
115 |
if not os.path.isdir(data_dir): |
---|
116 |
print 'Creating directory: '+data_dir |
---|
117 |
os.mkdir(data_dir) |
---|
118 |
|
---|
119 |
ofn_prefix = '%s_%s' % (id, 'wq') |
---|
120 |
ofn = os.path.join(data_dir, ofn_prefix) |
---|
121 |
ofn = '_'.join([ofn, month_str]) |
---|
122 |
ofn = '.'.join([ofn, 'dat']) |
---|
123 |
|
---|
124 |
if os.path.exists(ofn): |
---|
125 |
f = open(ofn, 'a') |
---|
126 |
f.write(line) |
---|
127 |
f.close |
---|
128 |
else: |
---|
129 |
print 'Creating file: '+ofn |
---|
130 |
f = open(ofn, 'w') |
---|
131 |
f.write(line) |
---|
132 |
f.close() |
---|
133 |
|
---|
134 |
if re.search('^15,',line) and len(csi)>=4: |
---|
135 |
data_dir = os.path.join(p[0],'flow',month_str) |
---|
136 |
|
---|
137 |
if not os.path.isdir(data_dir): |
---|
138 |
print 'Creating directory: '+data_dir |
---|
139 |
os.mkdir(data_dir) |
---|
140 |
|
---|
141 |
ofn_prefix = '%s_%s' % (id, 'flow') |
---|
142 |
ofn = os.path.join(data_dir, ofn_prefix) |
---|
143 |
ofn = '_'.join([ofn, month_str]) |
---|
144 |
ofn = '.'.join([ofn, 'dat']) |
---|
145 |
|
---|
146 |
if os.path.exists(ofn): |
---|
147 |
f = open(ofn, 'a') |
---|
148 |
f.write(line) |
---|
149 |
f.close |
---|
150 |
else: |
---|
151 |
print 'Creating file: '+ofn |
---|
152 |
f = open(ofn, 'w') |
---|
153 |
f.write(line) |
---|
154 |
f.close() |
---|
155 |
|
---|
156 |
|
---|
157 |
return |
---|
158 |
|
---|
159 |
|
---|
160 |
def load_data(inFile): |
---|
161 |
lines=None |
---|
162 |
if os.path.exists(inFile): |
---|
163 |
f = open(inFile, 'r') |
---|
164 |
lines = f.readlines() |
---|
165 |
f.close() |
---|
166 |
if len(lines)<=0: |
---|
167 |
print 'Empty file: '+ inFile |
---|
168 |
else: |
---|
169 |
print 'File does not exist: '+ inFile |
---|
170 |
return lines |
---|
171 |
|
---|
172 |
from raw2proc import * |
---|
173 |
|
---|
174 |
def test1(fn): |
---|
175 |
lines = load_data(fn) |
---|
176 |
return parse_csi_loggernet(fn, lines) |
---|
177 |
|
---|
178 |
def spin(): |
---|
179 |
|
---|
180 |
|
---|
181 |
fns = [ |
---|
182 |
'/seacoos/data/nccoos/level0/crow/cbc_loggernet_20060316-20080829.dat', |
---|
183 |
|
---|
184 |
'/seacoos/data/nccoos/level0/meet/mow_loggernet_20010510-20030925.dat', |
---|
185 |
'/seacoos/data/nccoos/level0/meet/mow_loggernet_20030925-20041209.dat', |
---|
186 |
'/seacoos/data/nccoos/level0/meet/mow_loggernet_20050325-20070726.dat', |
---|
187 |
'/seacoos/data/nccoos/level0/meet/mow_loggernet_20080404-20080826.dat', |
---|
188 |
] |
---|
189 |
|
---|
190 |
for fn in fns: |
---|
191 |
test1(fn) |
---|
192 |
|
---|
193 |
|
---|
194 |
if __name__ == '__main__': |
---|
195 |
pass |
---|
196 |
|
---|
197 |
|
---|
198 |
|
---|
199 |
|
---|
200 |
|
---|
201 |
|
---|
202 |
|
---|
203 |
|
---|
204 |
|
---|