1 |
|
---|
2 |
|
---|
3 |
"""Create plots from monthly netCDF data files |
---|
4 |
|
---|
5 |
This module plots different graphical products from different NCCOOS |
---|
6 |
sensors (ctd, adcp, waves-adcp, met) based on manual or automated |
---|
7 |
operation. If automated processing, latest and current month plots |
---|
8 |
are generated for all active sensors based on the current |
---|
9 |
configuration setting. If manual processing, generate graphics for |
---|
10 |
requested platform, sensor, and month. |
---|
11 |
|
---|
12 |
:Processing steps: |
---|
13 |
0. proc2plot auto or manual for platform, sensor, month |
---|
14 |
1. |
---|
15 |
|
---|
16 |
""" |
---|
17 |
|
---|
18 |
__version__ = "v0.1" |
---|
19 |
__author__ = "Sara Haines <sara_haines@unc.edu>" |
---|
20 |
|
---|
21 |
import sys |
---|
22 |
import os |
---|
23 |
import re |
---|
24 |
|
---|
25 |
|
---|
26 |
|
---|
27 |
|
---|
28 |
|
---|
29 |
defconfigs='/opt/env/haines/dataproc/raw2proc' |
---|
30 |
|
---|
31 |
import numpy |
---|
32 |
|
---|
33 |
sys.path.append('/opt/env/haines/dataproc/raw2proc') |
---|
34 |
del(sys) |
---|
35 |
|
---|
36 |
from procutil import * |
---|
37 |
from ncutil import * |
---|
38 |
|
---|
39 |
|
---|
40 |
def get_config(name): |
---|
41 |
"""Usage Example >>>sensor_info = get_config('bogue_config_20060918.sensor_info')""" |
---|
42 |
components = name.split('.') |
---|
43 |
mod = __import__(components[0]) |
---|
44 |
for comp in components[1:]: |
---|
45 |
attr = getattr(mod, comp) |
---|
46 |
return attr |
---|
47 |
|
---|
48 |
def find_configs(platform, yyyy_mm, config_dir=''): |
---|
49 |
"""Find which configuration files for specified platform and month |
---|
50 |
|
---|
51 |
:Parameters: |
---|
52 |
platform : string |
---|
53 |
Platfrom id to process (e.g. 'bogue') |
---|
54 |
yyyy_mm : string |
---|
55 |
Year and month of data to process (e.g. '2007_07') |
---|
56 |
|
---|
57 |
:Returns: |
---|
58 |
cns : list of str |
---|
59 |
List of configurations that overlap with desired month |
---|
60 |
If empty [], no configs were found |
---|
61 |
""" |
---|
62 |
import glob |
---|
63 |
|
---|
64 |
configs = glob.glob(os.path.join(config_dir, platform + '_config_*.py')) |
---|
65 |
configs.sort() |
---|
66 |
now_dt = datetime.utcnow() |
---|
67 |
now_dt.replace(microsecond=0) |
---|
68 |
|
---|
69 |
(prev_month, this_month, next_month) = find_months(yyyy_mm) |
---|
70 |
month_start_dt = this_month |
---|
71 |
month_end_dt = next_month - timedelta(seconds=1) |
---|
72 |
|
---|
73 |
|
---|
74 |
cns = [] |
---|
75 |
for config in configs: |
---|
76 |
|
---|
77 |
cn = os.path.splitext(os.path.basename(config))[0] |
---|
78 |
cndt = filt_datetime(os.path.basename(config))[0] |
---|
79 |
pi = get_config(cn+'.platform_info') |
---|
80 |
if pi['config_start_date']: |
---|
81 |
config_start_dt = filt_datetime(pi['config_start_date'])[0] |
---|
82 |
elif pi['config_start_date'] == None: |
---|
83 |
config_start_dt = now_dt |
---|
84 |
if pi['config_end_date']: |
---|
85 |
config_end_dt = filt_datetime(pi['config_end_date'])[0] |
---|
86 |
elif pi['config_end_date'] == None: |
---|
87 |
config_end_dt = now_dt |
---|
88 |
|
---|
89 |
if (config_start_dt <= month_start_dt or config_start_dt <= month_end_dt) and \ |
---|
90 |
(config_end_dt >= month_start_dt or config_end_dt >= month_end_dt): |
---|
91 |
cns.append(cn) |
---|
92 |
return cns |
---|
93 |
|
---|
94 |
|
---|
95 |
def find_active_configs(config_dir=''): |
---|
96 |
"""Find which configuration files are active |
---|
97 |
|
---|
98 |
:Returns: |
---|
99 |
cns : list of str |
---|
100 |
List of configurations that overlap with desired month |
---|
101 |
If empty [], no configs were found |
---|
102 |
""" |
---|
103 |
import glob |
---|
104 |
|
---|
105 |
configs = glob.glob(os.path.join(config_dir, '*_config_*.py')) |
---|
106 |
now_dt = datetime.utcnow() |
---|
107 |
now_dt.replace(microsecond=0) |
---|
108 |
|
---|
109 |
cns = [] |
---|
110 |
for config in configs: |
---|
111 |
|
---|
112 |
cn = os.path.splitext(os.path.basename(config))[0] |
---|
113 |
cndt = filt_datetime(os.path.basename(config))[0] |
---|
114 |
pi = get_config(cn+'.platform_info') |
---|
115 |
if pi['config_end_date'] == None: |
---|
116 |
cns.append(cn) |
---|
117 |
return cns |
---|
118 |
|
---|
119 |
|
---|
120 |
|
---|
121 |
def proc2plot(proctype, platform=None, package=None, yyyy_mm=None): |
---|
122 |
""" |
---|
123 |
Plot products either in auto-mode or manual-mode |
---|
124 |
|
---|
125 |
If auto-mode, process latest products for all platforms, all |
---|
126 |
sensors. Otherwise in manual-mode, process data for specified |
---|
127 |
platform, sensor package, and month. |
---|
128 |
|
---|
129 |
:Parameters: |
---|
130 |
proctype : string |
---|
131 |
'auto' or 'manual' |
---|
132 |
|
---|
133 |
platform : string |
---|
134 |
Platfrom id to process (e.g. 'bogue') |
---|
135 |
package : string |
---|
136 |
Sensor package id to process (e.g. 'adcp') |
---|
137 |
yyyy_mm : string |
---|
138 |
Year and month of data to process (e.g. '2007_07') |
---|
139 |
|
---|
140 |
Examples |
---|
141 |
-------- |
---|
142 |
>>> proc2plot(proctype='manual', platform='bogue', package='adcp', yyyy_mm='2007_06') |
---|
143 |
>>> proc2plot('manual', 'bogue', 'adcp', '2007_06') |
---|
144 |
|
---|
145 |
""" |
---|
146 |
print '\nStart time for proc2plot: %s\n' % start_dt.strftime("%Y-%b-%d %H:%M:%S UTC") |
---|
147 |
|
---|
148 |
if proctype == 'auto': |
---|
149 |
print 'Processing in auto-mode, all platforms, all packages, latest data' |
---|
150 |
auto() |
---|
151 |
elif proctype == 'manual': |
---|
152 |
if platform and package and yyyy_mm: |
---|
153 |
print 'Processing in manually ...' |
---|
154 |
print ' ... platform id : %s' % platform |
---|
155 |
print ' ... package name : %s' % package |
---|
156 |
print ' ... month : %s' % yyyy_mm |
---|
157 |
print ' ... starting at : %s' % start_dt.strftime("%Y-%m-%d %H:%M:%S UTC") |
---|
158 |
manual(platform, package, yyyy_mm) |
---|
159 |
else: |
---|
160 |
print 'proc2plot: Manual operation requires platform, package, and month' |
---|
161 |
print " >>> proc2plot(proctype='manual', platform='bogue', package='adcp', yyyy_mm='2007_07')" |
---|
162 |
else: |
---|
163 |
print 'proc2plot: requires either auto or manual operation' |
---|
164 |
|
---|
165 |
|
---|
166 |
def auto(): |
---|
167 |
"""Process all platforms, all packages, latest data |
---|
168 |
|
---|
169 |
Notes |
---|
170 |
----- |
---|
171 |
|
---|
172 |
1. determine which platforms (all platforms with currently active |
---|
173 |
config files i.e. config_end_date is None |
---|
174 |
2. for each platform |
---|
175 |
get latest config |
---|
176 |
for each package |
---|
177 |
yyyy_mm is the current month |
---|
178 |
load this months netcdf |
---|
179 |
load and execute plot module(s) |
---|
180 |
|
---|
181 |
""" |
---|
182 |
yyyy_mm = this_month() |
---|
183 |
months = find_months(yyyy_mm) |
---|
184 |
month_start_dt = months[1] |
---|
185 |
month_end_dt = months[2] - timedelta(seconds=1) |
---|
186 |
|
---|
187 |
configs = find_active_configs(config_dir=defconfigs) |
---|
188 |
if configs: |
---|
189 |
|
---|
190 |
for cn in configs: |
---|
191 |
print ' ... config file : %s' % cn |
---|
192 |
pi = get_config(cn+'.platform_info') |
---|
193 |
asi = get_config(cn+'.sensor_info') |
---|
194 |
platform = pi['id'] |
---|
195 |
|
---|
196 |
for package in asi.keys(): |
---|
197 |
print ' ... package name : %s' % package |
---|
198 |
si = asi[package] |
---|
199 |
si['proc_filename'] = '%s_%s_%s.nc' % (platform, package, yyyy_mm) |
---|
200 |
ofn = os.path.join(si['proc_dir'], si['proc_filename']) |
---|
201 |
si['proc_start_dt'] = month_start_dt |
---|
202 |
si['proc_end_dt'] = month_end_dt |
---|
203 |
if os.path.exists(ofn): |
---|
204 |
|
---|
205 |
(es, units) = nc_get_time(ofn) |
---|
206 |
last_dt = es2dt(es[-1]) |
---|
207 |
|
---|
208 |
if last_dt>=month_start_dt: |
---|
209 |
si['proc_start_dt'] = last_dt |
---|
210 |
|
---|
211 |
if 'plot_module' in si.keys(): |
---|
212 |
make_plots(pi, si, yyyy_mm, plot_type='latest') |
---|
213 |
|
---|
214 |
|
---|
215 |
else: |
---|
216 |
print ' ... ... NOTE: no plot module specified for %s %s for %s' % (package, platform, yyyy_mm) |
---|
217 |
|
---|
218 |
|
---|
219 |
|
---|
220 |
else: |
---|
221 |
print ' ... ... NOTE: No active platforms' |
---|
222 |
|
---|
223 |
def manual(platform, package, yyyy_mm): |
---|
224 |
"""Process data for specified platform, sensor package, and month |
---|
225 |
|
---|
226 |
Notes |
---|
227 |
----- |
---|
228 |
|
---|
229 |
1. determine which configs |
---|
230 |
2. for each config for specific platform |
---|
231 |
if have package in config |
---|
232 |
which nc files |
---|
233 |
""" |
---|
234 |
|
---|
235 |
months = find_months(yyyy_mm) |
---|
236 |
month_start_dt = months[1] |
---|
237 |
month_end_dt = months[2] - timedelta(seconds=1) |
---|
238 |
|
---|
239 |
configs = find_configs(platform, yyyy_mm, config_dir=defconfigs) |
---|
240 |
|
---|
241 |
if configs: |
---|
242 |
|
---|
243 |
for index in range(len(configs)): |
---|
244 |
cn = configs[index] |
---|
245 |
print ' ... config file : %s' % cn |
---|
246 |
pi = get_config(cn+'.platform_info') |
---|
247 |
|
---|
248 |
asi = get_config(cn+'.sensor_info') |
---|
249 |
if package in pi['packages']: |
---|
250 |
si = asi[package] |
---|
251 |
if si['utc_offset']: |
---|
252 |
print ' ... ... utc_offset : %g (hours)' % si['utc_offset'] |
---|
253 |
si['proc_start_dt'] = month_start_dt |
---|
254 |
si['proc_end_dt'] = month_end_dt |
---|
255 |
si['proc_filename'] = '%s_%s_%s.nc' % (platform, package, yyyy_mm) |
---|
256 |
ofn = os.path.join(si['proc_dir'], si['proc_filename']) |
---|
257 |
|
---|
258 |
if os.path.exists(ofn): |
---|
259 |
|
---|
260 |
(es, units) = nc_get_time(ofn) |
---|
261 |
last_dt = es2dt(es[-1]) |
---|
262 |
|
---|
263 |
if 'plot_module' in si.keys(): |
---|
264 |
make_plots(pi, si, yyyy_mm, plot_type='monthly') |
---|
265 |
|
---|
266 |
|
---|
267 |
else: |
---|
268 |
print ' ... ... NOTE: no plot module specified for %s %s for %s' % (package, platform, yyyy_mm) |
---|
269 |
|
---|
270 |
else: |
---|
271 |
print ' ... ... NOTE: %s not operational on %s for %s' % (package, platform, yyyy_mm) |
---|
272 |
else: |
---|
273 |
print ' ... ... ... NOTE: %s not operational for %s' % (platform, yyyy_mm) |
---|
274 |
|
---|
275 |
def import_plotter(mod_name, plot_name): |
---|
276 |
mod = __import__(mod_name) |
---|
277 |
plotter = getattr(mod, plot_name) |
---|
278 |
return plotter |
---|
279 |
|
---|
280 |
def make_plots(pi, si, yyyy_mm, plot_type='latest'): |
---|
281 |
|
---|
282 |
mod_name = si['plot_module'] |
---|
283 |
plot_names = si['plot_names'] |
---|
284 |
|
---|
285 |
for pn in plot_names: |
---|
286 |
plot = import_plotter(mod_name, pn) |
---|
287 |
plot(pi, si, yyyy_mm, plot_type) |
---|
288 |
|
---|
289 |
|
---|
290 |
|
---|
291 |
|
---|
292 |
|
---|
293 |
|
---|
294 |
|
---|
295 |
|
---|
296 |
start_dt = datetime.utcnow() |
---|
297 |
start_dt.replace(microsecond=0) |
---|
298 |
|
---|
299 |
if __name__ == "__main__": |
---|
300 |
import optparse |
---|
301 |
proc2plot('auto') |
---|
302 |
|
---|
303 |
|
---|
304 |
|
---|
305 |
|
---|
306 |
|
---|