NCCOOS Trac Projects: Top | Web | Platforms | Processing | Viz | Sprints | Sandbox | (Wind)

root/raw2proc/trunk/raw2proc/proc_cr1000_met.py

Revision 494 (checked in by haines, 12 years ago)

Processing mods for buoy data

Line 
1 #!/usr/bin/env python
2 # Last modified:  Time-stamp: <2012-06-28 15:45:31 haines>
3 """
4 how to parse data, and assert what data and info goes into
5 creating and updating monthly netcdf files
6
7 parse data met data collected on Campbell Scientific DataLogger (loggernet) (csi)
8
9 parser : sample date and time,
10
11 creator : lat, lon, z, time,
12 updator : time,
13
14
15 Examples
16 --------
17
18 >> (parse, create, update) = load_processors('proc_csi_adcp_v2')
19 or
20 >> si = get_config(cn+'.sensor_info')
21 >> (parse, create, update) = load_processors(si['adcp']['proc_module'])
22
23 >> lines = load_data(filename)
24 >> data = parse(platform_info, sensor_info, lines)
25 >> create(platform_info, sensor_info, data) or
26 >> update(platform_info, sensor_info, data)
27
28 """
29
30
31 from raw2proc import *
32 from procutil import *
33 from ncutil import *
34
35 now_dt = datetime.utcnow()
36 now_dt.replace(microsecond=0)
37
38 def parser(platform_info, sensor_info, lines):
39     """
40     Example met data
41
42     "TOA5","CR1000_B1","CR1000","37541","CR1000.Std.21","CPU:NCWIND_12_Buoy_All.CR1","58723","AMet_6Min"
43     "TIMESTAMP","RECORD","Baro_mbar_Avg","RHumidity_Avg","RHumidity_Std","AirTempC_Avg","AirTempC_Std","Rain","Psp_Avg","Psp_Std","Pir_Wm2_Avg","Pir_Wm2_Std"
44     "TS","RN","","","","","","","","","",""
45     "","","Avg","Avg","Std","Avg","Std","Smp","Avg","Std","Avg","Std"
46     "2011-11-01 00:00:59",4590,14.3792,75.59,0.579,15.67,0.05,-22.35,1197.037,45.58967,371.5126,0.9030571
47     "2011-11-01 00:06:59",4591,14.37995,74.96,0.912,16.61,0.048,-21,-1071.813,129.5147,381.2539,0.2076943
48     "2011-11-01 00:12:59",4592,14.3792,72.71,2.677,17.29,0.032,-15.58,-2056.658,0,381.1828,0.1402813
49     "2011-11-01 00:18:59",4593,14.3791,72.63,0.928,17.67,0.041,-19.64,-1895.86,9.866026,381.0333,0.2442325
50    
51     """
52
53     import numpy
54     from datetime import datetime
55     from time import strptime
56
57     # get sample datetime from filename
58     fn = sensor_info['fn']
59     sample_dt_start = filt_datetime(fn)
60
61     # how many samples (don't count header 4 lines)
62     nsamp = len(lines[4:])
63
64     N = nsamp
65     data = {
66         'dt' : numpy.array(numpy.ones((N,), dtype=object)*numpy.nan),
67         'time' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan),
68         'air_press' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
69         'rh' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
70         'rh_std' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
71         'air_temp' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
72         'air_temp_std' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
73         'rain' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
74         'psp' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
75         'psp_std' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
76         'pir' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
77         'pir_std' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
78         }
79
80     # sample count
81     i = 0
82
83     for line in lines[4:]:
84         csi = []
85         # split line
86         sw = re.split(',', line)
87         if len(sw)<=0:
88             print ' ... skipping line %d ' % (i,)
89             continue
90
91         # replace any "NAN" text with a number
92         for index, s in enumerate(sw):
93             m = re.search(NAN_RE_STR, s)
94             if m:
95                 sw[index] = '-99999'
96
97         # parse date-time, and all other float and integers
98         for s in sw[1:]:
99             m = re.search(REAL_RE_STR, s)
100             if m:
101                 csi.append(float(m.groups()[0]))
102
103         if  sensor_info['utc_offset']:
104             sample_dt = scanf_datetime(sw[0], fmt='"%Y-%m-%d %H:%M:%S"') + \
105                         timedelta(hours=sensor_info['utc_offset'])
106         else:
107             sample_dt = scanf_datetime(sw[0], fmt='"%Y-%m-%d %H:%M:%S"')
108
109         data['dt'][i] = sample_dt # sample datetime
110         data['time'][i] = dt2es(sample_dt) # sample time in epoch seconds
111        
112         if len(csi)==11:
113             #
114             # data['samplenum'][i] = csi[0] # sample number assigned by datalogger in table
115             data['air_press'][i] =  csi[1] # Heise Barometer (psi) to mbar
116             data['rh'][i] = csi[2] # relative humidity avg (60 samples for 1 min)
117             data['rh_std'][i] = csi[3] # relative humidity std
118             data['air_temp'][i] = csi[4] # air temperature avg (deg C)
119             data['air_temp_std'][i] = csi[5] # air temperature std (deg C)
120             data['rain'][i] = csi[6]/100. # precip gauge cummulative (mm)
121             data['psp'][i] = csi[7] # PSP avg
122             data['psp_std'][i] = csi[8] # PSP std
123             data['pir'][i] = csi[9] # PIR avg (W m-2)
124             data['pir_std'][i] = csi[10] # PIR std (W m-2)
125             i=i+1
126         else:
127             print ' ... skipping line %d -- %s ' % (i,line)
128             continue
129
130         # if re.search
131     # for line
132
133     data['air_press'] = udconvert(data['air_press'], 'psi', 'mbar')[0]
134     # some QC
135     # good = -40<at & at<60 # does not work
136     # good = (-40<at) & (at<60) # THIS WORKS!
137     good = (5<data['air_temp']) & (data['air_temp']<30)
138     bad = ~good
139     data['air_temp'][bad] = numpy.nan
140     data['air_temp_std'][bad] = numpy.nan
141     data['rh'][bad] = numpy.nan
142     data['rh_std'][bad] = numpy.nan
143     data['rain'][bad] = numpy.nan
144
145     # specific to buoy B1 and B2
146     if platform_info['id'] == 'b1':
147         data['psp'] = -1*data['psp']/1000
148         data['psp_std'] = -1*data['psp_std']/1000
149     if platform_info['id'] == 'b2':
150         data['psp'] = numpy.nan*data['psp']
151         data['psp_std'] = numpy.nan*data['psp_std']
152    
153     # check that no data[dt] is set to Nan or anything but datetime
154     # keep only data that has a resolved datetime
155     keep = numpy.array([type(datetime(1970,1,1)) == type(dt) for dt in data['dt'][:]])
156     if keep.any():
157         for param in data.keys():
158             data[param] = data[param][keep]
159
160     return data
161  
162
163 def creator(platform_info, sensor_info, data):
164     #
165     #
166     # subset data only to month being processed (see raw2proc.process())
167     i = data['in']
168
169     title_str = sensor_info['description']+' at '+ platform_info['location']
170     global_atts = {
171         'title' : title_str,
172         'institution' : platform_info['institution'],
173         'institution_url' : platform_info['institution_url'],
174         'institution_dods_url' : platform_info['institution_dods_url'],
175         'metadata_url' : platform_info['metadata_url'],
176         'references' : platform_info['references'],
177         'contact' : platform_info['contact'],
178         #
179         'source' : platform_info['source']+' '+sensor_info['source'],
180         'history' : 'raw2proc using ' + sensor_info['process_module'],
181         'comment' : 'File created using pycdf'+pycdfVersion()+' and numpy '+pycdfArrayPkg(),
182         # conventions
183         'Conventions' : platform_info['conventions'],
184         # SEACOOS CDL codes
185         'format_category_code' : platform_info['format_category_code'],
186         'institution_code' : platform_info['institution_code'],
187         'platform_code' : platform_info['id'],
188         'package_code' : sensor_info['id'],
189         # institution specific
190         'project' : platform_info['project'],
191         'project_url' : platform_info['project_url'],
192         # timeframe of data contained in file yyyy-mm-dd HH:MM:SS
193         # first date in monthly file
194         'start_date' : data['dt'][i][0].strftime("%Y-%m-%d %H:%M:%S"),
195         # last date in monthly file
196         'end_date' : data['dt'][i][-1].strftime("%Y-%m-%d %H:%M:%S"),
197         'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
198         #
199         'creation_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
200         'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
201         'process_level' : 'level1',
202         #
203         # must type match to data (e.g. fillvalue is real if data is real)
204         '_FillValue' : -99999.,
205         }
206
207     var_atts = {
208         # coordinate variables
209         'time' : {'short_name': 'time',
210                   'long_name': 'Time',
211                   'standard_name': 'time',
212                   'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC
213                   'axis': 'T',
214                   },
215         'lat' : {'short_name': 'lat',
216                  'long_name': 'Latitude',
217                  'standard_name': 'latitude',
218                  'reference':'geographic coordinates',
219                  'units': 'degrees_north',
220                  'valid_range':(-90.,90.),
221                  'axis': 'Y',
222                  },
223         'lon' : {'short_name': 'lon',
224                  'long_name': 'Longitude',
225                  'standard_name': 'longitude',
226                  'reference':'geographic coordinates',
227                  'units': 'degrees_east',
228                  'valid_range':(-180.,180.),
229                  'axis': 'Y',
230                  },
231         'z' : {'short_name': 'z',
232                'long_name': 'Altitude',
233                'standard_name': 'altitude',
234                'reference':'zero at mean sea level',
235                'positive' : 'up',
236                'units': 'm',
237                'axis': 'Z',
238                },
239         # data variables
240         'air_press': {'short_name': 'air_press',
241                   'long_name': 'Air Pressure',
242                   'standard_name': 'air_pressure',                         
243                   'units': 'mbar',
244                   'z': sensor_info['barometer_height'],
245                   'z_units' : 'meter',
246                   },
247         'air_temp': {'short_name': 'air_temp',
248                   'long_name': 'Air Temperature',
249                   'standard_name': 'air_temperature',                         
250                   'units': 'degC',
251                   'z': sensor_info['temperature_height'],
252                   'z_units' : 'meter',
253                   },
254         'air_temp_std': {'short_name': 'air_temp_std',
255                   'long_name': 'Standard Deviation of Air Temperature',
256                   'standard_name': 'air_temperature',                         
257                   'units': 'degC',
258                   },
259         'rh': {'short_name': 'rh',
260                   'long_name': 'Relative Humidity',
261                   'standard_name': 'relative_humidity',                         
262                   'units': '%',
263                   'z': sensor_info['temperature_height'],
264                   'z_units' : 'meter',
265                   },
266         'rh_std': {'short_name': 'rh_std',
267                   'long_name': 'Standard Deviation of Relative Humidity',
268                   'standard_name': 'relative_humidity',                         
269                   'units': '%',
270                   },
271         'rain': {'short_name': 'rain',
272                  'long_name': '6-Minute Rain',
273                  'standard_name': 'rain',                         
274                  'units': 'inches',
275                   },
276         'psp': {'short_name': 'psp',
277                   'long_name': 'Short-wave Radiation',
278                   'standard_name': 'downwelling_shortwave_irradiance',                         
279                   'units': 'W m-2',
280                   },
281         'psp_std': {'short_name': 'psp_std',
282                   'long_name': 'Standard Deviation of Short-wave Radiation',
283                   'standard_name': 'shortwave_radiation',                         
284                   'units': 'W m-2',
285                   },
286         'pir': {'short_name': 'pir',
287                   'long_name': 'Long-wave Radiation',
288                   'standard_name': 'longwave_radiation',                         
289                   'units': 'W m-2',
290                   },
291         'pir_std': {'short_name': 'pir_std',
292                   'long_name': 'Standard Deviation of Long-wave Radiation',
293                   'standard_name': 'longwave_radiation',                         
294                   'units': 'W m-2',
295                   },
296         }
297
298     # dimension names use tuple so order of initialization is maintained
299     dim_inits = (
300         ('ntime', NC.UNLIMITED),
301         ('nlat', 1),
302         ('nlon', 1),
303         ('nz', 1),
304         )
305    
306     # using tuple of tuples so order of initialization is maintained
307     # using dict for attributes order of init not important
308     # use dimension names not values
309     # (varName, varType, (dimName1, [dimName2], ...))
310     var_inits = (
311         # coordinate variables
312         ('time', NC.INT, ('ntime',)),
313         ('lat', NC.FLOAT, ('nlat',)),
314         ('lon', NC.FLOAT, ('nlon',)),
315         ('z',  NC.FLOAT, ('nz',)),
316         # data variables
317         ('air_press', NC.FLOAT, ('ntime',)),
318         ('rh', NC.FLOAT, ('ntime',)),
319         ('rh_std', NC.FLOAT, ('ntime',)),
320         ('air_temp', NC.FLOAT, ('ntime',)),
321         ('air_temp_std', NC.FLOAT, ('ntime',)),
322         ('rain', NC.FLOAT, ('ntime',)),
323         ('psp', NC.FLOAT, ('ntime',)),
324         ('psp_std', NC.FLOAT, ('ntime',)),
325         ('pir', NC.FLOAT, ('ntime',)),
326         ('pir_std', NC.FLOAT, ('ntime',)),
327         )
328
329     # subset data only to month being processed (see raw2proc.process())
330     i = data['in']
331    
332     # var data
333     var_data = (
334         ('lat',  platform_info['lat']),
335         ('lon', platform_info['lon']),
336         ('z', platform_info['altitude']),
337         #
338         ('time', data['time'][i]),
339         #
340         ('air_press', data['air_press'][i]),
341         ('rh', data['rh'][i]),
342         ('rh_std', data['rh_std'][i]),
343         ('air_temp', data['air_temp'][i]),
344         ('air_temp_std', data['air_temp_std'][i]),
345         ('rain', data['rain'][i]),
346         ('psp', data['psp'][i]),
347         ('psp_std', data['psp_std'][i]),
348         ('pir', data['pir'][i]),
349         ('pir_std', data['pir_std'][i]),
350         )
351
352     return (global_atts, var_atts, dim_inits, var_inits, var_data)
353
354 def updater(platform_info, sensor_info, data):
355     #
356     # subset data only to month being processed (see raw2proc.process())
357     i = data['in']
358
359     global_atts = {
360         # update times of data contained in file (yyyy-mm-dd HH:MM:SS)
361         # last date in monthly file
362         'end_date' : data['dt'][i][-1].strftime("%Y-%m-%d %H:%M:%S"),
363         'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
364         #
365         'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
366         }
367
368     # data variables
369     # update any variable attributes like range, min, max
370     var_atts = {}
371     # var_atts = {
372     #    'wtemp': {'max': max(data.u),
373     #          'min': min(data.v),
374     #          },
375     #    'cond': {'max': max(data.u),
376     #          'min': min(data.v),
377     #          },
378     #    }
379    
380     # data
381     var_data = (
382         ('time', data['time'][i]),
383         #
384         ('air_press', data['air_press'][i]),
385         ('rh', data['rh'][i]),
386         ('rh_std', data['rh_std'][i]),
387         ('air_temp', data['air_temp'][i]),
388         ('air_temp_std', data['air_temp_std'][i]),
389         ('rain', data['rain'][i]),
390         ('psp', data['psp'][i]),
391         ('psp_std', data['psp_std'][i]),
392         ('pir', data['pir'][i]),
393         ('pir_std', data['pir_std'][i]),
394         )
395
396     return (global_atts, var_atts, var_data)
397 #
Note: See TracBrowser for help on using the browser.