Revision Log

proc_cr1000_met.py

Revision 511 (checked in by cbc, 10 years ago)	Change to axis metadata for Scintec sodar.

Line
1	#!/usr/bin/env python
2	# Last modified: Time-stamp: <2013-11-26 09:02:09 haines>
3	"""
4	how to parse data, and assert what data and info goes into
5	creating and updating monthly netcdf files
6
7	parse data met data collected on Campbell Scientific DataLogger (loggernet) (csi)
8
9	parser : sample date and time,
10
11	creator : lat, lon, z, time,
12	updator : time,
13
14
15	Examples
16	--------
17
18	>> (parse, create, update) = load_processors('proc_csi_adcp_v2')
19	or
20	>> si = get_config(cn+'.sensor_info')
21	>> (parse, create, update) = load_processors(si['adcp']['proc_module'])
22
23	>> lines = load_data(filename)
24	>> data = parse(platform_info, sensor_info, lines)
25	>> create(platform_info, sensor_info, data) or
26	>> update(platform_info, sensor_info, data)
27
28	"""
29
30
31	from raw2proc import *
32	from procutil import *
33	from ncutil import *
34
35	now_dt = datetime.utcnow()
36	now_dt.replace(microsecond=0)
37
38	def parser(platform_info, sensor_info, lines):
39	"""
40	Example met data
41
42	"TOA5","CR1000_B1","CR1000","37541","CR1000.Std.21","CPU:NCWIND_12_Buoy_All.CR1","58723","AMet_6Min"
43	"TIMESTAMP","RECORD","Baro_mbar_Avg","RHumidity_Avg","RHumidity_Std","AirTempC_Avg","AirTempC_Std","Rain","Psp_Avg","Psp_Std","Pir_Wm2_Avg","Pir_Wm2_Std"
44	"TS","RN","","","","","","","","","",""
45	"","","Avg","Avg","Std","Avg","Std","Smp","Avg","Std","Avg","Std"
46	"2011-11-01 00:00:59",4590,14.3792,75.59,0.579,15.67,0.05,-22.35,1197.037,45.58967,371.5126,0.9030571
47	"2011-11-01 00:06:59",4591,14.37995,74.96,0.912,16.61,0.048,-21,-1071.813,129.5147,381.2539,0.2076943
48	"2011-11-01 00:12:59",4592,14.3792,72.71,2.677,17.29,0.032,-15.58,-2056.658,0,381.1828,0.1402813
49	"2011-11-01 00:18:59",4593,14.3791,72.63,0.928,17.67,0.041,-19.64,-1895.86,9.866026,381.0333,0.2442325
50
51	"""
52
53	import numpy
54	from datetime import datetime
55	from time import strptime
56
57	# get sample datetime from filename
58	fn = sensor_info['fn']
59	sample_dt_start = filt_datetime(fn)
60
61	# how many samples (don't count header 4 lines)
62	nsamp = len(lines[4:])
63
64	N = nsamp
65	data = {
66	'dt' : numpy.array(numpy.ones((N,), dtype=object)*numpy.nan),
67	'time' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan),
68	'air_press' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
69	'rh' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
70	'rh_std' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
71	'air_temp' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
72	'air_temp_std' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
73	'rain' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
74	'psp' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
75	'psp_std' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
76	'pir' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
77	'pir_std' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
78	}
79
80	# sample count
81	i = 0
82
83	for line in lines[4:]:
84	csi = []
85	# split line
86	sw = re.split(',', line)
87	if len(sw)<=0:
88	print ' ... skipping line %d ' % (i,)
89	continue
90
91	# replace any "NAN" text with a number
92	for index, s in enumerate(sw):
93	m = re.search(NAN_RE_STR, s)
94	if m:
95	sw[index] = '-99999'
96
97	# parse date-time, and all other float and integers
98	for s in sw[1:]:
99	m = re.search(REAL_RE_STR, s)
100	if m:
101	csi.append(float(m.groups()[0]))
102
103	if sensor_info['utc_offset']:
104	sample_dt = scanf_datetime(sw[0], fmt='"%Y-%m-%d %H:%M:%S"') + \
105	timedelta(hours=sensor_info['utc_offset'])
106	else:
107	sample_dt = scanf_datetime(sw[0], fmt='"%Y-%m-%d %H:%M:%S"')
108
109	data['dt'][i] = sample_dt # sample datetime
110	data['time'][i] = dt2es(sample_dt) # sample time in epoch seconds
111
112	if len(csi)==11:
113	#
114	# data['samplenum'][i] = csi[0] # sample number assigned by datalogger in table
115	data['air_press'][i] = csi[1] # Campbell Sci (Viasala) CS106 barometer (mbar)
116	# Before Jan 2012, Heise Barometer (psi) to mbar
117	data['rh'][i] = csi[2] # relative humidity avg (60 samples for 1 min)
118	data['rh_std'][i] = csi[3] # relative humidity std
119	data['air_temp'][i] = csi[4] # air temperature avg (deg C)
120	data['air_temp_std'][i] = csi[5] # air temperature std (deg C)
121	data['rain'][i] = csi[6]/100. # precip gauge cummulative (mm)
122	data['psp'][i] = csi[7] # PSP avg
123	data['psp_std'][i] = csi[8] # PSP std
124	data['pir'][i] = csi[9] # PIR avg (W m-2)
125	data['pir_std'][i] = csi[10] # PIR std (W m-2)
126	i=i+1
127	else:
128	print ' ... skipping line %d -- %s ' % (i,line)
129	continue
130
131	# if re.search
132	# for line
133
134	# Specific to buoys using CR1000 in Fall of 2011
135	# prior to Jan 01, 2012, pressure sensor was a Heise with units psi
136	# afterwards, Campbell Sci CS106 in units mbar,
137	# also handle b1/b2 PSP data for each buoy
138	if data['dt'][0] < datetime(2012, 1, 1):
139	data['air_press'] = udconvert(data['air_press'], 'psi', 'mbar')[0]
140	# specific to buoy B1 and B2
141	if platform_info['id'] == 'b1':
142	data['psp'] = -1data['psp'*]/1000
143	data['psp_std'] = -1data['psp_std'*]/1000
144	if platform_info['id'] == 'b2':
145	data['psp'] = numpy.nandata['psp'*]
146	data['psp_std'] = numpy.nandata['psp_std'*]
147
148	# some QC
149	# good = -40<at & at<60 # does not work
150	# good = (-40<at) & (at<60) # THIS WORKS!
151	good = (5<data['air_temp']) & (data['air_temp']<30)
152	bad = ~good
153	data['air_temp'][bad] = numpy.nan
154	data['air_temp_std'][bad] = numpy.nan
155	data['rh'][bad] = numpy.nan
156	data['rh_std'][bad] = numpy.nan
157	data['rain'][bad] = numpy.nan
158
159	# check that no data[dt] is set to Nan or anything but datetime
160	# keep only data that has a resolved datetime
161	keep = numpy.array([type(datetime(1970,1,1)) == type(dt) for dt in data['dt'][:]])
162	if keep.any():
163	for param in data.keys():
164	data[param] = data[param][keep]
165
166	return data
167
168
169	def creator(platform_info, sensor_info, data):
170	#
171	#
172	# subset data only to month being processed (see raw2proc.process())
173	i = data['in']
174
175	title_str = sensor_info['description']+' at '+ platform_info['location']
176	global_atts = {
177	'title' : title_str,
178	'institution' : platform_info['institution'],
179	'institution_url' : platform_info['institution_url'],
180	'institution_dods_url' : platform_info['institution_dods_url'],
181	'metadata_url' : platform_info['metadata_url'],
182	'references' : platform_info['references'],
183	'contact' : platform_info['contact'],
184	#
185	'source' : platform_info['source']+' '+sensor_info['source'],
186	'history' : 'raw2proc using ' + sensor_info['process_module'],
187	'comment' : 'File created using pycdf'+pycdfVersion()+' and numpy '+pycdfArrayPkg(),
188	# conventions
189	'Conventions' : platform_info['conventions'],
190	# SEACOOS CDL codes
191	'format_category_code' : platform_info['format_category_code'],
192	'institution_code' : platform_info['institution_code'],
193	'platform_code' : platform_info['id'],
194	'package_code' : sensor_info['id'],
195	# institution specific
196	'project' : platform_info['project'],
197	'project_url' : platform_info['project_url'],
198	# timeframe of data contained in file yyyy-mm-dd HH:MM:SS
199	# first date in monthly file
200	'start_date' : data['dt'][i][0].strftime("%Y-%m-%d %H:%M:%S"),
201	# last date in monthly file
202	'end_date' : data['dt'][i][-1].strftime("%Y-%m-%d %H:%M:%S"),
203	'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
204	#
205	'creation_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
206	'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
207	'process_level' : 'level1',
208	#
209	# must type match to data (e.g. fillvalue is real if data is real)
210	'_FillValue' : -99999.,
211	}
212
213	var_atts = {
214	# coordinate variables
215	'time' : {'short_name': 'time',
216	'long_name': 'Time',
217	'standard_name': 'time',
218	'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC
219	'axis': 'T',
220	},
221	'lat' : {'short_name': 'lat',
222	'long_name': 'Latitude',
223	'standard_name': 'latitude',
224	'reference':'geographic coordinates',
225	'units': 'degrees_north',
226	'valid_range':(-90.,90.),
227	'axis': 'Y',
228	},
229	'lon' : {'short_name': 'lon',
230	'long_name': 'Longitude',
231	'standard_name': 'longitude',
232	'reference':'geographic coordinates',
233	'units': 'degrees_east',
234	'valid_range':(-180.,180.),
235	'axis': 'Y',
236	},
237	'z' : {'short_name': 'z',
238	'long_name': 'Altitude',
239	'standard_name': 'altitude',
240	'reference':'zero at mean sea level',
241	'positive' : 'up',
242	'units': 'm',
243	'axis': 'Z',
244	},
245	# data variables
246	'air_press': {'short_name': 'air_press',
247	'long_name': 'Air Pressure',
248	'standard_name': 'air_pressure',
249	'units': 'mbar',
250	'z': sensor_info['barometer_height'],
251	'z_units' : 'meter',
252	},
253	'air_temp': {'short_name': 'air_temp',
254	'long_name': 'Air Temperature',
255	'standard_name': 'air_temperature',
256	'units': 'degC',
257	'z': sensor_info['temperature_height'],
258	'z_units' : 'meter',
259	},
260	'air_temp_std': {'short_name': 'air_temp_std',
261	'long_name': 'Standard Deviation of Air Temperature',
262	'standard_name': 'air_temperature',
263	'units': 'degC',
264	},
265	'rh': {'short_name': 'rh',
266	'long_name': 'Relative Humidity',
267	'standard_name': 'relative_humidity',
268	'units': '%',
269	'z': sensor_info['temperature_height'],
270	'z_units' : 'meter',
271	},
272	'rh_std': {'short_name': 'rh_std',
273	'long_name': 'Standard Deviation of Relative Humidity',
274	'standard_name': 'relative_humidity',
275	'units': '%',
276	},
277	'rain': {'short_name': 'rain',
278	'long_name': '6-Minute Rain',
279	'standard_name': 'rain',
280	'units': 'inches',
281	},
282	'psp': {'short_name': 'psp',
283	'long_name': 'Short-wave Radiation',
284	'standard_name': 'downwelling_shortwave_irradiance',
285	'units': 'W m-2',
286	},
287	'psp_std': {'short_name': 'psp_std',
288	'long_name': 'Standard Deviation of Short-wave Radiation',
289	'standard_name': 'shortwave_radiation',
290	'units': 'W m-2',
291	},
292	'pir': {'short_name': 'pir',
293	'long_name': 'Long-wave Radiation',
294	'standard_name': 'longwave_radiation',
295	'units': 'W m-2',
296	},
297	'pir_std': {'short_name': 'pir_std',
298	'long_name': 'Standard Deviation of Long-wave Radiation',
299	'standard_name': 'longwave_radiation',
300	'units': 'W m-2',
301	},
302	}
303
304	# dimension names use tuple so order of initialization is maintained
305	dim_inits = (
306	('ntime', NC.UNLIMITED),
307	('nlat', 1),
308	('nlon', 1),
309	('nz', 1),
310	)
311
312	# using tuple of tuples so order of initialization is maintained
313	# using dict for attributes order of init not important
314	# use dimension names not values
315	# (varName, varType, (dimName1, [dimName2], ...))
316	var_inits = (
317	# coordinate variables
318	('time', NC.INT, ('ntime',)),
319	('lat', NC.FLOAT, ('nlat',)),
320	('lon', NC.FLOAT, ('nlon',)),
321	('z', NC.FLOAT, ('nz',)),
322	# data variables
323	('air_press', NC.FLOAT, ('ntime',)),
324	('rh', NC.FLOAT, ('ntime',)),
325	('rh_std', NC.FLOAT, ('ntime',)),
326	('air_temp', NC.FLOAT, ('ntime',)),
327	('air_temp_std', NC.FLOAT, ('ntime',)),
328	('rain', NC.FLOAT, ('ntime',)),
329	('psp', NC.FLOAT, ('ntime',)),
330	('psp_std', NC.FLOAT, ('ntime',)),
331	('pir', NC.FLOAT, ('ntime',)),
332	('pir_std', NC.FLOAT, ('ntime',)),
333	)
334
335	# subset data only to month being processed (see raw2proc.process())
336	i = data['in']
337
338	# var data
339	var_data = (
340	('lat', platform_info['lat']),
341	('lon', platform_info['lon']),
342	('z', platform_info['altitude']),
343	#
344	('time', data['time'][i]),
345	#
346	('air_press', data['air_press'][i]),
347	('rh', data['rh'][i]),
348	('rh_std', data['rh_std'][i]),
349	('air_temp', data['air_temp'][i]),
350	('air_temp_std', data['air_temp_std'][i]),
351	('rain', data['rain'][i]),
352	('psp', data['psp'][i]),
353	('psp_std', data['psp_std'][i]),
354	('pir', data['pir'][i]),
355	('pir_std', data['pir_std'][i]),
356	)
357
358	return (global_atts, var_atts, dim_inits, var_inits, var_data)
359
360	def updater(platform_info, sensor_info, data):
361	#
362	# subset data only to month being processed (see raw2proc.process())
363	i = data['in']
364
365	global_atts = {
366	# update times of data contained in file (yyyy-mm-dd HH:MM:SS)
367	# last date in monthly file
368	'end_date' : data['dt'][i][-1].strftime("%Y-%m-%d %H:%M:%S"),
369	'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
370	#
371	'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
372	}
373
374	# data variables
375	# update any variable attributes like range, min, max
376	var_atts = {}
377	# var_atts = {
378	# 'wtemp': {'max': max(data.u),
379	# 'min': min(data.v),
380	# },
381	# 'cond': {'max': max(data.u),
382	# 'min': min(data.v),
383	# },
384	# }
385
386	# data
387	var_data = (
388	('time', data['time'][i]),
389	#
390	('air_press', data['air_press'][i]),
391	('rh', data['rh'][i]),
392	('rh_std', data['rh_std'][i]),
393	('air_temp', data['air_temp'][i]),
394	('air_temp_std', data['air_temp_std'][i]),
395	('rain', data['rain'][i]),
396	('psp', data['psp'][i]),
397	('psp_std', data['psp_std'][i]),
398	('pir', data['pir'][i]),
399	('pir_std', data['pir_std'][i]),
400	)
401
402	return (global_atts, var_atts, var_data)
403	#
404

Note: See TracBrowser for help on using the browser.

root/raw2proc/trunk/raw2proc/proc_cr1000_met.py

Download in other formats: