Revision Log

proc_cr1000_wind.py

Revision 478 (checked in by haines, 12 years ago)	Add Billy Mitchell config for 15-195 meters altitude.

Line
1	#!/usr/bin/env python
2	# Last modified: Time-stamp: <2011-12-16 14:36:47 haines>
3	"""
4	how to parse data, and assert what data and info goes into
5	creating and updating monthly netcdf files
6
7	parse data met data collected on Campbell Scientific DataLogger (loggernet) (csi)
8
9	parser : sample date and time,
10
11	creator : lat, lon, z, time,
12	updator : time,
13
14
15	Examples
16	--------
17
18	>> (parse, create, update) = load_processors('proc_csi_adcp_v2')
19	or
20	>> si = get_config(cn+'.sensor_info')
21	>> (parse, create, update) = load_processors(si['adcp']['proc_module'])
22
23	>> lines = load_data(filename)
24	>> data = parse(platform_info, sensor_info, lines)
25	>> create(platform_info, sensor_info, data) or
26	>> update(platform_info, sensor_info, data)
27
28	"""
29
30
31	from raw2proc import *
32	from procutil import *
33	from ncutil import *
34
35	now_dt = datetime.utcnow()
36	now_dt.replace(microsecond=0)
37
38	def parser(platform_info, sensor_info, lines):
39	"""
40	Example wind data
41
42	Stats (avg, std, and max) for wind sampled every second for one minute DURING a 6 minute time period. Stats are NOT over 6 minutes, as
43	the time stamp would have you believe.
44
45	"TOA5","CR1000_B1","CR1000","37541","CR1000.Std.21","CPU:NCWIND_12_Buoy_All.CR1","58723","AWind_6Min"
46	"TIMESTAMP","RECORD","W1_SpeedAvg","W1_DirAvg","W1_SpeedMax","W1_SpeedStd","W2_SpeedAvg","W2_DirAvg","W2_SpeedMax","W2_SpeedStd"
47	"TS","RN","","Deg","","","","Deg","",""
48	"","","WVc","WVc","Max","Std","WVc","WVc","Max","Std"
49	"2011-12-01 00:01:59",6507,8.32,319.1,10.09,0.781,8.15,310.9,10.09,0.832
50	"2011-12-01 00:07:59",6508,9.43,323.3,11.27,1.094,9.11,315.8,10.68,1.015
51	"2011-12-01 00:13:59",6509,9.94,308.6,12.35,1.077,9.74,301.3,11.96,1.027
52	"2011-12-01 00:19:59",6510,8.86,304.5,10.98,1.003,8.8,296.4,11.27,1.066
53	"2011-12-01 00:25:59",6511,9.02,310.8,10.98,1.023,8.95,302.4,10.78,0.964
54	"2011-12-01 00:31:59",6512,9.58,304.9,11.76,1.156,9.39,296.7,11.76,1.167
55
56	"""
57
58	import numpy
59	from datetime import datetime
60	from time import strptime
61
62	# get sample datetime from filename
63	fn = sensor_info['fn']
64	sample_dt_start = filt_datetime(fn)
65
66	# how many samples (don't count header 4 lines)
67	nsamp = len(lines[4:])
68
69	N = nsamp
70	data = {
71	'dt' : numpy.array(numpy.ones((N,), dtype=object)*numpy.nan),
72	'time' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan),
73	'wspd1' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
74	'wspd1_std' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
75	'wgust1' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
76	'wdir1' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
77	'wspd2' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
78	'wspd2_std' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
79	'wgust2' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
80	'wdir2' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
81	}
82
83	# sample count
84	i = 0
85
86	for line in lines[4:]:
87	csi = []
88	# split line
89	sw = re.split(',', line)
90	if len(sw)<=0:
91	print ' ... skipping line %d ' % (i,)
92	continue
93
94	# replace any "NAN" text with a number
95	for index, s in enumerate(sw):
96	m = re.search(NAN_RE_STR, s)
97	if m:
98	sw[index] = '-99999'
99
100	# parse date-time, and all other float and integers
101	for s in sw[1:]:
102	m = re.search(REAL_RE_STR, s)
103	if m:
104	csi.append(float(m.groups()[0]))
105
106	if sensor_info['utc_offset']:
107	sample_dt = scanf_datetime(sw[0], fmt='"%Y-%m-%d %H:%M:%S"') + \
108	timedelta(hours=sensor_info['utc_offset'])
109	else:
110	sample_dt = scanf_datetime(sw[0], fmt='"%Y-%m-%d %H:%M:%S"')
111
112	data['dt'][i] = sample_dt # sample datetime
113	data['time'][i] = dt2es(sample_dt) # sample time in epoch seconds
114
115	if len(csi)==9:
116	#
117	# data['samplenum'][i] = csi[0] # sample number assigned by datalogger in table
118	data['wspd1'][i] = csi[1] #
119	data['wdir1'][i] = csi[2] #
120	data['wgust1'][i] = csi[3] # relative humidity std
121	data['wspd1_std'][i] = csi[4] # air temperature avg (deg C)
122	data['wspd2'][i] = csi[5] # air temperature std (deg C)
123	data['wdir2'][i] = csi[6] # precip gauge cummulative
124	data['wgust2'][i] = csi[7] # PSP avg
125	data['wspd2_std'][i] = csi[8] # PSP std
126	i=i+1
127	else:
128	print ' ... skipping line %d -- %s ' % (i,line)
129	continue
130
131	# if re.search
132	# for line
133
134
135	# check that no data[dt] is set to Nan or anything but datetime
136	# keep only data that has a resolved datetime
137	keep = numpy.array([type(datetime(1970,1,1)) == type(dt) for dt in data['dt'][:]])
138	if keep.any():
139	for param in data.keys():
140	data[param] = data[param][keep]
141
142	return data
143
144
145	def creator(platform_info, sensor_info, data):
146	#
147	#
148	title_str = sensor_info['description']+' at '+ platform_info['location']
149	global_atts = {
150	'title' : title_str,
151	'institution' : 'University of North Carolina at Chapel Hill (UNC-CH)',
152	'institution_url' : 'http://nccoos.org',
153	'institution_dods_url' : 'http://nccoos.org',
154	'metadata_url' : 'http://nccoos.org',
155	'references' : 'http://nccoos.org',
156	'contact' : 'Sara Haines (haines@email.unc.edu)',
157	#
158	'source' : 'buoy station',
159	'history' : 'raw2proc using ' + sensor_info['process_module'],
160	'comment' : 'File created using pycdf'+pycdfVersion()+' and numpy '+pycdfArrayPkg(),
161	# conventions
162	'Conventions' : 'CF-1.0; SEACOOS-CDL-v2.0',
163	# SEACOOS CDL codes
164	'format_category_code' : 'fixed-point',
165	'institution_code' : platform_info['institution'],
166	'platform_code' : platform_info['id'],
167	'package_code' : sensor_info['id'],
168	# institution specific
169	'project' : 'North Carolina Coastal Ocean Observing System (NCCOOS)',
170	'project_url' : 'http://nccoos.org',
171	# timeframe of data contained in file yyyy-mm-dd HH:MM:SS
172	'start_date' : data['dt'][0].strftime("%Y-%m-%d %H:%M:%S"),
173	'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"),
174	'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
175	#
176	'creation_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
177	'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
178	'process_level' : 'level1',
179	#
180	# must type match to data (e.g. fillvalue is real if data is real)
181	'_FillValue' : -99999.,
182	}
183
184	var_atts = {
185	# coordinate variables
186	'time' : {'short_name': 'time',
187	'long_name': 'Time',
188	'standard_name': 'time',
189	'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC
190	'axis': 'T',
191	},
192	'lat' : {'short_name': 'lat',
193	'long_name': 'Latitude',
194	'standard_name': 'latitude',
195	'reference':'geographic coordinates',
196	'units': 'degrees_north',
197	'valid_range':(-90.,90.),
198	'axis': 'Y',
199	},
200	'lon' : {'short_name': 'lon',
201	'long_name': 'Longitude',
202	'standard_name': 'longitude',
203	'reference':'geographic coordinates',
204	'units': 'degrees_east',
205	'valid_range':(-180.,180.),
206	'axis': 'Y',
207	},
208	'z' : {'short_name': 'z',
209	'long_name': 'Altitude',
210	'standard_name': 'altitude',
211	'reference':'zero at mean sea level',
212	'positive' : 'up',
213	'units': 'm',
214	'axis': 'Z',
215	},
216	# data variables
217	'wspd1' : {'short_name': 'wspd',
218	'long_name': 'Wind Speed',
219	'standard_name': 'wind_speed',
220	'units': 'm s-1',
221	'can_be_normalized': 'no',
222	'z' : sensor_info['anemometer1_height'],
223	'z_units' : 'meter',
224	},
225	'wdir1' : {'short_name': 'wdir',
226	'long_name': 'Wind Direction from',
227	'standard_name': 'wind_from_direction',
228	'reference': 'clockwise from Magnetic North',
229	'valid_range': (0., 360),
230	'units': 'degrees',
231	'z' : sensor_info['anemometer1_height'],
232	'z_units' : 'meter',
233	},
234	'wgust1' : {'short_name': 'wgust',
235	'long_name': 'Wind Gust',
236	'standard_name': 'wind_gust',
237	'units': 'm s-1',
238	'can_be_normalized': 'no',
239	'z' : sensor_info['anemometer1_height'],
240	'z_units' : 'meter',
241	},
242	'wspd1_std' : {'short_name': 'wspd std',
243	'long_name': 'Standard Deviation of Wind Speed ',
244	'standard_name': 'wind_speed standard_deviation',
245	'units': 'm s-1',
246	'can_be_normalized': 'no',
247	'z' : sensor_info['anemometer1_height'],
248	'z_units' : 'meter',
249	},
250	# Second anemometer
251	'wspd2' : {'short_name': 'wspd',
252	'long_name': 'Wind Speed',
253	'standard_name': 'wind_speed',
254	'units': 'm s-1',
255	'can_be_normalized': 'no',
256	'z' : sensor_info['anemometer2_height'],
257	'z_units' : 'meter',
258	},
259	'wdir2' : {'short_name': 'wdir',
260	'long_name': 'Wind Direction from',
261	'standard_name': 'wind_from_direction',
262	'reference': 'clockwise from Magnetic North',
263	'valid_range': (0., 360),
264	'units': 'degrees',
265	'z' : sensor_info['anemometer2_height'],
266	'z_units' : 'meter',
267	},
268	'wgust2' : {'short_name': 'wgust',
269	'long_name': 'Wind Gust',
270	'standard_name': 'wind_gust',
271	'units': 'm s-1',
272	'can_be_normalized': 'no',
273	'z' : sensor_info['anemometer2_height'],
274	'z_units' : 'meter',
275	},
276	'wspd2_std' : {'short_name': 'wspd std',
277	'long_name': 'Standard Deviation of Wind Speed ',
278	'standard_name': 'wind_speed standard_deviation',
279	'units': 'm s-1',
280	'can_be_normalized': 'no',
281	'z' : sensor_info['anemometer2_height'],
282	'z_units' : 'meter',
283	},
284	}
285
286	# dimension names use tuple so order of initialization is maintained
287	dim_inits = (
288	('ntime', NC.UNLIMITED),
289	('nlat', 1),
290	('nlon', 1),
291	('nz', 1),
292	)
293
294	# using tuple of tuples so order of initialization is maintained
295	# using dict for attributes order of init not important
296	# use dimension names not values
297	# (varName, varType, (dimName1, [dimName2], ...))
298	var_inits = (
299	# coordinate variables
300	('time', NC.INT, ('ntime',)),
301	('lat', NC.FLOAT, ('nlat',)),
302	('lon', NC.FLOAT, ('nlon',)),
303	('z', NC.FLOAT, ('nz',)),
304	# data variables
305	('wspd1', NC.FLOAT, ('ntime',)),
306	('wdir1', NC.FLOAT, ('ntime',)),
307	('wgust1', NC.FLOAT, ('ntime',)),
308	('wspd1_std', NC.FLOAT, ('ntime',)),
309	('wspd2', NC.FLOAT, ('ntime',)),
310	('wdir2', NC.FLOAT, ('ntime',)),
311	('wgust2', NC.FLOAT, ('ntime',)),
312	('wspd2_std', NC.FLOAT, ('ntime',)),
313	)
314
315	# subset data only to month being processed (see raw2proc.process())
316	i = data['in']
317
318	# var data
319	var_data = (
320	('lat', platform_info['lat']),
321	('lon', platform_info['lon']),
322	('z', platform_info['altitude']),
323	#
324	('time', data['time'][i]),
325	#
326	('wspd1', data['wspd1'][i]),
327	('wdir1', data['wdir1'][i]),
328	('wgust1', data['wgust1'][i]),
329	('wspd1_std', data['wspd1_std'][i]),
330	('wspd2', data['wspd2'][i]),
331	('wdir2', data['wdir2'][i]),
332	('wgust2', data['wgust2'][i]),
333	('wspd2_std', data['wspd2_std'][i]),
334	)
335
336	return (global_atts, var_atts, dim_inits, var_inits, var_data)
337
338	def updater(platform_info, sensor_info, data):
339	#
340	global_atts = {
341	# update times of data contained in file (yyyy-mm-dd HH:MM:SS)
342	# last date in monthly file
343	'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"),
344	'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
345	#
346	'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
347	}
348
349	# data variables
350	# update any variable attributes like range, min, max
351	var_atts = {}
352	# var_atts = {
353	# 'wtemp': {'max': max(data.u),
354	# 'min': min(data.v),
355	# },
356	# 'cond': {'max': max(data.u),
357	# 'min': min(data.v),
358	# },
359	# }
360
361	# subset data only to month being processed (see raw2proc.process())
362	i = data['in']
363
364	# data
365	var_data = (
366	('time', data['time'][i]),
367	#
368	('wspd1', data['wspd1'][i]),
369	('wdir1', data['wdir1'][i]),
370	('wgust1', data['wgust1'][i]),
371	('wspd1_std', data['wspd1_std'][i]),
372	('wspd2', data['wspd2'][i]),
373	('wdir2', data['wdir2'][i]),
374	('wgust2', data['wgust2'][i]),
375	('wspd2_std', data['wspd2_std'][i]),
376	)
377
378	return (global_atts, var_atts, var_data)
379	#

Note: See TracBrowser for help on using the browser.

root/raw2proc/trunk/raw2proc/proc_cr1000_wind.py

Download in other formats: