Revision Log

proc_nortek_wpa_adcp.py

Revision 320 (checked in by haines, 14 years ago)	catch-up trunk to production code running on cromwell

Line
1	#!/usr/bin/env python
2	# Last modified: Time-stamp: <2010-01-22 12:11:55 haines>
3	"""
4	how to parse data, and assert what data and info goes into
5	creating and updating monthly netcdf files
6
7	RDI/Wavesmon processed adcp current profile data
8
9	parser : sample date and time, currents, water temperature, pressure and water_depth
10
11	creator : lat, lon, z, time, ens, u, v, w, water_depth, water_temp (at tranducer depth), pressure
12	updator : time, ens, u, v, w, water_depth, water_temp (at tranducer depth), pressure
13
14
15	Examples
16	--------
17
18	>> (parse, create, update) = load_processors('proc_rdi_logdata_adcp')
19	or
20	>> si = get_config(cn+'.sensor_info')
21	>> (parse, create, update) = load_processors(si['adcp']['proc_module'])
22
23	>> lines = load_data(filename)
24	>> data = parse(platform_info, sensor_info, lines)
25	>> create(platform_info, sensor_info, data) or
26	>> update(platform_info, sensor_info, data)
27
28	"""
29
30
31	from raw2proc import *
32	from procutil import *
33	from ncutil import *
34
35	import seawater
36
37	now_dt = datetime.utcnow()
38	now_dt.replace(microsecond=0)
39
40	def parser(platform_info, sensor_info, lines):
41	"""
42	parse and assign ocean profile current data from Nortek AWAC ADCP Data
43
44	Notes
45	-----
46	1. This parser requires date/time be parsed from .wap for each to
47	get sig_wave_ht for determining depth of each bin and surface mask
48	and check time same as in .wpa file.
49
50	2. multiple profiles in one file separated by header w/time,
51	pitch, roll, heading, ducer pressure, bottom temp, top bin#
52	bottom bin# (??). The profile data is several lines one for
53	each bin.
54
55	MM DD YYYY HH MM SS ERR STATUS BATT SNDSPD HDG PITCH ROLL PRESS WTEMP ?? ?? TBIN BBIN
56	07 31 2008 23 54 00 0 48 18.2 1525.8 270.1 -2.4 0.2 10.503 21.64 0 0 3 34
57	1 0.9 0.071 24.04 0.029 0.065 -0.058 123 126 124
58	2 1.4 0.089 342.38 -0.027 0.085 -0.057 110 111 113
59	3 1.9 0.065 310.03 -0.050 0.042 -0.063 102 104 104
60	4 2.4 0.063 46.93 0.046 0.043 -0.045 93 95 99
61	5 2.9 0.049 355.33 -0.004 0.049 -0.047 87 89 92
62	...
63	NBIN DEPTH SPEED DIR U V W E1? E2? E3?
64	32 16.4 0.184 331.76 -0.087 0.162 -0.162 26 25 27
65	33 16.9 0.137 288.70 -0.130 0.044 -0.181 26 24 26
66	34 17.4 0.070 32.78 0.038 0.059 -0.248 25 25 26
67
68	3. not sure if depth column is hab or down from surface?
69
70
71	"""
72
73	# get sample datetime from filename
74	fn = sensor_info['fn']
75	sample_dt_start = filt_datetime(fn)[0]
76
77	nbins = sensor_info['nbins'] # Number of bins in data
78	nbursts = len(lines)/(nbins+1)
79
80	data = {
81	'dt' : numpy.array(numpy.ones((nbursts,), dtype=object)*numpy.nan),
82	'time' : numpy.array(numpy.ones((nbursts,), dtype=long)*numpy.nan),
83	'z' : numpy.array(numpy.ones((nbins,), dtype=float)*numpy.nan),
84	'u' : numpy.array(numpy.ones((nbursts,nbins), dtype=float)*numpy.nan),
85	'v' : numpy.array(numpy.ones((nbursts,nbins), dtype=float)*numpy.nan),
86	'w' : numpy.array(numpy.ones((nbursts,nbins), dtype=float)*numpy.nan),
87	'e1' : numpy.array(numpy.ones((nbursts,nbins), dtype=int)*numpy.nan),
88	'e2' : numpy.array(numpy.ones((nbursts,nbins), dtype=int)*numpy.nan),
89	'e3' : numpy.array(numpy.ones((nbursts,nbins), dtype=int)*numpy.nan),
90	'wd' : numpy.array(numpy.ones((nbursts), dtype=float)*numpy.nan),
91	'wl' : numpy.array(numpy.ones((nbursts), dtype=float)*numpy.nan),
92	'water_temp' : numpy.array(numpy.ones((nbursts), dtype=float)*numpy.nan),
93	'pressure' : numpy.array(numpy.ones((nbursts), dtype=float)*numpy.nan),
94	}
95
96	# these items can also be teased out of raw adcp but for now get from config file
97	th = sensor_info['transducer_ht'] # Transducer height above bottom (meters)
98	bh = sensor_info['blanking_ht'] # Blanking height above Transducer (meters)
99	bin_size = sensor_info['bin_size'] # Bin Size (meters)
100
101	# compute height for each bin above the bottom
102	bins = numpy.arange(1,nbins+1)
103	# bin_habs = (bins*bin_size+bin_size/2)+th+bh
104	bin_habs = (bins*bin_size+bin_size/2)+th+bh
105
106	# added by SH -- 15 Oct 2008
107	# raw2proc:ticket:27 adjust bin_habs along beam to nadir
108	# Nortek awac beam angle is fixed at 25 deg
109	# adjustment is cos(25 deg) (which is approx .90*height)
110	bin_habs = (bin_habsnumpy.cos(25.numpy.pi/180))
111	iaboveblank = bin_habs > th+bh+(bin_size)
112
113	# current profile count
114	i = 0
115
116	wpa = []
117	for line in lines:
118	wpa = []
119	# split line and parse float and integers
120	sw = re.split(' ', line)
121	for s in sw:
122	m = re.search(REAL_RE_STR, s)
123	if m:
124	wpa.append(float(m.groups()[0]))
125
126	if len(wpa)==19:
127	# get sample datetime from data
128	sample_str = '%02d-%02d-%4d %02d:%02d:%02d' % tuple(wpa[0:6])
129	if sensor_info['utc_offset']:
130	sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%Y %H:%M:%S') + \
131	timedelta(hours=sensor_info['utc_offset'])
132	else:
133	sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%Y %H:%M:%S')
134
135	# these items can also be teased out of raw adcp but for now get from config file
136	# th = sensor_info['transducer_ht'] # Transducer height above bottom (meters)
137
138	error_code = int(wpa[6])
139	status_code = int(wpa[7])
140	battery_voltage = wpa[8] # volts
141	sound_speed = wpa[9] # m/s
142	heading = wpa[10] # deg
143	pitch = wpa[11] # deg
144	roll = wpa[12] # deg
145
146	pressure = wpa[13] # dbar
147	# pressure (dbar) converted to water depth
148	wd = th + seawater.depth(pressure, platform_info['lat']) # m
149	temperature = wpa[14] # deg C
150
151	start_bin = int(wpa[17]) # first good bin from transducer (?)
152	wpa_nbins = int(wpa[18]) # Number of bins
153	# check this is same as in sensor_info
154
155	# initialize for new profile
156	hab = numpy.ones(nbins)*numpy.nan
157	spd = numpy.ones(nbins)*numpy.nan
158	dir = numpy.ones(nbins)*numpy.nan
159	u = numpy.ones(nbins)*numpy.nan
160	v = numpy.ones(nbins)*numpy.nan
161	w = numpy.ones(nbins)*numpy.nan
162	e1 = numpy.array(numpy.ones((nbins), dtype=int)*numpy.nan)
163	e2 = numpy.array(numpy.ones((nbins), dtype=int)*numpy.nan)
164	e3 = numpy.array(numpy.ones((nbins), dtype=int)*numpy.nan)
165
166	elif len(wpa)==10:
167	# current profile data at each bin
168	bin_number = wpa[0]
169	j = wpa[0]-1
170	print j
171	hab[j] = wpa[1]
172
173	spd[j] = wpa[2] # m/s
174	dir[j] = wpa[3] # deg N
175
176	u[j] = wpa[4] # m/s
177	v[j] = wpa[5] # m/s
178	w[j] = wpa[6] # m/s
179
180	e1[j] = int(wpa[7]) # echo dB ??
181	e2[j] = int(wpa[8]) #
182	e3[j] = int(wpa[9]) #
183
184	# ibad = (current_spd==-32768) \| (current_dir==-32768)
185	# current_spd[ibad] = numpy.nan
186	# current_dir[ibad] = numpy.nan
187
188	# if done reading profile, just read data for last bin
189	if bin_number==nbins:
190	# compute water mask
191	# if positive is up, in water is less than zero depth
192	bin_depths = (bin_habs)-(wd)
193	iwater = bin_depths+bin_size/2 < 0
194	iwater = iwater*iaboveblank
195
196	# use nominal water depth (MSL) averaged from full pressure record
197	# this should be checked/recalulated every so often
198	z = bin_habs+platform_info['mean_water_depth']
199
200	data['dt'][i] = sample_dt # sample datetime
201	data['time'][i] = dt2es(sample_dt) # sample time in epoch seconds
202	data['z'] = z
203	data['wd'][i] = -1*wd
204	data['wl'][i] = platform_info['mean_water_depth'] - (-1*wd)
205	data['water_temp'][i] = temperature
206	data['pressure'][i] = pressure
207
208	data['u'][i][iwater] = u[iwater]
209	data['v'][i][iwater] = v[iwater]
210	data['w'][i][iwater] = w[iwater]
211
212	data['e1'][i] = e1
213	data['e2'][i] = e2
214	data['e3'][i] = e3
215
216	# ready for next burst
217	i = i+1
218	# if j+1==nbins
219	# if len(wpa)==19 elif ==10
220	# for line
221
222	return data
223
224
225	def creator(platform_info, sensor_info, data):
226	#
227	#
228	title_str = sensor_info['description']+' at '+ platform_info['location']
229
230	if 'mean_water_depth' in platform_info.keys():
231	msl_str = platform_info['mean_water_depth']
232	else:
233	msl_str = 'None'
234	if 'mean_water_depth_time_period' in platform_info.keys():
235	msl_tp_str = platform_info['mean_water_depth_time_period']
236	else:
237	msl_tp_str = 'None'
238
239	global_atts = {
240	'title' : title_str,
241	'institution' : 'University of North Carolina at Chapel Hill (UNC-CH)',
242	'institution_url' : 'http://nccoos.unc.edu',
243	'institution_dods_url' : 'http://nccoos.unc.edu',
244	'metadata_url' : 'http://nccoos.unc.edu',
245	'references' : 'http://nccoos.unc.edu',
246	'contact' : 'Sara Haines (haines@email.unc.edu)',
247	#
248	'source' : 'fixed-profiler (acoustic doppler) observation',
249	'history' : 'raw2proc using ' + sensor_info['process_module'],
250	'comment' : 'File created using pycdf'+pycdfVersion()+' and numpy '+pycdfArrayPkg(),
251	# conventions
252	'Conventions' : 'CF-1.0; SEACOOS-CDL-v2.0',
253	# SEACOOS CDL codes
254	'format_category_code' : 'fixed-profiler',
255	'institution_code' : platform_info['institution'],
256	'platform_code' : platform_info['id'],
257	'package_code' : sensor_info['id'],
258	# institution specific
259	'project' : 'North Carolina Coastal Ocean Observing System (NCCOOS)',
260	'project_url' : 'http://nccoos.unc.edu',
261	# timeframe of data contained in file yyyy-mm-dd HH:MM:SS
262	# first date in monthly file
263	'start_date' : data['dt'][0].strftime("%Y-%m-%d %H:%M:%S"),
264	# last date in monthly file
265	'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"),
266	'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
267	#
268	'mean_water_depth' : msl_str,
269	'mean_water_depth_time_period' : msl_tp_str,
270	#
271	'creation_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
272	'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
273	'process_level' : 'level1',
274	#
275	# must type match to data (e.g. fillvalue is real if data is real)
276	'_FillValue' : numpy.nan,
277	}
278
279	var_atts = {
280	# coordinate variables
281	'time' : {'short_name': 'time',
282	'long_name': 'Time',
283	'standard_name': 'time',
284	'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC
285	'axis': 'T',
286	},
287	'lat' : {'short_name': 'lat',
288	'long_name': 'Latitude',
289	'standard_name': 'latitude',
290	'reference':'geographic coordinates',
291	'units': 'degrees_north',
292	'valid_range':(-90.,90.),
293	'axis': 'Y',
294	},
295	'lon' : {'short_name': 'lon',
296	'long_name': 'Longitude',
297	'standard_name': 'longitude',
298	'reference':'geographic coordinates',
299	'units': 'degrees_east',
300	'valid_range':(-180.,180.),
301	'axis': 'Y',
302	},
303	'z' : {'short_name': 'z',
304	'long_name': 'Height',
305	'standard_name': 'height',
306	'reference':'zero at mean-sea-level',
307	'positive' : 'up',
308	'units': 'm',
309	'axis': 'Z',
310	},
311	# data variables
312	'u': {'short_name' : 'u',
313	'long_name': 'East/West Component of Current',
314	'standard_name': 'eastward_current',
315	'units': 'm s-1',
316	'reference': 'clockwise from True East',
317	},
318	'v': {'short_name' : 'v',
319	'long_name': 'North/South Component of Current',
320	'standard_name': 'northward_current',
321	'units': 'm s-1',
322	'reference': 'clockwise from True North',
323	},
324	'w': {'short_name' : 'w',
325	'long_name': 'Vertical Component of Current',
326	'standard_name': 'upward_current',
327	'units': 'm s-1',
328	'reference': 'clockwise from True North',
329	},
330	'e1': {'short_name' : 'e1',
331	'long_name': 'Echo Beam 1 (??)',
332	'standard_name': 'beam_echo',
333	'units': 'dB',
334	},
335	'e2': {'short_name' : 'e2',
336	'long_name': 'Echo Beam 2 (??)',
337	'standard_name': 'beam_echo',
338	'units': 'dB',
339	},
340	'e3': {'short_name' : 'e3',
341	'long_name': 'Echo Beam 3 (??)',
342	'standard_name': 'beam_echo',
343	'units': 'dB',
344	},
345	'wd': {'short_name': 'wd',
346	'long_name': 'Water Depth',
347	'standard_name': 'water_depth',
348	'reference':'zero at surface',
349	'positive' : 'up',
350	'units': 'm',
351	},
352	'wl': {'short_name': 'wl',
353	'long_name': 'Water Level',
354	'standard_name': 'water_level',
355	'reference':'MSL',
356	'reference_to_MSL' : 0.,
357	'reference_MSL_datum' : platform_info['mean_water_depth'],
358	'reference_MSL_datum_time_period' : platform_info['mean_water_depth_time_period'],
359	'positive' : 'up',
360	'z' : 0.,
361	'units': 'm',
362	},
363	'pressure': {'short_name': 'p',
364	'long_name': 'Pressure',
365	'standard_name': 'pressure',
366	'units': 'dbar',
367	},
368	'water_temp': {'short_name': 'wtemp',
369	'long_name': 'Water Temperature at Transducer',
370	'standard_name': 'water_temperature',
371	'units': 'deg_C',
372	},
373	}
374
375
376	# dimension names use tuple so order of initialization is maintained
377	dim_inits = (
378	('ntime', NC.UNLIMITED),
379	('nlat', 1),
380	('nlon', 1),
381	('nz', sensor_info['nbins'])
382	)
383
384	# using tuple of tuples so order of initialization is maintained
385	# using dict for attributes order of init not important
386	# use dimension names not values
387	# (varName, varType, (dimName1, [dimName2], ...))
388	var_inits = (
389	# coordinate variables
390	('time', NC.INT, ('ntime',)),
391	('lat', NC.FLOAT, ('nlat',)),
392	('lon', NC.FLOAT, ('nlon',)),
393	('z', NC.FLOAT, ('nz',)),
394	# data variables
395	('u', NC.FLOAT, ('ntime', 'nz')),
396	('v', NC.FLOAT, ('ntime', 'nz')),
397	('w', NC.FLOAT, ('ntime', 'nz')),
398	('e1', NC.INT, ('ntime', 'nz')),
399	('e2', NC.INT, ('ntime', 'nz')),
400	('e3', NC.INT, ('ntime', 'nz')),
401	('wd', NC.FLOAT, ('ntime',)),
402	('wl', NC.FLOAT, ('ntime',)),
403	('pressure', NC.FLOAT, ('ntime',)),
404	('water_temp', NC.FLOAT, ('ntime',)),
405	)
406
407	# subset data only to month being processed (see raw2proc.process())
408	i = data['in']
409
410	# var data
411	var_data = (
412	('lat', platform_info['lat']),
413	('lon', platform_info['lon']),
414	('z', data['z']),
415	#
416	('time', data['time'][i]),
417	('u', data['u'][i]),
418	('v', data['v'][i]),
419	('w', data['w'][i]),
420	('e1', data['e1'][i]),
421	('e2', data['e2'][i]),
422	('e3', data['e3'][i]),
423	('wd', data['wd'][i]),
424	('wl', data['wl'][i]),
425	('pressure', data['pressure'][i]),
426	('water_temp', data['water_temp'][i]),
427	)
428
429	return (global_atts, var_atts, dim_inits, var_inits, var_data)
430
431	def updater(platform_info, sensor_info, data):
432	#
433	global_atts = {
434	# update times of data contained in file (yyyy-mm-dd HH:MM:SS)
435	# last date in monthly file
436	'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"),
437	'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
438	#
439	'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
440	}
441
442	# data variables
443	# update any variable attributes like range, min, max
444	var_atts = {}
445	# var_atts = {
446	# 'u': {'max': max(data.u),
447	# 'min': min(data.v),
448	# },
449	# 'v': {'max': max(data.u),
450	# 'min': min(data.v),
451	# },
452	# }
453
454	# subset data only to month being processed (see raw2proc.process())
455	i = data['in']
456
457	# data
458	var_data = (
459	('time', data['time'][i]),
460	('u', data['u'][i]),
461	('v', data['v'][i]),
462	('w', data['w'][i]),
463	('e1', data['e1'][i]),
464	('e2', data['e2'][i]),
465	('e3', data['e3'][i]),
466	('wd', data['wd'][i]),
467	('wl', data['wl'][i]),
468	('pressure', data['pressure'][i]),
469	('water_temp', data['water_temp'][i]),
470	)
471
472	return (global_atts, var_atts, var_data)
473	#

Note: See TracBrowser for help on using the browser.

root/raw2proc/tags/raw2proc-1.0/proc_nortek_wpa_adcp.py

Download in other formats: