Revision Log

proc_mini_andi_current.py

Revision 375 (checked in by cbc, 14 years ago)	Process and config modules for Spongenet.

Line
1	"""
2	Parse data and assert what data creates and updates monthly NetCDF files.
3
4	Spongenet mini_andi current parameters sponge data.
5	"""
6
7	import math
8	import numpy as n
9	import pycdf
10	import datetime
11	import procutil
12	from spongenet.parse import Data
13
14	nowDt = datetime.datetime.utcnow().replace(microsecond=0)
15
16	def parser(platform_info, sensor_info, lines):
17	"""
18	Parse and assign sponge data from XML file.
19	"""
20
21	_data = Data(''.join(lines))
22
23	# Each Device tag represents a time sample.
24	num_samples = len(_data.devices)
25
26	data = {
27	'dt' : n.array(n.ones((num_samples,)) * n.nan, dtype=object),
28	'time' : n.array(n.ones((num_samples,)) * n.nan, dtype=long),
29	'pdt' : n.array(n.ones((num_samples,)) * n.nan, dtype=object),
30	'ptime' : n.array(n.ones((num_samples,)) * n.nan, dtype=long),
31	'ds' : n.array(n.ones((num_samples,)) * n.nan, dtype=object),
32	'session' : n.array(n.ones((num_samples,)) * n.nan, dtype=long),
33	'pds' : n.array(n.ones((num_samples,)) * n.nan, dtype=object),
34	'psession' : n.array(n.ones((num_samples,)) * n.nan, dtype=long),
35	'record' : n.array(n.ones((num_samples,)) * n.nan, dtype=int),
36	'status' : n.array(n.ones((num_samples,)) * n.nan, dtype=int),
37	'pstatus' : n.array(n.ones((num_samples,)) * n.nan, dtype=int),
38	'abs_speed' : n.array(n.ones((num_samples,)) * n.nan, dtype=float),
39	'direction' : n.array(n.ones((num_samples,)) * n.nan, dtype=float),
40	'v' : n.array(n.ones((num_samples,)) * n.nan, dtype=float),
41	'u' : n.array(n.ones((num_samples,)) * n.nan, dtype=float),
42	'heading' : n.array(n.ones((num_samples,)) * n.nan, dtype=float),
43	'tiltx' : n.array(n.ones((num_samples,)) * n.nan, dtype=float),
44	'tilty' : n.array(n.ones((num_samples,)) * n.nan, dtype=float),
45	'std_speed' : n.array(n.ones((num_samples,)) * n.nan, dtype=float),
46	'strength' : n.array(n.ones((num_samples,)) * n.nan, dtype=float),
47	'pings' : n.array(n.ones((num_samples,)) * n.nan, dtype=int),
48	}
49
50	for (sample_index, sample) in enumerate(_data.devices):
51	# sample time at the platform
52	dt = {'month' : int(sample['time'][5:7]),
53	'day' : int(sample['time'][8:10]),
54	'year' : int(sample['time'][0:4]),
55	'hour' : int(sample['time'][11:13]),
56	'min' : int(sample['time'][14:16]),
57	'sec' : int(sample['time'][17:19]),
58	}
59	dt = '%(month)02d-%(day)02d-%(year)04d %(hour)02d:%(min)02d:%(sec)02d' \
60	% dt
61	dt = procutil.scanf_datetime(dt, fmt='%m-%d-%Y %H:%M:%S')
62	if sensor_info['utc_offset']:
63	dt = dt + datetime.timedelta(hours=sensor_info['utc_offset'])
64	data['dt'][sample_index] = dt
65	data['time'][sample_index] = procutil.dt2es(dt)
66
67	# sample time at the package
68	package_dt = {'month' : int(sample['data_time'][5:7]),
69	'day' : int(sample['data_time'][8:10]),
70	'year' : int(sample['data_time'][0:4]),
71	'hour' : int(sample['data_time'][11:13]),
72	'min' : int(sample['data_time'][14:16]),
73	'sec' : int(sample['data_time'][17:19]),
74	}
75	package_dt = ('%(month)02d-%(day)02d-%(year)04d ' +
76	'%(hour)02d:%(min)02d:%(sec)02d') \
77	% package_dt
78	package_dt = procutil.scanf_datetime(package_dt, fmt='%m-%d-%Y %H:%M:%S')
79	if sensor_info['utc_offset']:
80	package_dt = package_dt + \
81	datetime.timedelta(hours=sensor_info['utc_offset'])
82	data['pdt'][sample_index] = package_dt
83	data['ptime'][sample_index] = procutil.dt2es(package_dt)
84
85	# platform session time
86	ds = {'month' : int(sample['sessionid'][14:16]),
87	'day' : int(sample['sessionid'][17:19]),
88	'year' : int(sample['sessionid'][9:13]),
89	'hour' : int(sample['sessionid'][20:22]),
90	'min' : int(sample['sessionid'][23:25]),
91	'sec' : int(sample['sessionid'][26:28]),
92	}
93	ds = '%(month)02d-%(day)02d-%(year)04d %(hour)02d:%(min)02d:%(sec)02d' \
94	% ds
95	ds = procutil.scanf_datetime(ds, fmt='%m-%d-%Y %H:%M:%S')
96	if sensor_info['utc_offset']:
97	ds = ds + datetime.timedelta(hours=sensor_info['utc_offset'])
98	data['ds'][sample_index] = ds
99	data['session'][sample_index] = procutil.dt2es(ds)
100
101	# package session time
102	package_ds = {'month' : int(sample['data_sessionid'][5:7]),
103	'day' : int(sample['data_sessionid'][8:10]),
104	'year' : int(sample['data_sessionid'][0:4]),
105	'hour' : int(sample['data_sessionid'][11:13]),
106	'min' : int(sample['data_sessionid'][14:16]),
107	'sec' : int(sample['data_sessionid'][17:19]),
108	}
109	package_ds = ('%(month)02d-%(day)02d-%(year)04d ' +
110	'%(hour)02d:%(min)02d:%(sec)02d') \
111	% package_ds
112	package_ds = procutil.scanf_datetime(package_ds, fmt='%m-%d-%Y %H:%M:%S')
113	if sensor_info['utc_offset']:
114	package_ds = package_ds + \
115	datetime.timedelta(hours=sensor_info['utc_offset'])
116	data['pds'][sample_index] = package_ds
117	data['psession'][sample_index] = procutil.dt2es(package_ds)
118
119	# platform variables
120	try:
121	data['record'][sample_index] = int(sample["recordnumber"])
122	except KeyError:
123	pass
124
125	try:
126	data['status'][sample_index] = int(sample["status"].
127	partition(":")[0])
128	except (KeyError, AttributeError, ):
129	pass
130
131	# package variables
132	try:
133	data['pstatus'][sample_index] = int(sample.sensors
134	[sensor_info["id_number"]]
135	["status"].
136	partition(":")[0])
137	except (KeyError, AttributeError, ):
138	pass
139
140	try:
141	data['abs_speed'][sample_index] = float(sample.sensors
142	[sensor_info["id_number"]].
143	points[sensor_info
144	["abs_speed_description"]]
145	["value"])
146	except (KeyError, AttributeError, ):
147	pass
148
149	try:
150	data['direction'][sample_index] = float(sample.sensors
151	[sensor_info["id_number"]].
152	points[sensor_info
153	["direction_description"]]
154	["value"])
155	except (KeyError, AttributeError, ):
156	pass
157
158	try:
159	data['v'][sample_index] = float(sample.sensors
160	[sensor_info["id_number"]].
161	points[sensor_info
162	["v_description"]]
163	["value"])
164	except (KeyError, AttributeError, ):
165	pass
166
167	try:
168	data['u'][sample_index] = float(sample.sensors
169	[sensor_info["id_number"]].
170	points[sensor_info
171	["u_description"]]
172	["value"])
173	except (KeyError, AttributeError, ):
174	pass
175
176	try:
177	data['heading'][sample_index] = float(sample.sensors
178	[sensor_info["id_number"]].
179	points[sensor_info
180	["heading_description"]]
181	["value"])
182	except (KeyError, AttributeError, ):
183	pass
184
185	try:
186	data['tiltx'][sample_index] = float(sample.sensors
187	[sensor_info["id_number"]].
188	points[sensor_info
189	["tiltx_description"]]
190	["value"])
191	except (KeyError, AttributeError, ):
192	pass
193
194	try:
195	data['tilty'][sample_index] = float(sample.sensors
196	[sensor_info["id_number"]].
197	points[sensor_info
198	["tilty_description"]]
199	["value"])
200	except (KeyError, AttributeError, ):
201	pass
202
203	try:
204	data['std_speed'][sample_index] = float(sample.sensors
205	[sensor_info["id_number"]].
206	points[sensor_info
207	["std_speed_description"]]
208	["value"])
209	except (KeyError, AttributeError, ):
210	pass
211
212	try:
213	data['strength'][sample_index] = float(sample.sensors
214	[sensor_info["id_number"]].
215	points[sensor_info
216	["strength_description"]]
217	["value"])
218	except (KeyError, AttributeError, ):
219	pass
220
221	try:
222	data['pings'][sample_index] = int(sample.sensors
223	[sensor_info["id_number"]].
224	points[sensor_info
225	["pings_description"]]
226	["value"])
227	except (KeyError, AttributeError, ):
228	pass
229
230	return data
231
232	def creator(platform_info, sensor_info, data):
233	#
234	#
235	title_str = sensor_info['description']+' at '+ sensor_info['location']
236	global_atts = {
237	# Required
238	'title' : title_str,
239	'institution' : platform_info['institution'],
240	'institution_url' : platform_info['institution_url'],
241	'institution_dods_url' : platform_info['institution_dods_url'],
242	'contact' : platform_info['contact'],
243	'Conventions' : platform_info['conventions'],
244	# Required by Scout
245	'format_category_code' : platform_info['format_category_code'],
246	'institution_code' : platform_info['institution_code'],
247	'platform_code' : platform_info['id'],
248	'package_code' : sensor_info['id'],
249	# Required by Version tracking
250	'format' : platform_info['format'],
251	'seacoos_rt_version' : platform_info['seacoos_rt_version'],
252	# Recommended
253	'_FillValue' : n.nan,
254	'missing_value' : n.nan,
255	'source' : platform_info['source'],
256	'references' : platform_info['references'],
257	'metadata_url' : platform_info['metadata_url'],
258	'history' : 'raw2proc using ' + sensor_info['process_module'],
259	'comment' : 'File created using pycdf ' + \
260	pycdf.pycdfVersion() + ' and ' + \
261	pycdf.pycdfArrayPkg() + ' ' + \
262	n.__version__,
263	'project' : platform_info['project'],
264	'project_url' : platform_info['project_url'],
265	# timeframe of data contained in file yyyy-mm-dd HH:MM:SS
266	# first date in monthly file
267	'start_date' : data['dt'][0].strftime("%Y-%m-%d %H:%M:%S"),
268	# last date in monthly file
269	'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"),
270	'release_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"),
271	'creation_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"),
272	'modification_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"),
273	'process_level' : 'level1',
274	# Custom
275	'id_number' : platform_info['id_number'],
276	'description' : platform_info['description'],
277	'serial_number' : platform_info['serial_number'],
278	'product_number' : platform_info['product_number'],
279	'product_name' : platform_info['product_name'],
280	'type' : platform_info['type'],
281	'protocol_version' : platform_info['protocol_version'],
282	'xmlns' : platform_info['xmlns'],
283	'location' : platform_info['location'],
284	'vertical_position': platform_info['vertical_position'],
285	'owner' : platform_info['owner'],
286	'package_id_number' : sensor_info['id_number'],
287	'package_description' : sensor_info['description'],
288	'package_serial_number' : sensor_info['serial_number'],
289	'package_product_number' : sensor_info['product_number'],
290	'package_product_name' : sensor_info['product_name'],
291	'package_adr' : sensor_info['adr'],
292	'package_protocol_version' : sensor_info['protocol_version'],
293	'package_vertical_position' : sensor_info['vertical_position'],
294	}
295
296	var_atts = {
297	# coordinate variables
298	'time' : {'short_name': 'time',
299	'long_name': 'Time',
300	'standard_name': 'time',
301	'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC
302	'axis': 'T',
303	},
304	'lat' : {'short_name': 'lat',
305	'long_name': 'Latitude',
306	'standard_name': 'latitude',
307	'reference':'geographic coordinates',
308	'units': 'degrees_north',
309	'valid_range':(-90.,90.),
310	'axis': 'Y',
311	},
312	'lon' : {'short_name': 'lon',
313	'long_name': 'Longtitude',
314	'standard_name': 'longtitude',
315	'reference':'geographic coordinates',
316	'units': 'degrees_east',
317	'valid_range':(-180.,180.),
318	'axis': 'Y',
319	},
320	'z' : {'short_name': 'z',
321	'long_name': 'Height',
322	'standard_name': 'height',
323	'reference':'zero at sea-surface',
324	'positive' : 'up',
325	'units': 'meters',
326	'axis': 'Z',
327	},
328	# data variables
329	'ptime' : {'short_name': 'ptime',
330	'long_name': 'Package Time',
331	'standard_name': 'none',
332	'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC
333	},
334	'session' : {'short_name': 'session',
335	'long_name': 'Session ID',
336	'standard_name': 'none',
337	'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC
338	},
339	'psession' : {'short_name': 'ptime',
340	'long_name': 'Package Session ID',
341	'standard_name': 'none',
342	'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC
343	},
344	'record' : {'short_name': 'record',
345	'long_name': 'Record Number',
346	'standard_name': 'none',
347	'units' : 'none',
348	},
349	'status': {'short_name' : 'status',
350	'long_name': 'Platform Status Code',
351	'standard_name': 'none',
352	'units' : 'none',
353	'value_map' : platform_info['status_map'],
354	},
355	'pstatus': {'short_name' : 'pstatus',
356	'long_name': 'Package Status Code',
357	'standard_name': 'none',
358	'units' : 'none',
359	'value_map' : sensor_info['status_map'],
360	},
361	'abs_speed' : {'short_name' : 'c_spd',
362	'long_name': sensor_info['abs_speed_description'],
363	'standard_name': 'current_speed',
364	'units': 'cm s-1',
365	'id_number' : sensor_info['abs_speed_id'],
366	'type' : sensor_info['abs_speed_type'],
367	'format' : sensor_info['abs_speed_format'],
368	'non_standard_units' : sensor_info['abs_speed_units'],
369	'range_min' : sensor_info['abs_speed_range_min'],
370	'range_max' : sensor_info['abs_speed_range_max'],
371	},
372	'direction' : {'short_name' : 'c_dir',
373	'long_name': sensor_info['direction_description'],
374	'standard_name': 'current_to_direction',
375	'units': 'degrees_true',
376	'id_number' : sensor_info['direction_id'],
377	'type' : sensor_info['direction_type'],
378	'format' : sensor_info['direction_format'],
379	'non_standard_units' : sensor_info['direction_units'],
380	'range_min' : sensor_info['direction_range_min'],
381	'range_max' : sensor_info['direction_range_max'],
382	},
383	'v' : {'short_name' : 'water_v',
384	'long_name': sensor_info['v_description'],
385	'standard_name': 'northward_current',
386	'units': 'cm s-1',
387	'id_number' : sensor_info['v_id'],
388	'type' : sensor_info['v_type'],
389	'format' : sensor_info['v_format'],
390	'non_standard_units' : sensor_info['v_units'],
391	'range_min' : sensor_info['v_range_min'],
392	'range_max' : sensor_info['v_range_max'],
393	},
394	'u' : {'short_name' : 'water_u',
395	'long_name': sensor_info['u_description'],
396	'standard_name': 'eastward_current',
397	'units': 'cm s-1',
398	'id_number' : sensor_info['u_id'],
399	'type' : sensor_info['u_type'],
400	'format' : sensor_info['u_format'],
401	'non_standard_units' : sensor_info['u_units'],
402	'range_min' : sensor_info['u_range_min'],
403	'range_max' : sensor_info['u_range_max'],
404	},
405	'heading' : {'short_name' : 'heading',
406	'long_name': sensor_info['heading_description'],
407	'standard_name': 'none',
408	'units': 'none',
409	'id_number' : sensor_info['heading_id'],
410	'type' : sensor_info['heading_type'],
411	'format' : sensor_info['heading_format'],
412	'non_standard_units' : sensor_info['heading_units'],
413	'range_min' : sensor_info['heading_range_min'],
414	'range_max' : sensor_info['heading_range_max'],
415	},
416	'tiltx' : {'short_name' : 'tiltx',
417	'long_name': sensor_info['tiltx_description'],
418	'standard_name': 'none',
419	'units': 'none',
420	'id_number' : sensor_info['tiltx_id'],
421	'type' : sensor_info['tiltx_type'],
422	'format' : sensor_info['tiltx_format'],
423	'non_standard_units' : sensor_info['tiltx_units'],
424	'range_min' : sensor_info['tiltx_range_min'],
425	'range_max' : sensor_info['tiltx_range_max'],
426	},
427	'tilty' : {'short_name' : 'tilty',
428	'long_name': sensor_info['tilty_description'],
429	'standard_name': 'none',
430	'units': 'none',
431	'id_number' : sensor_info['tilty_id'],
432	'type' : sensor_info['tilty_type'],
433	'format' : sensor_info['tilty_format'],
434	'non_standard_units' : sensor_info['tilty_units'],
435	'range_min' : sensor_info['tilty_range_min'],
436	'range_max' : sensor_info['tilty_range_max'],
437	},
438	'std_speed' : {'short_name' : 'beam_vel',
439	'long_name': sensor_info['std_speed_description'],
440	'standard_name': 'beam_velocity',
441	'units': 'cm s-1',
442	'id_number' : sensor_info['std_speed_id'],
443	'type' : sensor_info['std_speed_type'],
444	'format' : sensor_info['std_speed_format'],
445	'non_standard_units' : sensor_info['std_speed_units'],
446	'range_min' : sensor_info['std_speed_range_min'],
447	'range_max' : sensor_info['std_speed_range_max'],
448	},
449	'strength' : {'short_name' : 'beam_echo',
450	'long_name': sensor_info['strength_description'],
451	'standard_name': 'beam_echo_intensity',
452	'units': 'db',
453	'id_number' : sensor_info['strength_id'],
454	'type' : sensor_info['strength_type'],
455	'format' : sensor_info['strength_format'],
456	'non_standard_units' : sensor_info['strength_units'],
457	'range_min' : sensor_info['strength_range_min'],
458	'range_max' : sensor_info['strength_range_max'],
459	},
460	'pings' : {'short_name' : 'pings',
461	'long_name': sensor_info['pings_description'],
462	'standard_name': 'none',
463	'units': 'none',
464	'id_number' : sensor_info['pings_id'],
465	'type' : sensor_info['pings_type'],
466	'format' : sensor_info['pings_format'],
467	'non_standard_units' : sensor_info['pings_units'],
468	'range_min' : sensor_info['pings_range_min'],
469	'range_max' : sensor_info['pings_range_max'],
470	},
471	}
472
473	# dimension names use tuple so order of initialization is maintained
474	dim_inits = (
475	('ntime', pycdf.NC.UNLIMITED),
476	('nlat', 1),
477	('nlon', 1),
478	('nz', 1),
479	)
480
481	# using tuple of tuples so order of initialization is maintained
482	# using dict for attributes order of init not important
483	# use dimension names not values
484	# (varName, varType, (dimName1, [dimName2], ...))
485	var_inits = (
486	# coordinate variables
487	('time', pycdf.NC.INT, ('ntime',)),
488	('lat', pycdf.NC.FLOAT, ('nlat',)),
489	('lon', pycdf.NC.FLOAT, ('nlon',)),
490	('z', pycdf.NC.FLOAT, ('nz',)),
491	# data variables
492	('ptime', pycdf.NC.INT, ('ntime',)),
493	('session', pycdf.NC.INT, ('ntime',)),
494	('psession', pycdf.NC.INT, ('ntime',)),
495	('record', pycdf.NC.INT, ('ntime',)),
496	('status', pycdf.NC.INT, ('ntime',)),
497	('pstatus', pycdf.NC.INT, ('ntime',)),
498	('abs_speed', pycdf.NC.FLOAT, ('ntime',)),
499	('direction', pycdf.NC.FLOAT, ('ntime',)),
500	('v', pycdf.NC.FLOAT, ('ntime',)),
501	('u', pycdf.NC.FLOAT, ('ntime',)),
502	('heading', pycdf.NC.FLOAT, ('ntime',)),
503	('tiltx', pycdf.NC.FLOAT, ('ntime',)),
504	('tilty', pycdf.NC.FLOAT, ('ntime',)),
505	('std_speed', pycdf.NC.FLOAT, ('ntime',)),
506	('strength', pycdf.NC.FLOAT, ('ntime',)),
507	('pings', pycdf.NC.INT, ('ntime',)),
508	)
509
510	# subset data only to month being processed (see raw2proc.process())
511	i = data['in']
512
513	# var data
514	var_data = (
515	('time', data['time'][i]),
516	('lat', sensor_info['lat']),
517	('lon', sensor_info['lat']),
518	('z', sensor_info['elevation']),
519	('ptime', data['ptime'][i]),
520	('session', data['session'][i]),
521	('psession', data['psession'][i]),
522	('record', data['record'][i]),
523	('status', data['status'][i]),
524	('pstatus', data['pstatus'][i]),
525	('abs_speed', data['abs_speed'][i]),
526	('direction', data['direction'][i]),
527	('v', data['v'][i]),
528	('u', data['u'][i]),
529	('heading', data['heading'][i]),
530	('tiltx', data['tiltx'][i]),
531	('tilty', data['tilty'][i]),
532	('std_speed', data['std_speed'][i]),
533	('strength', data['strength'][i]),
534	('pings', data['pings'][i]),
535	)
536
537	return (global_atts, var_atts, dim_inits, var_inits, var_data)
538
539	def updater(platform_info, sensor_info, data):
540	#
541	global_atts = {
542	# update times of data contained in file (yyyy-mm-dd HH:MM:SS)
543	# last date in monthly file
544	'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"),
545	'release_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"),
546	#
547	'modification_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"),
548	}
549
550	# data variables
551	# update any variable attributes like range, min, max
552	var_atts = {}
553
554	# subset data only to month being processed (see raw2proc.process())
555	i = data['in']
556
557	# data
558	var_data = (
559	('time', data['time'][i]),
560	('ptime', data['ptime'][i]),
561	('session', data['session'][i]),
562	('psession', data['psession'][i]),
563	('record', data['record'][i]),
564	('status', data['status'][i]),
565	('pstatus', data['pstatus'][i]),
566	('abs_speed', data['abs_speed'][i]),
567	('direction', data['direction'][i]),
568	('v', data['v'][i]),
569	('u', data['u'][i]),
570	('heading', data['heading'][i]),
571	('tiltx', data['tiltx'][i]),
572	('tilty', data['tilty'][i]),
573	('std_speed', data['std_speed'][i]),
574	('strength', data['strength'][i]),
575	('pings', data['pings'][i]),
576	)
577
578	return (global_atts, var_atts, var_data)
579

Note: See TracBrowser for help on using the browser.

root/raw2proc/trunk/raw2proc/proc_mini_andi_current.py

Download in other formats: