Revision Log

proc_mini_andi_optode.py

Revision 375 (checked in by cbc, 14 years ago)	Process and config modules for Spongenet.

Line
1	"""
2	Parse data and assert what data creates and updates monthly NetCDF files.
3
4	Spongenet mini_andi optode parameters sponge data.
5	"""
6
7	import math
8	import numpy as n
9	import pycdf
10	import datetime
11	import procutil
12	from spongenet.parse import Data
13
14	nowDt = datetime.datetime.utcnow().replace(microsecond=0)
15
16	def parser(platform_info, sensor_info, lines):
17	"""
18	Parse and assign sponge data from XML file.
19	"""
20
21	_data = Data(''.join(lines))
22
23	# Each Device tag represents a time sample.
24	num_samples = len(_data.devices)
25
26	data = {
27	'dt' : n.array(n.ones((num_samples,)) * n.nan, dtype=object),
28	'time' : n.array(n.ones((num_samples,)) * n.nan, dtype=long),
29	'pdt' : n.array(n.ones((num_samples,)) * n.nan, dtype=object),
30	'ptime' : n.array(n.ones((num_samples,)) * n.nan, dtype=long),
31	'ds' : n.array(n.ones((num_samples,)) * n.nan, dtype=object),
32	'session' : n.array(n.ones((num_samples,)) * n.nan, dtype=long),
33	'pds' : n.array(n.ones((num_samples,)) * n.nan, dtype=object),
34	'psession' : n.array(n.ones((num_samples,)) * n.nan, dtype=long),
35	'record' : n.array(n.ones((num_samples,)) * n.nan, dtype=int),
36	'status' : n.array(n.ones((num_samples,)) * n.nan, dtype=int),
37	'pstatus' : n.array(n.ones((num_samples,)) * n.nan, dtype=int),
38	'o2concentration' : n.array(n.ones((num_samples,)) * n.nan, dtype=float),
39	'airsaturation' : n.array(n.ones((num_samples,)) * n.nan, dtype=float),
40	'temperature' : n.array(n.ones((num_samples,)) * n.nan, dtype=float),
41	'calphase' : n.array(n.ones((num_samples,)) * n.nan, dtype=float),
42	'tcphase' : n.array(n.ones((num_samples,)) * n.nan, dtype=float),
43	'c1rph' : n.array(n.ones((num_samples,)) * n.nan, dtype=float),
44	'c2rph' : n.array(n.ones((num_samples,)) * n.nan, dtype=float),
45	'c1amp' : n.array(n.ones((num_samples,)) * n.nan, dtype=float),
46	'c2amp' : n.array(n.ones((num_samples,)) * n.nan, dtype=float),
47	'rawtemp' : n.array(n.ones((num_samples,)) * n.nan, dtype=float),
48	}
49
50	for (sample_index, sample) in enumerate(_data.devices):
51	# sample time at the platform
52	dt = {'month' : int(sample['time'][5:7]),
53	'day' : int(sample['time'][8:10]),
54	'year' : int(sample['time'][0:4]),
55	'hour' : int(sample['time'][11:13]),
56	'min' : int(sample['time'][14:16]),
57	'sec' : int(sample['time'][17:19]),
58	}
59	dt = '%(month)02d-%(day)02d-%(year)04d %(hour)02d:%(min)02d:%(sec)02d' \
60	% dt
61	dt = procutil.scanf_datetime(dt, fmt='%m-%d-%Y %H:%M:%S')
62	if sensor_info['utc_offset']:
63	dt = dt + datetime.timedelta(hours=sensor_info['utc_offset'])
64	data['dt'][sample_index] = dt
65	data['time'][sample_index] = procutil.dt2es(dt)
66
67	# sample time at the package
68	package_dt = {'month' : int(sample['data_time'][5:7]),
69	'day' : int(sample['data_time'][8:10]),
70	'year' : int(sample['data_time'][0:4]),
71	'hour' : int(sample['data_time'][11:13]),
72	'min' : int(sample['data_time'][14:16]),
73	'sec' : int(sample['data_time'][17:19]),
74	}
75	package_dt = ('%(month)02d-%(day)02d-%(year)04d ' +
76	'%(hour)02d:%(min)02d:%(sec)02d') \
77	% package_dt
78	package_dt = procutil.scanf_datetime(package_dt, fmt='%m-%d-%Y %H:%M:%S')
79	if sensor_info['utc_offset']:
80	package_dt = package_dt + \
81	datetime.timedelta(hours=sensor_info['utc_offset'])
82	data['pdt'][sample_index] = package_dt
83	data['ptime'][sample_index] = procutil.dt2es(package_dt)
84
85	# platform session time
86	ds = {'month' : int(sample['sessionid'][14:16]),
87	'day' : int(sample['sessionid'][17:19]),
88	'year' : int(sample['sessionid'][9:13]),
89	'hour' : int(sample['sessionid'][20:22]),
90	'min' : int(sample['sessionid'][23:25]),
91	'sec' : int(sample['sessionid'][26:28]),
92	}
93	ds = '%(month)02d-%(day)02d-%(year)04d %(hour)02d:%(min)02d:%(sec)02d' \
94	% ds
95	ds = procutil.scanf_datetime(ds, fmt='%m-%d-%Y %H:%M:%S')
96	if sensor_info['utc_offset']:
97	ds = ds + datetime.timedelta(hours=sensor_info['utc_offset'])
98	data['ds'][sample_index] = ds
99	data['session'][sample_index] = procutil.dt2es(ds)
100
101	# package session time
102	package_ds = {'month' : int(sample['data_sessionid'][5:7]),
103	'day' : int(sample['data_sessionid'][8:10]),
104	'year' : int(sample['data_sessionid'][0:4]),
105	'hour' : int(sample['data_sessionid'][11:13]),
106	'min' : int(sample['data_sessionid'][14:16]),
107	'sec' : int(sample['data_sessionid'][17:19]),
108	}
109	package_ds = ('%(month)02d-%(day)02d-%(year)04d ' +
110	'%(hour)02d:%(min)02d:%(sec)02d') \
111	% package_ds
112	package_ds = procutil.scanf_datetime(package_ds, fmt='%m-%d-%Y %H:%M:%S')
113	if sensor_info['utc_offset']:
114	package_ds = package_ds + \
115	datetime.timedelta(hours=sensor_info['utc_offset'])
116	data['pds'][sample_index] = package_ds
117	data['psession'][sample_index] = procutil.dt2es(package_ds)
118
119	# platform variables
120	try:
121	data['record'][sample_index] = int(sample["recordnumber"])
122	except KeyError:
123	pass
124
125	try:
126	data['status'][sample_index] = int(sample["status"].
127	partition(":")[0])
128	except (KeyError, AttributeError, ):
129	pass
130
131	# package variables
132	try:
133	data['pstatus'][sample_index] = int(sample.sensors
134	[sensor_info["id_number"]]
135	["status"].
136	partition(":")[0])
137	except (KeyError, AttributeError, ):
138	pass
139
140	try:
141	data['o2concentration'][sample_index] = float(sample.sensors
142	[sensor_info["id_number"]].
143	points[sensor_info
144	["o2concentration_description"]]
145	["value"])
146	except (KeyError, AttributeError, ):
147	pass
148
149	try:
150	data['airsaturation'][sample_index] = float(sample.sensors
151	[sensor_info["id_number"]].
152	points[sensor_info
153	["airsaturation_description"]]
154	["value"])
155	except (KeyError, AttributeError, ):
156	pass
157
158	try:
159	data['temperature'][sample_index] = float(sample.sensors
160	[sensor_info["id_number"]].
161	points[sensor_info
162	["temperature_description"]]
163	["value"])
164	except (KeyError, AttributeError, ):
165	pass
166
167	try:
168	data['calphase'][sample_index] = float(sample.sensors
169	[sensor_info["id_number"]].
170	points[sensor_info
171	["calphase_description"]]
172	["value"])
173	except (KeyError, AttributeError, ):
174	pass
175
176	try:
177	data['tcphase'][sample_index] = float(sample.sensors
178	[sensor_info["id_number"]].
179	points[sensor_info
180	["tcphase_description"]]
181	["value"])
182	except (KeyError, AttributeError, ):
183	pass
184
185	try:
186	data['c1rph'][sample_index] = float(sample.sensors
187	[sensor_info["id_number"]].
188	points[sensor_info
189	["c1rph_description"]]
190	["value"])
191	except (KeyError, AttributeError, ):
192	pass
193
194	try:
195	data['c2rph'][sample_index] = float(sample.sensors
196	[sensor_info["id_number"]].
197	points[sensor_info
198	["c2rph_description"]]
199	["value"])
200	except (KeyError, AttributeError, ):
201	pass
202
203	try:
204	data['c1amp'][sample_index] = float(sample.sensors
205	[sensor_info["id_number"]].
206	points[sensor_info
207	["c1amp_description"]]
208	["value"])
209	except (KeyError, AttributeError, ):
210	pass
211
212	try:
213	data['c2amp'][sample_index] = float(sample.sensors
214	[sensor_info["id_number"]].
215	points[sensor_info
216	["c2amp_description"]]
217	["value"])
218	except (KeyError, AttributeError, ):
219	pass
220
221	try:
222	data['rawtemp'][sample_index] = float(sample.sensors
223	[sensor_info["id_number"]].
224	points[sensor_info
225	["rawtemp_description"]]
226	["value"])
227	except (KeyError, AttributeError, ):
228	pass
229
230	return data
231
232	def creator(platform_info, sensor_info, data):
233	#
234	#
235	title_str = sensor_info['description']+' at '+ sensor_info['location']
236	global_atts = {
237	# Required
238	'title' : title_str,
239	'institution' : platform_info['institution'],
240	'institution_url' : platform_info['institution_url'],
241	'institution_dods_url' : platform_info['institution_dods_url'],
242	'contact' : platform_info['contact'],
243	'Conventions' : platform_info['conventions'],
244	# Required by Scout
245	'format_category_code' : platform_info['format_category_code'],
246	'institution_code' : platform_info['institution_code'],
247	'platform_code' : platform_info['id'],
248	'package_code' : sensor_info['id'],
249	# Required by Version tracking
250	'format' : platform_info['format'],
251	'seacoos_rt_version' : platform_info['seacoos_rt_version'],
252	# Recommended
253	'_FillValue' : n.nan,
254	'missing_value' : n.nan,
255	'source' : platform_info['source'],
256	'references' : platform_info['references'],
257	'metadata_url' : platform_info['metadata_url'],
258	'history' : 'raw2proc using ' + sensor_info['process_module'],
259	'comment' : 'File created using pycdf ' + \
260	pycdf.pycdfVersion() + ' and ' + \
261	pycdf.pycdfArrayPkg() + ' ' + \
262	n.__version__,
263	'project' : platform_info['project'],
264	'project_url' : platform_info['project_url'],
265	# timeframe of data contained in file yyyy-mm-dd HH:MM:SS
266	# first date in monthly file
267	'start_date' : data['dt'][0].strftime("%Y-%m-%d %H:%M:%S"),
268	# last date in monthly file
269	'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"),
270	'release_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"),
271	'creation_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"),
272	'modification_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"),
273	'process_level' : 'level1',
274	# Custom
275	'id_number' : platform_info['id_number'],
276	'description' : platform_info['description'],
277	'serial_number' : platform_info['serial_number'],
278	'product_number' : platform_info['product_number'],
279	'product_name' : platform_info['product_name'],
280	'type' : platform_info['type'],
281	'protocol_version' : platform_info['protocol_version'],
282	'xmlns' : platform_info['xmlns'],
283	'location' : platform_info['location'],
284	'vertical_position': platform_info['vertical_position'],
285	'owner' : platform_info['owner'],
286	'package_id_number' : sensor_info['id_number'],
287	'package_description' : sensor_info['description'],
288	'package_serial_number' : sensor_info['serial_number'],
289	'package_product_number' : sensor_info['product_number'],
290	'package_product_name' : sensor_info['product_name'],
291	'package_adr' : sensor_info['adr'],
292	'package_protocol_version' : sensor_info['protocol_version'],
293	'package_vertical_position' : sensor_info['vertical_position'],
294	}
295
296	var_atts = {
297	# coordinate variables
298	'time' : {'short_name': 'time',
299	'long_name': 'Time',
300	'standard_name': 'time',
301	'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC
302	'axis': 'T',
303	},
304	'lat' : {'short_name': 'lat',
305	'long_name': 'Latitude',
306	'standard_name': 'latitude',
307	'reference':'geographic coordinates',
308	'units': 'degrees_north',
309	'valid_range':(-90.,90.),
310	'axis': 'Y',
311	},
312	'lon' : {'short_name': 'lon',
313	'long_name': 'Longtitude',
314	'standard_name': 'longtitude',
315	'reference':'geographic coordinates',
316	'units': 'degrees_east',
317	'valid_range':(-180.,180.),
318	'axis': 'Y',
319	},
320	'z' : {'short_name': 'z',
321	'long_name': 'Height',
322	'standard_name': 'height',
323	'reference':'zero at sea-surface',
324	'positive' : 'up',
325	'units': 'meters',
326	'axis': 'Z',
327	},
328	# data variables
329	'ptime' : {'short_name': 'ptime',
330	'long_name': 'Package Time',
331	'standard_name': 'none',
332	'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC
333	},
334	'session' : {'short_name': 'session',
335	'long_name': 'Session ID',
336	'standard_name': 'none',
337	'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC
338	},
339	'psession' : {'short_name': 'ptime',
340	'long_name': 'Package Session ID',
341	'standard_name': 'none',
342	'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC
343	},
344	'record' : {'short_name': 'record',
345	'long_name': 'Record Number',
346	'standard_name': 'none',
347	'units' : 'none',
348	},
349	'status': {'short_name' : 'status',
350	'long_name': 'Platform Status Code',
351	'standard_name': 'none',
352	'units' : 'none',
353	'value_map' : platform_info['status_map'],
354	},
355	'pstatus': {'short_name' : 'pstatus',
356	'long_name': 'Package Status Code',
357	'standard_name': 'none',
358	'units' : 'none',
359	'value_map' : sensor_info['status_map'],
360	},
361	'o2concentration' : {'short_name' : 'o2concentration',
362	'long_name': sensor_info['o2concentration_description'],
363	'standard_name': 'oxygen_concentration',
364	'units': 'none',
365	'id_number' : sensor_info['o2concentration_id'],
366	'type' : sensor_info['o2concentration_type'],
367	'format' : sensor_info['o2concentration_format'],
368	'non_standard_units' : sensor_info['o2concentration_units'],
369	'range_min' : sensor_info['o2concentration_range_min'],
370	'range_max' : sensor_info['o2concentration_range_max'],
371	},
372	'airsaturation' : {'short_name' : 'airsaturation',
373	'long_name': sensor_info['airsaturation_description'],
374	'standard_name': 'oxygen_saturation',
375	'units': 'none',
376	'id_number' : sensor_info['airsaturation_id'],
377	'type' : sensor_info['airsaturation_type'],
378	'format' : sensor_info['airsaturation_format'],
379	'non_standard_units' : sensor_info['airsaturation_units'],
380	'range_min' : sensor_info['airsaturation_range_min'],
381	'range_max' : sensor_info['airsaturation_range_max'],
382	},
383	'temperature' : {'short_name' : 'temperature',
384	'long_name': sensor_info['temperature_description'],
385	'standard_name': 'water_temperature',
386	'units': 'celsius',
387	'id_number' : sensor_info['temperature_id'],
388	'type' : sensor_info['temperature_type'],
389	'format' : sensor_info['temperature_format'],
390	'non_standard_units' : sensor_info['temperature_units'],
391	'range_min' : sensor_info['temperature_range_min'],
392	'range_max' : sensor_info['temperature_range_max'],
393	},
394	'calphase' : {'short_name' : 'calphase',
395	'long_name': sensor_info['calphase_description'],
396	'standard_name': 'none',
397	'units': 'none',
398	'id_number' : sensor_info['calphase_id'],
399	'type' : sensor_info['calphase_type'],
400	'format' : sensor_info['calphase_format'],
401	'non_standard_units' : sensor_info['calphase_units'],
402	'range_min' : sensor_info['calphase_range_min'],
403	'range_max' : sensor_info['calphase_range_max'],
404	},
405	'tcphase' : {'short_name' : 'tcphase',
406	'long_name': sensor_info['tcphase_description'],
407	'standard_name': 'none',
408	'units': 'none',
409	'id_number' : sensor_info['tcphase_id'],
410	'type' : sensor_info['tcphase_type'],
411	'format' : sensor_info['tcphase_format'],
412	'non_standard_units' : sensor_info['tcphase_units'],
413	'range_min' : sensor_info['tcphase_range_min'],
414	'range_max' : sensor_info['tcphase_range_max'],
415	},
416	'c1rph' : {'short_name' : 'c1rph',
417	'long_name': sensor_info['c1rph_description'],
418	'standard_name': 'none',
419	'units': 'none',
420	'id_number' : sensor_info['c1rph_id'],
421	'type' : sensor_info['c1rph_type'],
422	'format' : sensor_info['c1rph_format'],
423	'non_standard_units' : sensor_info['c1rph_units'],
424	'range_min' : sensor_info['c1rph_range_min'],
425	'range_max' : sensor_info['c1rph_range_max'],
426	},
427	'c2rph' : {'short_name' : 'c2rph',
428	'long_name': sensor_info['c2rph_description'],
429	'standard_name': 'none',
430	'units': 'none',
431	'id_number' : sensor_info['c2rph_id'],
432	'type' : sensor_info['c2rph_type'],
433	'format' : sensor_info['c2rph_format'],
434	'non_standard_units' : sensor_info['c2rph_units'],
435	'range_min' : sensor_info['c2rph_range_min'],
436	'range_max' : sensor_info['c2rph_range_max'],
437	},
438	'c1amp' : {'short_name' : 'c1amp',
439	'long_name': sensor_info['c1amp_description'],
440	'standard_name': 'none',
441	'units': 'none',
442	'id_number' : sensor_info['c1amp_id'],
443	'type' : sensor_info['c1amp_type'],
444	'format' : sensor_info['c1amp_format'],
445	'non_standard_units' : sensor_info['c1amp_units'],
446	'range_min' : sensor_info['c1amp_range_min'],
447	'range_max' : sensor_info['c1amp_range_max'],
448	},
449	'c2amp' : {'short_name' : 'c2amp',
450	'long_name': sensor_info['c2amp_description'],
451	'standard_name': 'none',
452	'units': 'none',
453	'id_number' : sensor_info['c2amp_id'],
454	'type' : sensor_info['c2amp_type'],
455	'format' : sensor_info['c2amp_format'],
456	'non_standard_units' : sensor_info['c2amp_units'],
457	'range_min' : sensor_info['c2amp_range_min'],
458	'range_max' : sensor_info['c2amp_range_max'],
459	},
460	'rawtemp' : {'short_name' : 'rawtemp',
461	'long_name': sensor_info['rawtemp_description'],
462	'standard_name': 'none',
463	'units': 'none',
464	'id_number' : sensor_info['rawtemp_id'],
465	'type' : sensor_info['rawtemp_type'],
466	'format' : sensor_info['rawtemp_format'],
467	'non_standard_units' : sensor_info['rawtemp_units'],
468	'range_min' : sensor_info['rawtemp_range_min'],
469	'range_max' : sensor_info['rawtemp_range_max'],
470	},
471	}
472
473	# dimension names use tuple so order of initialization is maintained
474	dim_inits = (
475	('ntime', pycdf.NC.UNLIMITED),
476	('nlat', 1),
477	('nlon', 1),
478	('nz', 1),
479	)
480
481	# using tuple of tuples so order of initialization is maintained
482	# using dict for attributes order of init not important
483	# use dimension names not values
484	# (varName, varType, (dimName1, [dimName2], ...))
485	var_inits = (
486	# coordinate variables
487	('time', pycdf.NC.INT, ('ntime',)),
488	('lat', pycdf.NC.FLOAT, ('nlat',)),
489	('lon', pycdf.NC.FLOAT, ('nlon',)),
490	('z', pycdf.NC.FLOAT, ('nz',)),
491	# data variables
492	('ptime', pycdf.NC.INT, ('ntime',)),
493	('session', pycdf.NC.INT, ('ntime',)),
494	('psession', pycdf.NC.INT, ('ntime',)),
495	('record', pycdf.NC.INT, ('ntime',)),
496	('status', pycdf.NC.INT, ('ntime',)),
497	('pstatus', pycdf.NC.INT, ('ntime',)),
498	('o2concentration', pycdf.NC.FLOAT, ('ntime',)),
499	('airsaturation', pycdf.NC.FLOAT, ('ntime',)),
500	('temperature', pycdf.NC.FLOAT, ('ntime',)),
501	('calphase', pycdf.NC.FLOAT, ('ntime',)),
502	('tcphase', pycdf.NC.FLOAT, ('ntime',)),
503	('c1rph', pycdf.NC.FLOAT, ('ntime',)),
504	('c2rph', pycdf.NC.FLOAT, ('ntime',)),
505	('c1amp', pycdf.NC.FLOAT, ('ntime',)),
506	('c2amp', pycdf.NC.FLOAT, ('ntime',)),
507	('rawtemp', pycdf.NC.FLOAT, ('ntime',)),
508	)
509
510	# subset data only to month being processed (see raw2proc.process())
511	i = data['in']
512
513	# var data
514	var_data = (
515	('time', data['time'][i]),
516	('lat', sensor_info['lat']),
517	('lon', sensor_info['lat']),
518	('z', sensor_info['elevation']),
519	('ptime', data['ptime'][i]),
520	('session', data['session'][i]),
521	('psession', data['psession'][i]),
522	('record', data['record'][i]),
523	('status', data['status'][i]),
524	('pstatus', data['pstatus'][i]),
525	('o2concentration', data['o2concentration'][i]),
526	('airsaturation', data['airsaturation'][i]),
527	('temperature', data['temperature'][i]),
528	('calphase', data['calphase'][i]),
529	('tcphase', data['tcphase'][i]),
530	('c1rph', data['c1rph'][i]),
531	('c2rph', data['c2rph'][i]),
532	('c1amp', data['c1amp'][i]),
533	('c2amp', data['c2amp'][i]),
534	('rawtemp', data['rawtemp'][i]),
535	)
536
537	return (global_atts, var_atts, dim_inits, var_inits, var_data)
538
539	def updater(platform_info, sensor_info, data):
540	#
541	global_atts = {
542	# update times of data contained in file (yyyy-mm-dd HH:MM:SS)
543	# last date in monthly file
544	'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"),
545	'release_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"),
546	#
547	'modification_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"),
548	}
549
550	# data variables
551	# update any variable attributes like range, min, max
552	var_atts = {}
553
554	# subset data only to month being processed (see raw2proc.process())
555	i = data['in']
556
557	# data
558	var_data = (
559	('time', data['time'][i]),
560	('ptime', data['ptime'][i]),
561	('session', data['session'][i]),
562	('psession', data['psession'][i]),
563	('record', data['record'][i]),
564	('status', data['status'][i]),
565	('pstatus', data['pstatus'][i]),
566	('o2concentration', data['o2concentration'][i]),
567	('airsaturation', data['airsaturation'][i]),
568	('temperature', data['temperature'][i]),
569	('calphase', data['calphase'][i]),
570	('tcphase', data['tcphase'][i]),
571	('c1rph', data['c1rph'][i]),
572	('c2rph', data['c2rph'][i]),
573	('c1amp', data['c1amp'][i]),
574	('c2amp', data['c2amp'][i]),
575	('rawtemp', data['rawtemp'][i]),
576	)
577
578	return (global_atts, var_atts, var_data)
579

Note: See TracBrowser for help on using the browser.

root/raw2proc/trunk/raw2proc/proc_mini_andi_optode.py

Download in other formats: