Changeset Navigation

← Previous Changeset
Next Changeset →

Changeset 492

Timestamp:

06/14/12 10:33:14

Author:

haines

Message:

minor to raw2proc:which_raw()

Files:

raw2proc/trunk/raw2proc/b1_config_20111112.py (modified) (5 diffs)
raw2proc/trunk/raw2proc/b2_config_20111005.py (modified) (5 diffs)
raw2proc/trunk/raw2proc/b2_config_20111112.py (modified) (5 diffs)
raw2proc/trunk/raw2proc/crow_config_20090122.py (modified) (2 diffs)
raw2proc/trunk/raw2proc/lb1_config_20120120.py (added)
raw2proc/trunk/raw2proc/lb2_config_20120119.py (added)
raw2proc/trunk/raw2proc/lb3_config_20111216.py (added)
raw2proc/trunk/raw2proc/meet_config_20090122.py (modified) (2 diffs)
raw2proc/trunk/raw2proc/proc_cr1000_met.py (modified) (3 diffs)
raw2proc/trunk/raw2proc/procutil.py (modified) (1 diff)
raw2proc/trunk/raw2proc/raw2proc.py (modified) (4 diffs)

Legend:

: Unmodified
: Added
: Removed
: Modified
: Copied
: Moved

raw2proc/trunk/raw2proc/b1_config_20111112.py

r490	r492
13	13	'institution' : 'nccoos',
14	14	#
15		'config_start_date' : '2011-11-1~~3 00~~:00:00',
16		'config_end_date' : '2012-04-0~~9 00~~:00:00', # None or yyyy-mm-dd HH:MM:SS
	15	'config_start_date' : '2011-11-12 16:00:00',
	16	'config_end_date' : '2012-04-08 19:00:00', # None or yyyy-mm-dd HH:MM:SS
17	17	'packages' : ('met', 'wind', 'ctd1', 'ctd2'),
18	18	# Required by CF
…	…
45	45	'height_units' : 'm',
46	46	'height_reference' : 'sea_surface',
	47	'source': 'Heise Baro, Rotronics Temp/RH, RM Young Precip, Eppley PSP/PIR',
47	48	# 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0',
48	49	# 'latest_vars' : ('time','lat','lon','z','atemp', 'baro', 'rh', 'rain', 'psp', 'pir'),
49		~~# Recommended~~
50		'~~source': 'Heise Baro, Rotronics Temp/RH, RM Young Precip, Eppley PSP/PIR'~~,
	50	'plot_module': 'plot_cr1000_met',
	51	'plot_names': ('timeseries',),
51	52	},
52	53	'wind' : { 'id' : 'wind',
…	…
61	62	'height_units' : 'm',
62	63	'height_reference' : 'sea_surface',
	64	# Recommended
	65	'source': 'RM Young Marine Wind Monitor 5106',
63	66	# 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0',
64	67	# 'latest_vars' : ('time','lat','lon','z','u', 'v', 'wspd', 'wdir'),
65		~~# Recommended~~
66		'~~source': 'RM Young Marine Wind Monitor 5106'~~,
	68	'plot_module': 'plot_cr1000_wind',
	69	'plot_names': ('timeseries',),
67	70	},
68	71	'ctd1' : { 'id' : 'ctd1',
…	…
76	79	'depth_units' : 'm',
77	80	'depth_reference' : 'sea_surface',
	81	# Recommended
	82	'source': 'Seabird (SBE) 37 IMP',
78	83	# 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0',
79	84	# 'latest_vars' : ('time','lat','lon','z','depth', 'wtemp', 'cond', 'salin', 'density'),
80		~~# Recommended~~
81		'~~source': 'Seabird (SBE) 37 IMP'~~,
	85	'plot_module': 'plot_cr1000_ctd',
	86	'plot_names': ('timeseries',),
82	87	},
83	88	'ctd2' : { 'id' : 'ctd2',
…	…
91	96	'depth_units' : 'm',
92	97	'depth_reference' : 'sea_surface',
	98	# Recommended
	99	'source': 'Seabird (SBE) 37 IMP',
93	100	# 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0',
94	101	# 'latest_vars' : ('time','lat','lon','z','depth', 'wtemp', 'cond', 'salin', 'density'),
95		~~# Recommended~~
96		'~~source': 'Seabird (SBE) 37 IMP'~~,
	102	'plot_module': 'plot_cr1000_ctd',
	103	'plot_names': ('timeseries',),
97	104	},
98	105	}

raw2proc/trunk/raw2proc/b2_config_20111005.py

r491	r492
13	13	'institution' : 'nccoos',
14	14	#
15		'config_start_date' : '2011-10-0~~6 00~~:00:00',
	15	'config_start_date' : '2011-10-05 22:00:00',
16	16	'config_end_date' : '2011-11-13 00:00:00', # None or yyyy-mm-dd HH:MM:SS
17	17	'packages' : ('met', 'wind', 'ctd1', 'ctd2'),
…	…
45	45	'height_units' : 'm',
46	46	'height_reference' : 'sea_surface',
	47	# Recommended
	48	'source': 'Heise Baro, Rotronics Temp/RH, RM Young Precip, Eppley PSP/PIR',
47	49	# 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0',
48	50	# 'latest_vars' : ('time','lat','lon','z','u', 'v', 'wspd', 'wdir'),
49		~~# Recommended~~
50		'~~source': 'Heise Baro, Rotronics Temp/RH, RM Young Precip, Eppley PSP/PIR'~~,
	51	'plot_module': 'plot_cr1000_met',
	52	'plot_names': ('timeseries',),
51	53	},
52	54	'wind' : { 'id' : 'wind',
…	…
61	63	'height_units' : 'm',
62	64	'height_reference' : 'sea_surface',
	65	# Recommended
	66	'source': 'RM Young Marine Wind Monitor 5106',
63	67	# 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0',
64	68	# 'latest_vars' : ('time','lat','lon','z','u', 'v', 'wspd', 'wdir'),
65		~~# Recommended~~
66		'~~source': 'RM Young Marine Wind Monitor 5106'~~,
	69	'plot_module': 'plot_cr1000_wind',
	70	'plot_names': ('timeseries',),
67	71	},
68	72	'ctd1' : { 'id' : 'ctd1',
…	…
76	80	'depth_units' : 'm',
77	81	'depth_reference' : 'sea_surface',
	82	# Recommended
	83	'source': 'Seabird (SBE) 37 IMP',
78	84	# 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0',
79	85	# 'latest_vars' : ('time','lat','lon','z','depth', 'wtemp', 'cond', 'salin', 'density'),
80		~~# Recommended~~
81		'~~source': 'Seabird (SBE) 37 IMP'~~,
	86	'plot_module': 'plot_cr1000_ctd',
	87	'plot_names': ('timeseries',),
82	88	},
83	89	'ctd2' : { 'id' : 'ctd2',
…	…
91	97	'depth_units' : 'm',
92	98	'depth_reference' : 'sea_surface',
	99	# Recommended
	100	'source': 'Seabird (SBE) 37 IMP',
93	101	# 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0',
94	102	# 'latest_vars' : ('time','lat','lon','z','depth', 'wtemp', 'cond', 'salin', 'density'),
95		~~# Recommended~~
96		'~~source': 'Seabird (SBE) 37 IMP'~~,
	103	'plot_module': 'plot_cr1000_ctd',
	104	'plot_names': ('timeseries',),
97	105	},
98	106	}

raw2proc/trunk/raw2proc/b2_config_20111112.py

r491	r492
13	13	'institution' : 'nccoos',
14	14	#
15		'config_start_date' : '2011-11-12 00:00:00',
16		'config_end_date' : '2012-04-0~~4 0~~0:00:00', # None or yyyy-mm-dd HH:MM:SS
	15	'config_start_date' : '2011-11-13 00:00:00',
	16	'config_end_date' : '2012-04-03 20:00:00', # None or yyyy-mm-dd HH:MM:SS
17	17	'packages' : ('met', 'wind', 'ctd1', 'ctd2'),
18	18	# Required by CF
…	…
45	45	'height_units' : 'm',
46	46	'height_reference' : 'sea_surface',
	47	# Recommended
	48	'source': 'Heise Baro, Rotronics Temp/RH, RM Young Precip, Eppley PSP/PIR',
47	49	# 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0',
48	50	# 'latest_vars' : ('time','lat','lon','z','u', 'v', 'wspd', 'wdir'),
49		~~# Recommended~~
50		'~~source': 'Heise Baro, Rotronics Temp/RH, RM Young Precip, Eppley PSP/PIR'~~,
	51	'plot_module': 'plot_cr1000_met',
	52	'plot_names': ('timeseries',),
51	53	},
52	54	'wind' : { 'id' : 'wind',
…	…
61	63	'height_units' : 'm',
62	64	'height_reference' : 'sea_surface',
	65	# Recommended
	66	'source': 'RM Young Marine Wind Monitor 5106',
63	67	# 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0',
64	68	# 'latest_vars' : ('time','lat','lon','z','u', 'v', 'wspd', 'wdir'),
65		~~# Recommended~~
66		'~~source': 'RM Young Marine Wind Monitor 5106'~~,
	69	'plot_module': 'plot_cr1000_wind',
	70	'plot_names': ('timeseries',),
67	71	},
68	72	'ctd1' : { 'id' : 'ctd1',
…	…
76	80	'depth_units' : 'm',
77	81	'depth_reference' : 'sea_surface',
	82	# Recommended
	83	'source': 'Seabird (SBE) 37 IMP',
78	84	# 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0',
79	85	# 'latest_vars' : ('time','lat','lon','z','depth', 'wtemp', 'cond', 'salin', 'density'),
80		~~# Recommended~~
81		'~~source': 'Seabird (SBE) 37 IMP'~~,
	86	'plot_module': 'plot_cr1000_ctd',
	87	'plot_names': ('timeseries',),
82	88	},
83	89	'ctd2' : { 'id' : 'ctd2',
…	…
91	97	'depth_units' : 'm',
92	98	'depth_reference' : 'sea_surface',
	99	# Recommended
	100	'source': 'Seabird (SBE) 37 IMP',
93	101	# 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0',
94	102	# 'latest_vars' : ('time','lat','lon','z','depth', 'wtemp', 'cond', 'salin', 'density'),
95		~~# Recommended~~
96		'~~source': 'Seabird (SBE) 37 IMP'~~,
	103	'plot_module': 'plot_cr1000_ctd',
	104	'plot_names': ('timeseries',),
97	105	},
98	106	}

raw2proc/trunk/raw2proc/crow_config_20090122.py

r448	r492
25	25	'utc_offset' : 4, # hours offset to utc
26	26	'press_offset' : 0./12., # pressure gauge offset to staff gauge
27		'plot_module' : '~~crow_flow_plot~~',
	27	'plot_module' : 'plot_cr1000_flow',
28	28	'plot_names' : ('timeseries',),
29	29	'csv_dir' : '/seacoos/data/nccoos/latest_csv',
…	…
41	41	'process_module' : 'proc_cr1000_wq',
42	42	'utc_offset' : 4, # hours offset to utc
43		'plot_module' : '~~crow_wq_plot~~',
	43	'plot_module' : 'plot_cr1000_wq',
44	44	'plot_names' : ('timeseries',),
45	45	'csv_dir' : '/seacoos/data/nccoos/latest_csv',

raw2proc/trunk/raw2proc/meet_config_20090122.py

r320	r492
25	25	'utc_offset' : 4, # hours offset to utc
26	26	'press_offset' : 0./12., # pressure gauge offset to staff gauge
27		'plot_module' : '~~meet_flow_plot~~',
	27	'plot_module' : 'plot_cr1000_flow',
28	28	'plot_names' : ('timeseries',),
29	29	'csv_dir' : '/seacoos/data/nccoos/latest_csv',
…	…
37	37	'process_module' : 'proc_cr1000_wq',
38	38	'utc_offset' : 4, # hours offset to utc
39		'plot_module' : '~~meet_wq_plot~~',
	39	'plot_module' : 'plot_cr1000_wq',
40	40	'plot_names' : ('timeseries',),
41	41	'csv_dir' : '/seacoos/data/nccoos/latest_csv',

raw2proc/trunk/raw2proc/proc_cr1000_met.py

r488	r492
1	1	#!/usr/bin/env python
2		# Last modified: Time-stamp: <2012-0~~4-23 14:12:38~~ haines>
	2	# Last modified: Time-stamp: <2012-05-15 15:44:51 haines>
3	3	"""
4	4	how to parse data, and assert what data and info goes into
…	…
113	113	#
114	114	# data['samplenum'][i] = csi[0] # sample number assigned by datalogger in table
115		data['air_press'][i] = ~~csi[1] # Heise Barometer (mbar)~~
	115	data['air_press'][i] = csi[1] # Heise Barometer (psi) to mbar
116	116	data['rh'][i] = csi[2] # relative humidity avg (60 samples for 1 min)
117	117	data['rh_std'][i] = csi[3] # relative humidity std
118	118	data['air_temp'][i] = csi[4] # air temperature avg (deg C)
119	119	data['air_temp_std'][i] = csi[5] # air temperature std (deg C)
120		data['rain'][i] = csi[6] ~~# precip gauge cummulative~~
	120	data['rain'][i] = csi[6]/100. # precip gauge cummulative (mm)
121	121	data['psp'][i] = csi[7] # PSP avg
122	122	data['psp_std'][i] = csi[8] # PSP std
…	…
131	131	# for line
132	132
	133	data['air_press'] = udconvert(data['air_press'], 'psi', 'mbar')[0]
	134	# cannot figure out how to combine the two operations
	135	# for some reason, this one liner does not work
	136	# good = -40<at & at<60
	137	above_tol=-40<data['air_temp']
	138	below_tol=data['air_temp']<60
	139	good = above_tol & below_tol
	140	bad = ~good
	141	data['air_temp'][bad] = numpy.nan
133	142
134	143	# check that no data[dt] is set to Nan or anything but datetime

raw2proc/trunk/raw2proc/procutil.py

r488	r492
1	1	#!/usr/bin/env python
2		# Last modified: Time-stamp: <2012-0~~4-17 12:09:47~~ haines>
	2	# Last modified: Time-stamp: <2012-05-15 10:51:49 haines>
3	3	"""Utilities to help data processing
4	4

raw2proc/trunk/raw2proc/raw2proc.py

r491	r492
1	1	#!/usr/bin/env python
2		# Last modified: Time-stamp: <2012-0~~5-01 16:40:55~~ haines>
	2	# Last modified: Time-stamp: <2012-06-01 17:01:51 haines>
3	3	"""Process raw data to monthly netCDF data files
4	4
…	…
362	362	# list all the raw files in prev-month, this-month, and next-month
363	363	all_raw_files = []
364		m = re.search('\d{4}_\d{2}$', si['raw_dir'])
	364	m = re.search('\d{4}_\d{2}\/*$', si['raw_dir'])
365	365	if m:
366	366	# look for raw_file_glob in specific directory ending in YYYY_MM
…	…
377	377
378	378	all_raw_files.sort()
379
	379
380	380	#
381	381	dt_start = si['proc_start_dt']-timedelta(days=1)
…	…
385	385	for fn in all_raw_files:
386	386	(fndt, granularity) = filt_datetime(os.path.basename(fn), gran=True)
	387	# print (fndt, granularity)
387	388	if granularity == 4:
388	389	# change dt_start to before monthly filename filt_datetime() date

Changeset Navigation

Changeset 492

Legend:

Download in other formats: