Index: raw2proc/trunk/raw2proc/proc_jpier_ascii_met.py
===================================================================
---  (revision )
+++ raw2proc/trunk/raw2proc/proc_jpier_ascii_met.py (revision 113)
@@ -1,0 +1,356 @@
+#!/usr/bin/env python
+# Last modified:  Time-stamp: <2008-02-15 10:51:00 jcleary>
+"""
+how to parse data, and assert what data and info goes into
+creating and updating monthly netcdf files
+
+Texas Weather Instruments - Weather Processing System (WPS) met data
+	delimited ASCII file like:
+	year mon day hhmm	epoch  tmean hmean wsmean wdmean barom   dewPt wchill rrmean rday rmonth rterm
+
+parser : output delimited ASCII file from onsite perl script
+creator : lat, lon, time, air_temp, humidity, wspd, wdir, air_pressure,
+		  dew_temp, wchill, rainfall_rate, rainfall_day, rainfall_month, rainfall_term 
+updater : time, air_temp, humidity, wspd, wdir, air_pressure,
+		  dew_temp, wchill, rainfall_rate, rainfall_day, rainfall_month, rainfall_term
+
+Examples
+--------
+
+>> (parse, create, update) = load_processors('proc_jpier_ascii_met')
+or
+>> si = get_config(cn+'.sensor_info')
+>> (parse, create, update) = load_processors(si['met']['proc_module'])
+
+>> lines = load_data(filename)
+>> data = parse(platform_info, sensor_info, lines)
+>> create(platform_info, sensor_info, data) or
+>> update(platform_info, sensor_info, data)
+
+"""
+
+from raw2proc import *
+from procutil import *
+from ncutil import *
+
+now_dt = datetime.utcnow()
+now_dt.replace(microsecond=0)
+
+def parser(platform_info, sensor_info, lines):
+	"""
+	parse and assign met data from JPIER TWS WPS text file
+
+	"""
+ 
+	i = 0
+
+	# drop header row from incoming lines list
+	if lines[0].startswith('year',0,5):
+		print "... Header row present, skipping ..."
+		del lines[0]
+   
+	for line in lines:
+		# split line and parse float and integers
+		tws = []
+		# data row looks like:
+		# 2008  1  1 0000 1199163600  17.2  84.8   0.00	0  1015.10  14.4  17.2 1729.1	0.0	0.0 1031.5
+       
+ 		sw = re.split('\s+', line)
+		for s in sw:
+			m = re.search(REAL_RE_STR, s)
+			if m:
+				tws.append(float(m.groups()[0]))
+		# assign specific fields
+		n = len(tws)
+		# get sample datetime from data
+		sample_str = '%04d-%02d-%02d %04d:00' % tuple(tws[0:4])
+		sample_dt = scanf_datetime(sample_str, fmt='%Y-%m-%d %H%M:%S')
+		#   datetime(*strptime(sample_str, "%y-%m-%d %H:%M:%S")[0:6])
+
+		air_temp = tws[5]	   # Air Temperature (deg F)
+		humidity = tws[6]		# Humidity (%)
+		dew_temp = tws[10]	 # Dew Point (deg F)	
+		air_pressure = tws[9]   # Air Pressure (Tmean, sec)
+		wspd = tws[7]	 # Mean Wind Speed (knots)
+		wdir = tws[8]	   # Mean Wind Direction (deg from N)
+		wchill = tws[11]	# Wind Chill (deg F)
+		rainfall_rate = tws[12]  # Rainfall Rate ()
+		rainfall_day = tws[13]   # Rainfall amount last 24 hours (in)
+		rainfall_month = tws[14] # Rainfall amount last month (in)
+		rainfall_term = tws[15]  # Rainfall amount since installation (in)
+
+		# set up dict of data if first line
+		if i==0:
+			data = {
+				'dt' : numpy.array(numpy.ones((len(lines),), dtype=object)*numpy.nan),
+				'time' : numpy.array(numpy.ones((len(lines),), dtype=long)*numpy.nan),
+				'air_temp' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan),
+				'humidity' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan),
+				'dew_temp' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan),
+				'air_pressure' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan),
+				'wspd' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan),
+				'wdir' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan),
+				'wchill' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan),
+				'rainfall_rate' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan),
+				'rainfall_day' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan),
+				'rainfall_month' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan),
+				'rainfall_term' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan),
+				}
+		
+		data['dt'][i] = sample_dt # sample datetime
+		data['time'][i] = dt2es(sample_dt) # sample time in epoch seconds
+		data['air_temp'][i] = air_temp
+		data['humidity'][i] = humidity
+		data['dew_temp'][i] = dew_temp
+		data['air_pressure'][i] =  air_pressure
+		data['wspd'][i] =  wspd
+		data['wdir'][i] = wdir
+		data['wchill'][i] = wchill
+		data['rainfall_rate'][i] = rainfall_rate
+		data['rainfall_day'][i] =  rainfall_day
+		data['rainfall_month'][i] =  rainfall_month
+		data['rainfall_term'][i] = rainfall_term 
+		i = i+1
+
+	return data
+
+def creator(platform_info, sensor_info, data):
+	#
+	# 
+	title_str = sensor_info['description']+' at '+ platform_info['location']
+	global_atts = { 
+		'title' : title_str,
+		'institution' : 'Unversity of North Carolina at Chapel Hill (UNC-CH)',
+		'institution_url' : 'http://nccoos.org',
+		'institution_dods_url' : 'http://nccoos.org',
+		'metadata_url' : 'http://nccoos.org',
+		'references' : 'http://nccoos.org',
+		'contact' : 'Jesse Cleary (jcleary@email.unc.edu)',
+		# 
+		'source' : 'TWS Met station observation',
+		'history' : 'raw2proc using ' + sensor_info['process_module'],
+		'comment' : 'File created using pycdf'+pycdfVersion()+' and numpy '+pycdfArrayPkg(),
+		# conventions
+		'Conventions' : 'CF-1.0; SEACOOS-CDL-v2.0',
+		# SEACOOS CDL codes
+		'format_category_code' : 'fixed-point',
+		'institution_code' : platform_info['institution'],
+		'platform_code' : platform_info['id'],
+		'package_code' : sensor_info['id'],
+		# institution specific
+		'project' : 'North Carolina Coastal Ocean Observing System (NCCOOS)',
+		'project_url' : 'http://nccoos.org',
+		# timeframe of data contained in file yyyy-mm-dd HH:MM:SS
+		'start_date' : data['dt'][0].strftime("%Y-%m-%d %H:%M:%S"),
+		'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), 
+		'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
+		#
+		'creation_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
+		'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
+		'process_level' : 'level1',
+		#
+		# must type match to data (e.g. fillvalue is real if data is real)
+		'_FillValue' : -99999.,
+		}
+
+	var_atts = {
+		# coordinate variables
+		'time' : {'short_name': 'time',
+				  'long_name': 'Sample Time',
+				  'standard_name': 'time',
+				  'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC
+				  'axis': 'T',
+				  },
+		'lat' : {'short_name': 'lat',
+				 'long_name': 'Latitude in Decimal Degrees',
+				 'standard_name': 'latitude',
+				 'reference':'geographic coordinates',
+				 'units': 'degrees_north',
+				 'valid_range':(-90.,90.),
+				 'axis': 'Y',
+				 },
+		'lon' : {'short_name': 'lon',
+				 'long_name': 'Longtitude in Decimal Degrees',
+				 'standard_name': 'longtitude',
+				 'reference':'geographic coordinates',
+				 'units': 'degrees_east',
+				 'valid_range':(-180.,180.),
+				 'axis': 'Y',
+				 },
+		'z' : {'short_name': 'z',
+				'long_name': 'Height',
+				'standard_name': 'height',
+				'reference':'zero at sea-surface',
+				'positive': 'up',
+				'units': 'm',
+				'axis': 'Z',
+				},
+		# data variables
+		
+		'air_temp' :	{'short_name': 'air_temp',
+						'long_name': 'Air Temperature',
+						'standard_name': 'air_temperature',
+						'units': 'degrees Celsius',
+						},
+		'humidity' : 	{'short_name': 'humidity',
+						'long_name': 'Humidity',
+						'standard_name': 'humidity',
+						'units': '%',
+						},
+		'dew_temp' : 	{'short_name': 'dew_temp',
+						'long_name': 'Dew Temperature',
+						'standard_name': 'dew_temp',						  
+						'units': 'degrees Celsius',
+						},
+		'air_pressure' : {'short_name': 'air_pressure',
+						'long_name': 'Air Pressure at Barometer Height',
+						'standard_name': 'air_pressure',						  
+						'units': 'hPa',
+						},
+		'wspd' : 	{'short_name': 'wspd',
+						'long_name': 'Wind Speed',
+						'standard_name': 'wind_speed',
+						'units': 'm s-1',
+						'can_be_normalized': '?',
+						},
+		'wdir' :	{'short_name': 'wdir',
+						'long_name': 'Wind Direction from',
+						'standard_name': 'wind_from_direction',
+						'reference': 'clockwise from True North',
+						'valid_range': '0., 360',
+						'units': 'degrees',
+						},
+		'wchill' : 	{'short_name': 'wchill',
+						'long_name': 'Wind Chill',
+						'standard_name': 'wind_chill',
+						'units': 'degrees Celsius',
+						},
+		'rainfall_rate' : 	{'short_name': 'rR',
+						'long_name': 'Rainfall Rate',
+						'standard_name': 'rainfall_rate',
+						'units': 'mm hr-1',
+						},
+		'rainfall_day' : {'short_name': 'rD',
+						'long_name': 'Rainfall Day',
+						'standard_name': 'rainfall_day',
+						'units': 'mm',
+						},
+		'rainfall_month' : {'short_name': 'rM',
+						'long_name': 'Rainfall Month',
+						'standard_name': 'rainfall_month',
+						'units': 'mm',
+						},
+		'rainfall_term' : {'short_name': 'rT',
+						'long_name': 'Rainfall Term',
+						'standard_name': 'rainfall_term',
+						'units': 'mm',
+						},
+	}
+	# integer values 
+	ntime=NC.UNLIMITED
+	nlat=1
+	nlon=1
+	nz=1
+	
+	# dimension names use tuple so order of initialization is maintained
+	dim_inits = (
+		('ntime', NC.UNLIMITED),
+		('nlat', 1),
+		('nlon', 1),
+		('nz', 1)
+		)
+
+	# using tuple of tuples so order of initialization is maintained
+	# using dict for attributes order of init not important
+	# use dimension names not values
+	# (varName, varType, (dimName1, [dimName2], ...))
+	var_inits = (
+		# coordinate variables
+		('time', NC.INT, ('ntime',)),
+		('lat', NC.FLOAT, ('nlat',)),
+		('lon', NC.FLOAT, ('nlon',)),
+		('z',  NC.FLOAT, ('nz',)),
+		# data variables
+		('air_temp', NC.FLOAT, ('ntime',)),
+		('humidity', NC.FLOAT, ('ntime',)),
+		('dew_temp', NC.FLOAT, ('ntime',)),
+		('air_pressure', NC.FLOAT, ('ntime',)),
+		('wspd', NC.FLOAT, ('ntime',)),
+		('wdir', NC.FLOAT, ('ntime',)),
+		('wchill', NC.FLOAT, ('ntime',)),
+		('rainfall_rate', NC.FLOAT, ('ntime',)),
+		('rainfall_day', NC.FLOAT, ('ntime',)),
+		('rainfall_month', NC.FLOAT, ('ntime',)),
+		('rainfall_term', NC.FLOAT, ('ntime',)),
+		)
+
+	# subset data only to month being processed (see raw2proc.process())
+	i = data['in']
+
+	# var data 
+	var_data = (
+		('lat',  platform_info['lat']),
+		('lon', platform_info['lon']),
+		('z', 6),
+		#
+		('time', data['time'][i]),
+		('air_temp', data['air_temp'][i]),
+		('humidity', data['humidity'][i]),
+		('dew_temp', data['dew_temp'][i]),
+		('air_pressure', data['air_pressure'][i]),
+		('wspd', data['wspd'][i]),
+		('wdir', data['wdir'][i]),
+		('wchill', data['wchill'][i]),
+		('rainfall_rate', data['rainfall_rate'][i]),
+		('rainfall_day', data['rainfall_day'][i]),
+		('rainfall_month', data['rainfall_month'][i]),
+		('rainfall_term', data['rainfall_term'][i]),
+		)
+		
+	return (global_atts, var_atts, dim_inits, var_inits, var_data)
+
+def updater(platform_info, sensor_info, data):
+	#
+	global_atts = { 
+		# update times of data contained in file (yyyy-mm-dd HH:MM:SS)
+		# last date in monthly file
+		'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), 
+		'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
+		#
+		'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
+		}
+
+	# data variables
+	# update any variable attributes like range, min, max
+	var_atts = {}
+	# var_atts = {
+	#	'u': {'max': max(data.u),
+	#		  'min': min(data.v),
+	#		  },
+	#	'v': {'max': max(data.u),
+	#		  'min': min(data.v),
+	#		  },
+	#	}
+	
+	# subset data only to month being processed (see raw2proc.process())
+	i = data['in']
+
+	# data 
+	var_data = (
+		('time', data['time'][i]),
+		('air_temp', data['air_temp'][i]),
+		('humidity', data['humidity'][i]),
+		('dew_temp', data['dew_temp'][i]),
+		('air_pressure', data['air_pressure'][i]),
+		('wspd', data['wspd'][i]),
+		('wdir', data['wdir'][i]),
+		('wchill', data['wchill'][i]),
+		('rainfall_rate', data['rainfall_rate'][i]),
+		('rainfall_day', data['rainfall_day'][i]),
+		('rainfall_month', data['rainfall_month'][i]),
+		('rainfall_term', data['rainfall_term'][i]),
+		)
+
+	return (global_atts, var_atts, var_data)
+
+#