root/raw2proc/trunk/raw2proc/proc_jpier_ascii_met.py

#!/usr/bin/env python
# Last modified:  Time-stamp: <2008-02-15 10:51:00 jcleary>
"""
how to parse data, and assert what data and info goes into
creating and updating monthly netcdf files

Texas Weather Instruments - Weather Processing System (WPS) met data
        delimited ASCII file like:
        year mon day hhmm       epoch  tmean hmean wsmean wdmean barom   dewPt wchill rrmean rday rmonth rterm

parser : output delimited ASCII file from onsite perl script
creator : lat, lon, time, air_temp, humidity, wspd, wdir, air_pressure,
                  dew_temp, wchill, rainfall_rate, rainfall_day, rainfall_month, rainfall_term 
updater : time, air_temp, humidity, wspd, wdir, air_pressure,
                  dew_temp, wchill, rainfall_rate, rainfall_day, rainfall_month, rainfall_term

Examples
--------

>> (parse, create, update) = load_processors('proc_jpier_ascii_met')
or
>> si = get_config(cn+'.sensor_info')
>> (parse, create, update) = load_processors(si['met']['proc_module'])

>> lines = load_data(filename)
>> data = parse(platform_info, sensor_info, lines)
>> create(platform_info, sensor_info, data) or
>> update(platform_info, sensor_info, data)

"""

from raw2proc import *
from procutil import *
from ncutil import *
import time

now_dt = datetime.utcnow()
now_dt.replace(microsecond=0)

def parser(platform_info, sensor_info, lines):
        """
        parse and assign met data from JPIER TWS WPS text file

        """
 
        i = 0

        # drop header row from incoming lines list
        if lines[0].startswith('year',0,5):
                print "... Header row present, skipping ..."
                del lines[0]
   
        for line in lines:
                # split line and parse float and integers
                tws = []
                # data row looks like:
                # 2008  1  1 0000 1199163600  17.2  84.8   0.00 0  1015.10  14.4  17.2 1729.1   0.0     0.0 1031.5
       
                sw = re.split('\s+', line)
                for s in sw:
                        m = re.search(REAL_RE_STR, s)
                        if m:
                                tws.append(float(m.groups()[0]))
                # assign specific fields
                n = len(tws)

                # get sample datetime in UTC from data
                # use epoch tws[4] to get time in UTC
                sample_str = '%04d-%02d-%02d %02d:%02d:00' % tuple(time.gmtime(float(tws[4]))[0:5])
                sample_dt = scanf_datetime(sample_str, fmt='%Y-%m-%d %H:%M:%S')

                air_temp = tws[5]          # Air Temperature (deg F)
                humidity = tws[6]               # Humidity (%)
                dew_temp = tws[10]       # Dew Point (deg F)    
                air_pressure = tws[9]   # Air Pressure (Tmean, sec)
                wspd = tws[7]    # Mean Wind Speed (knots)
                wdir = tws[8]      # Mean Wind Direction (deg from N)
                wchill = tws[11]        # Wind Chill (deg F)
                rainfall_rate = tws[12]  # Rainfall Rate ()
                rainfall_day = tws[13]   # Rainfall amount last 24 hours (in)
                rainfall_month = tws[14] # Rainfall amount last month (in)
                rainfall_term = tws[15]  # Rainfall amount since installation (in)

                # set up dict of data if first line
                if i==0:
                        data = {
                                'dt' : numpy.array(numpy.ones((len(lines),), dtype=object)*numpy.nan),
                                'time' : numpy.array(numpy.ones((len(lines),), dtype=long)*numpy.nan),
                                'air_temp' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan),
                                'humidity' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan),
                                'dew_temp' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan),
                                'air_pressure' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan),
                                'wspd' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan),
                                'wdir' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan),
                                'wchill' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan),
                                'rainfall_rate' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan),
                                'rainfall_day' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan),
                                'rainfall_month' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan),
                                'rainfall_term' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan),
                                }
                
                data['dt'][i] = sample_dt # sample datetime
                data['time'][i] = dt2es(sample_dt) # sample time in epoch seconds
                data['air_temp'][i] = air_temp
                data['humidity'][i] = humidity
                data['dew_temp'][i] = dew_temp
                data['air_pressure'][i] =  air_pressure
                data['wspd'][i] =  wspd
                data['wdir'][i] = wdir
                data['wchill'][i] = wchill
                data['rainfall_rate'][i] = rainfall_rate
                data['rainfall_day'][i] =  rainfall_day
                data['rainfall_month'][i] =  rainfall_month
                data['rainfall_term'][i] = rainfall_term 
                i = i+1

        return data

def creator(platform_info, sensor_info, data):
        #
        # 
        title_str = sensor_info['description']+' at '+ platform_info['location']
        global_atts = { 
                'title' : title_str,
                'institution' : 'Unversity of North Carolina at Chapel Hill (UNC-CH)',
                'institution_url' : 'http://nccoos.org',
                'institution_dods_url' : 'http://nccoos.org',
                'metadata_url' : 'http://nccoos.org',
                'references' : 'http://nccoos.org',
                'contact' : 'Jesse Cleary (jcleary@email.unc.edu)',
                # 
                'source' : 'TWS Met station observation',
                'history' : 'raw2proc using ' + sensor_info['process_module'],
                'comment' : 'File created using pycdf'+pycdfVersion()+' and numpy '+pycdfArrayPkg(),
                # conventions
                'Conventions' : 'CF-1.0; SEACOOS-CDL-v2.0',
                # SEACOOS CDL codes
                'format_category_code' : 'fixed-point',
                'institution_code' : platform_info['institution'],
                'platform_code' : platform_info['id'],
                'package_code' : sensor_info['id'],
                # institution specific
                'project' : 'North Carolina Coastal Ocean Observing System (NCCOOS)',
                'project_url' : 'http://nccoos.org',
                # timeframe of data contained in file yyyy-mm-dd HH:MM:SS
                'start_date' : data['dt'][0].strftime("%Y-%m-%d %H:%M:%S"),
                'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), 
                'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
                #
                'creation_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
                'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
                'process_level' : 'level1',
                #
                # must type match to data (e.g. fillvalue is real if data is real)
                '_FillValue' : -99999.,
                }

        var_atts = {
                # coordinate variables
                'time' : {'short_name': 'time',
                                  'long_name': 'Sample Time',
                                  'standard_name': 'time',
                                  'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC
                                  'axis': 'T',
                                  },
                'lat' : {'short_name': 'lat',
                                 'long_name': 'Latitude in Decimal Degrees',
                                 'standard_name': 'latitude',
                                 'reference':'geographic coordinates',
                                 'units': 'degrees_north',
                                 'valid_range':(-90.,90.),
                                 'axis': 'Y',
                                 },
                'lon' : {'short_name': 'lon',
                                 'long_name': 'Longtitude in Decimal Degrees',
                                 'standard_name': 'longtitude',
                                 'reference':'geographic coordinates',
                                 'units': 'degrees_east',
                                 'valid_range':(-180.,180.),
                                 'axis': 'Y',
                                 },
                'z' : {'short_name': 'z',
                                'long_name': 'Height',
                                'standard_name': 'height',
                                'reference':'zero at sea-surface',
                                'positive': 'up',
                                'units': 'm',
                                'axis': 'Z',
                                },
                # data variables
                
                'air_temp' :    {'short_name': 'air_temp',
                                                'long_name': 'Air Temperature',
                                                'standard_name': 'air_temperature',
                                                'units': 'degrees Celsius',
                                                },
                'humidity' :    {'short_name': 'humidity',
                                                'long_name': 'Humidity',
                                                'standard_name': 'humidity',
                                                'units': '%',
                                                },
                'dew_temp' :    {'short_name': 'dew_temp',
                                                'long_name': 'Dew Temperature',
                                                'standard_name': 'dew_temp',                                              
                                                'units': 'degrees Celsius',
                                                },
                'air_pressure' : {'short_name': 'air_pressure',
                                                'long_name': 'Air Pressure at Barometer Height',
                                                'standard_name': 'air_pressure',                                                  
                                                'units': 'hPa',
                                                },
                'wspd' :        {'short_name': 'wspd',
                                                'long_name': 'Wind Speed',
                                                'standard_name': 'wind_speed',
                                                'units': 'm s-1',
                                                'can_be_normalized': '?',
                                                },
                'wdir' :        {'short_name': 'wdir',
                                                'long_name': 'Wind Direction from',
                                                'standard_name': 'wind_from_direction',
                                                'reference': 'clockwise from True North',
                                                'valid_range': '0., 360',
                                                'units': 'degrees',
                                                },
                'wchill' :      {'short_name': 'wchill',
                                                'long_name': 'Wind Chill',
                                                'standard_name': 'wind_chill',
                                                'units': 'degrees Celsius',
                                                },
                'rainfall_rate' :       {'short_name': 'rR',
                                                'long_name': 'Rainfall Rate',
                                                'standard_name': 'rainfall_rate',
                                                'units': 'mm hr-1',
                                                },
                'rainfall_day' : {'short_name': 'rD',
                                                'long_name': 'Rainfall Day',
                                                'standard_name': 'rainfall_day',
                                                'units': 'mm',
                                                },
                'rainfall_month' : {'short_name': 'rM',
                                                'long_name': 'Rainfall Month',
                                                'standard_name': 'rainfall_month',
                                                'units': 'mm',
                                                },
                'rainfall_term' : {'short_name': 'rT',
                                                'long_name': 'Rainfall Term',
                                                'standard_name': 'rainfall_term',
                                                'units': 'mm',
                                                },
        }
        # integer values 
        ntime=NC.UNLIMITED
        nlat=1
        nlon=1
        nz=1
        
        # dimension names use tuple so order of initialization is maintained
        dim_inits = (
                ('ntime', NC.UNLIMITED),
                ('nlat', 1),
                ('nlon', 1),
                ('nz', 1)
                )

        # using tuple of tuples so order of initialization is maintained
        # using dict for attributes order of init not important
        # use dimension names not values
        # (varName, varType, (dimName1, [dimName2], ...))
        var_inits = (
                # coordinate variables
                ('time', NC.INT, ('ntime',)),
                ('lat', NC.FLOAT, ('nlat',)),
                ('lon', NC.FLOAT, ('nlon',)),
                ('z',  NC.FLOAT, ('nz',)),
                # data variables
                ('air_temp', NC.FLOAT, ('ntime',)),
                ('humidity', NC.FLOAT, ('ntime',)),
                ('dew_temp', NC.FLOAT, ('ntime',)),
                ('air_pressure', NC.FLOAT, ('ntime',)),
                ('wspd', NC.FLOAT, ('ntime',)),
                ('wdir', NC.FLOAT, ('ntime',)),
                ('wchill', NC.FLOAT, ('ntime',)),
                ('rainfall_rate', NC.FLOAT, ('ntime',)),
                ('rainfall_day', NC.FLOAT, ('ntime',)),
                ('rainfall_month', NC.FLOAT, ('ntime',)),
                ('rainfall_term', NC.FLOAT, ('ntime',)),
                )

        # subset data only to month being processed (see raw2proc.process())
        i = data['in']

        # var data 
        var_data = (
                ('lat',  platform_info['lat']),
                ('lon', platform_info['lon']),
                ('z', 6),
                #
                ('time', data['time'][i]),
                ('air_temp', data['air_temp'][i]),
                ('humidity', data['humidity'][i]),
                ('dew_temp', data['dew_temp'][i]),
                ('air_pressure', data['air_pressure'][i]),
                ('wspd', data['wspd'][i]),
                ('wdir', data['wdir'][i]),
                ('wchill', data['wchill'][i]),
                ('rainfall_rate', data['rainfall_rate'][i]),
                ('rainfall_day', data['rainfall_day'][i]),
                ('rainfall_month', data['rainfall_month'][i]),
                ('rainfall_term', data['rainfall_term'][i]),
                )
                
        return (global_atts, var_atts, dim_inits, var_inits, var_data)

def updater(platform_info, sensor_info, data):
        #
        global_atts = { 
                # update times of data contained in file (yyyy-mm-dd HH:MM:SS)
                # last date in monthly file
                'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), 
                'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
                #
                'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
                }

        # data variables
        # update any variable attributes like range, min, max
        var_atts = {}
        # var_atts = {
        #       'u': {'max': max(data.u),
        #                 'min': min(data.v),
        #                 },
        #       'v': {'max': max(data.u),
        #                 'min': min(data.v),
        #                 },
        #       }
        
        # subset data only to month being processed (see raw2proc.process())
        i = data['in']

        # data 
        var_data = (
                ('time', data['time'][i]),
                ('air_temp', data['air_temp'][i]),
                ('humidity', data['humidity'][i]),
                ('dew_temp', data['dew_temp'][i]),
                ('air_pressure', data['air_pressure'][i]),
                ('wspd', data['wspd'][i]),
                ('wdir', data['wdir'][i]),
                ('wchill', data['wchill'][i]),
                ('rainfall_rate', data['rainfall_rate'][i]),
                ('rainfall_day', data['rainfall_day'][i]),
                ('rainfall_month', data['rainfall_month'][i]),
                ('rainfall_term', data['rainfall_term'][i]),
                )

        return (global_atts, var_atts, var_data)

#