Index: raw2proc/trunk/raw2proc/hampton_config_20080930.py =================================================================== --- raw2proc/trunk/raw2proc/hampton_config_20080930.py (revision 320) +++ raw2proc/trunk/raw2proc/hampton_config_20080930.py (revision 336) @@ -22,6 +22,6 @@ 'bin_size' : 0.1, # meters 'nbins' : 150, # max number of samples in profile - # 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', - # 'latest_vars' : ('time','lat','lon','z','wtemp','salin'), + 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + 'latest_vars' : ('time','lat','lon','z','wtemp','salin'), }, 'met' : { 'id' : 'met', @@ -33,6 +33,6 @@ 'utc_offset' : 5., # hours offset to Eastern Standard 'anemometer_height' : 2., # meters - # 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', - # 'latest_vars' : ('time','lat','lon','z','u','v','wspd', 'wdir'), + 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + 'latest_vars' : ('time','lat','lon','z','u','v','wspd', 'wdir'), }, } Index: raw2proc/trunk/raw2proc/morgan_config_20080701.py =================================================================== --- raw2proc/trunk/raw2proc/morgan_config_20080701.py (revision 320) +++ raw2proc/trunk/raw2proc/morgan_config_20080701.py (revision 336) @@ -24,6 +24,6 @@ 'bin_size' : 0.1, # meters 'nbins' : 150, # max number of samples in profile - # 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', - # 'latest_vars' : ('time','lat','lon','z','wtemp','salin'), + 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + 'latest_vars' : ('time','lat','lon','z','wtemp','salin'), }, 'met' : { 'id' : 'met', @@ -35,6 +35,6 @@ 'utc_offset' : 5., # hours offset to Eastern Standard 'anemometer_height' : 2., # meters - # 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', - # 'latest_vars' : ('time','lat','lon','z','u','v','wspd', 'wdir'), + 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + 'latest_vars' : ('time','lat','lon','z','u','v','wspd', 'wdir'), }, } Index: raw2proc/trunk/raw2proc/ncutil.py =================================================================== --- raw2proc/trunk/raw2proc/ncutil.py (revision 320) +++ raw2proc/trunk/raw2proc/ncutil.py (revision 336) @@ -1,4 +1,4 @@ #!/usr/bin/env python -# Last modified: Time-stamp: <2009-12-30 15:45:27 haines> +# Last modified: Time-stamp: <2010-07-30 13:00:15 haines> """ Create, update and load utilities for netcdf files @@ -51,4 +51,5 @@ # print varName # print varData + # print varData.shape ncvar = nc.var(varName) # e.g. lat = array(var_data['lat']) Index: raw2proc/trunk/raw2proc/ouba_config_20100701.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/ouba_config_20100701.py (revision 336) @@ -1,0 +1,44 @@ +platform_info = { + 'id' : 'ouba', + 'location' : 'Outer Banks, NC', + ##### use bounding box (limits or polygon) to describe location + 'lat' : (34.5, 38.), # degrees true (-) south, (+) north + 'lon' : (-76, -73.), # degrees true (-) west, (+) east + 'mvar' : -11, # degrees (-) west, (+) east + 'nlat' : 65., + 'nlon' : 45., + # 'mean_water_depth' : -8.14, # meters (-) down, (+) up + # 'mean_water_depth_time_period' : 'June 2006 - June 2008', + 'institution' : 'nccoos', + # + 'config_start_date' : '2010-07-01 00:00:00', + 'config_end_date' : None, # None or yyyy-mm-dd HH:MM:SS + 'packages' : ('hfr', ), + } +sensor_info = { + 'hfr' : { 'id' : 'hfr', + 'description' : 'High Frequency RADAR Surface Current Totals', + 'raw_dir' : '/seacoos/data/nccoos/level0/ouba/hfr_totals', + 'raw_file_glob' : '*.tuv', + 'proc_dir' : '/seacoos/data/nccoos/level1/ouba/hfr_totals', + 'process_module' : 'proc_codar_totals', + 'utc_offset' : 0, # hours offset to utc + 'operating_frequency' : 4.5, # MHz + 'averaging_radius' : 9.0, # kilometers + # 'plot_module' : 'ouba_totals_plot', + # 'plot_names' : ('vecmap',), + # 'csv_dir' : '/seacoos/data/nccoos/latest_csv', + # 'cvs_vars' : ('time','lat','lon','z','u','v'), + 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + 'latest_vars' : ('time','lat','lon','z','u','v'), + }, + } + +## NOTE: grid definition for totals based on 6km spacing and the bounding box +# minlat, maxlat = (34.5, 38.) +# minlon, maxlon = (-76, -73.) +# midlat = minlat + 0.5*(maxlat-minlat) +## ~111 km = 1 deg latitude +# nlat = numpy.round((maxlat-minlat) *111/6) +# nlon = numpy.round((maxlon-minlon) * math.cos(midlat*math.pi/180)*111/6) + Index: raw2proc/trunk/raw2proc/proc_avp_ysi_6600_v1_CDL2.py =================================================================== --- raw2proc/trunk/raw2proc/proc_avp_ysi_6600_v1_CDL2.py (revision 320) +++ raw2proc/trunk/raw2proc/proc_avp_ysi_6600_v1_CDL2.py (revision 336) @@ -1,4 +1,4 @@ #!/usr/bin/env python -# Last modified: Time-stamp: <2009-12-16 16:33:00 haines> +# Last modified: Time-stamp: <2010-05-12 11:19:45 haines> """ how to parse data, and assert what data and info goes into @@ -219,9 +219,15 @@ # get sample datetime from data sample_str = '%02d-%02d-%02d %02d:%02d:%02d' % tuple(ysi[0:6]) - if sensor_info['utc_offset']: - sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%y %H:%M:%S') + \ - timedelta(hours=sensor_info['utc_offset']) - else: - sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%y %H:%M:%S') + try: + if sensor_info['utc_offset']: + sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%y %H:%M:%S') + \ + timedelta(hours=sensor_info['utc_offset']) + else: + sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%y %H:%M:%S') + except TypeError: + if verbose: + print 'bad time stamp, skipping data line .... ' + str(line) + continue + if j +# Last modified: Time-stamp: <2010-05-26 14:26:42 haines> """ how to parse data, and assert what data and info goes into @@ -214,22 +214,26 @@ except: sample_dt = datetime(1970,1,1) - - if sensor_info['utc_offset']: - sample_dt = sample_dt + timedelta(hours=sensor_info['utc_offset']) - - if j +""" +how to parse data, and assert what data and info goes into +creating and updating monthly netcdf files + +CODAR SeaSonde Total Sea Surface Currents (LLUV TOT4) + +parser : sample date and time from header (%TimeStamp:) + table time version (%TableType:) +creator : lat, lon, z, time, u(time, lat, lon), v(time, lat, lon), +updater : time, u(time, lat, lon), v(time, lat, lon), + +Check that grid that totals are calculated over has not changed. +(%Origin, %GridAxis, %GridAxisType, %GridSpacing all the same) + +Examples +-------- + +>> (parse, create, update) = load_processors(module_name_without_dot_py) +For example, +>> (parse, create, update) = load_processors('proc_rdi_logdata_adcp') +or +>> si = get_config(cn+'.sensor_info') +>> (parse, create, update) = load_processors(si['adcp']['proc_module']) + +Then use the generic name of processor to parse data, create or update +monthly output file + +>> lines = load_data(filename) +>> data = parse(platform_info, sensor_info, lines) +>> create(platform_info, sensor_info, data) +or +>> update(platform_info, sensor_info, data) + +""" + +from raw2proc import * +from procutil import * +from ncutil import * + +now_dt = datetime.utcnow() +now_dt.replace(microsecond=0) + +def parser(platform_info, sensor_info, lines): + """ + parse and assign data to variables from CODAR Totals LLUV format + + Notes + ----- + 1. Requires grid definition obtained from sensor_info + For best coverage of totals, this includes overlapping foot print of HATY, DUCK, LISL and CEDR + + """ + + import numpy + from datetime import datetime + from time import strptime + from StringIO import StringIO + from matplotlib.mlab import griddata + + # define the lat/lon grid based on 6km resolution + minlat, maxlat = platform_info['lat'] # (34.5, 38) + minlon, maxlon = platform_info['lon'] # (-76, -73.) + nlat = platform_info['nlat'] + nlon = platform_info['nlon'] + yi = numpy.linspace(minlat, maxlat, nlat) + xi = numpy.linspace(minlon, maxlon, nlon) + + data = { + 'dt' : numpy.array(numpy.ones((1,), dtype=object)*numpy.nan), + 'time' : numpy.array(numpy.ones((1,), dtype=long)*numpy.nan), + 'lon' : numpy.array(numpy.ones((nlon,), dtype=float)*numpy.nan), + 'lat' : numpy.array(numpy.ones((nlat,), dtype=float)*numpy.nan), + 'u' : numpy.array(numpy.ones((1,nlon,nlat), dtype=float)*numpy.nan), + 'v' : numpy.array(numpy.ones((1,nlon,nlat), dtype=float)*numpy.nan), + } + + sample_dt, ftype, ncol, nrow = (None, None, None, None) + # read header that match '%(k): (v)\n' pairs on each line + m = re.findall(r'^(%.*):\s*(.*)$', ''.join(lines), re.MULTILINE) + for k,v in m: + if k == '%TimeStamp': + sample_dt = scanf_datetime(v, fmt='%Y %m %d %H %M %S') + elif k == '%TableType': + ftype = v + elif k == '%TableColumns': + ncol = int(v) + elif k == '%TableRows': + nrow = int(v) + + if nrow: + # read data from string of lines but make it behave like a file object with StringIO + s = StringIO(''.join(lines)) + s.seek(0) # ensures start posn of file + d = numpy.loadtxt(s, comments='%') + # lat, lon, u, v = numpy.loadtxt(s, usecols=(0,1,2,3), comments='%', unpack=True) + if 'TOT4' in ftype: + lon = d[:,0] + lat = d[:,1] + wu = d[:,2] + wv = d[:,3] + gridflag = d[:,4] + wu_std_qual = d[:,5] + wv_std_qual = d[:,6] + cov_qual = d[:,7] + x_dist = d[:,8] + y_dist = d[:,9] + rang = d[:,10] + bearing = d[:,11] + vel_mag = d[:,12] + vel_dir = d[:,13] + s1 = d[:,14] + s2 = d[:,15] + s3 = d[:,16] + s4 = d[:,17] + s5 = d[:,18] + s6 = d[:,19] + + uim = griddata(lon, lat, wu, xi, yi) + vim = griddata(lon, lat, wv, xi, yi) + # returned masked array as an ndarray with masked values filled with fill_value + ui = uim.filled(fill_value=numpy.nan) + vi = vim.filled(fill_value=numpy.nan) + + # --------------------------------------------------------------- + + i = 0 + data['dt'][i] = sample_dt # + data['time'][i] = dt2es(sample_dt) # + data['lon'] = xi # new longitude grid + data['lat'] = yi # new latitude grid + + if nrow: + # use transpose so order is (time, x, y) for netcdf convention + data['u'][i] = ui.T # u-component of water velocity (cm/s) + data['v'][i] = vi.T # v-component of water velocity + + return data + +def creator(platform_info, sensor_info, data): + # + # + title_str = sensor_info['description']+' at '+ platform_info['location'] + global_atts = { + 'title' : title_str, + 'institution' : 'University of North Carolina at Chapel Hill (UNC-CH)', + 'institution_url' : 'http://nccoos.unc.edu', + 'institution_dods_url' : 'http://nccoos.unc.edu', + 'metadata_url' : 'http://nccoos.unc.edu', + 'references' : 'http://nccoos.unc.edu', + 'contact' : 'Sara Haines (haines@email.unc.edu)', + # + 'source' : 'surface current observation', + 'history' : 'raw2proc using ' + sensor_info['process_module'], + 'comment' : 'File created using pycdf'+pycdfVersion()+' and numpy '+pycdfArrayPkg(), + # conventions + 'Conventions' : 'CF-1.0; SEACOOS-CDL-v2.0', + # SEACOOS CDL codes + 'format_category_code' : 'fixed-map', + 'institution_code' : platform_info['institution'], + 'platform_code' : platform_info['id'], + 'package_code' : sensor_info['id'], + # institution specific + 'project' : 'North Carolina Coastal Ocean Observing System (NCCOOS)', + 'project_url' : 'http://nccoos.org', + # timeframe of data contained in file yyyy-mm-dd HH:MM:SS + 'start_date' : data['dt'][0].strftime("%Y-%m-%d %H:%M:%S"), + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + # + 'creation_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + 'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + 'process_level' : 'level1', + # + # must type match to data (e.g. fillvalue is real if data is real) + '_FillValue' : numpy.nan, + } + + var_atts = { + # coordinate variables + 'time' : {'short_name': 'time', + 'long_name': 'Time', + 'standard_name': 'time', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + 'axis': 'T', + }, + 'lat' : {'short_name': 'lat', + 'long_name': 'Latitude', + 'standard_name': 'latitude', + 'reference':'geographic coordinates', + 'units': 'degrees_north', + 'valid_range':(-90.,90.), + 'axis': 'Y', + }, + 'lon' : {'short_name': 'lon', + 'long_name': 'Longitude', + 'standard_name': 'longitude', + 'reference':'geographic coordinates', + 'units': 'degrees_east', + 'valid_range':(-180.,180.), + 'axis': 'Y', + }, + 'z' : {'short_name': 'z', + 'long_name': 'Height', + 'standard_name': 'height', + 'reference':'zero at sea-surface', + 'units': 'm', + 'axis': 'Z', + }, + # data variables + 'u' : {'short_name': 'u', + 'long_name': 'E/W component of current', + 'standard_name': 'eastward_current', + 'units': 'cm sec-1', + 'reference' : 'clockwise from True East', + }, + 'v' : {'short_name': 'v', + 'long_name': 'N/S component of current', + 'standard_name': 'northward_current', + 'units': 'cm sec-1', + 'reference' : 'clockwise from True North', + }, + } + + + # dimension names use tuple so order of initialization is maintained + dim_inits = ( + ('ntime', NC.UNLIMITED), + ('nlat', platform_info['nlat']), + ('nlon', platform_info['nlon']), + ('nz', 1), + ) + + # using tuple of tuples so order of initialization is maintained + # using dict for attributes order of init not important + # use dimension names not values + # (varName, varType, (dimName1, [dimName2], ...)) + var_inits = ( + # coordinate variables + ('time', NC.INT, ('ntime',)), + ('lat', NC.FLOAT, ('nlat',)), + ('lon', NC.FLOAT, ('nlon',)), + ('z', NC.FLOAT, ('nz',)), + # data variables + ('u', NC.FLOAT, ('ntime','nlon','nlat')), + ('v', NC.FLOAT, ('ntime','nlon','nlat')), + ) + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # var data + var_data = ( + ('lat', data['lat']), + ('lon', data['lon']), + ('z', 0.), + # + ('time', data['time'][i]), + ('u', data['u'][i]), + ('v', data['v'][i]), + ) + + return (global_atts, var_atts, dim_inits, var_inits, var_data) + +def updater(platform_info, sensor_info, data): + # + global_atts = { + # update times of data contained in file (yyyy-mm-dd HH:MM:SS) + # last date in monthly file + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + # + 'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + } + + # data variables + # update any variable attributes like range, min, max + var_atts = {} + # var_atts = { + # 'u': {'max': max(data.u), + # 'min': min(data.v), + # }, + # 'v': {'max': max(data.u), + # 'min': min(data.v), + # }, + # } + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # data + var_data = ( + ('time', data['time'][i]), + ('u', data['u'][i]), + ('v', data['v'][i]), + ) + + return (global_atts, var_atts, var_data) + +# Index: raw2proc/trunk/raw2proc/procutil.py =================================================================== --- raw2proc/trunk/raw2proc/procutil.py (revision 333) +++ raw2proc/trunk/raw2proc/procutil.py (revision 336) @@ -1,4 +1,4 @@ #!/usr/bin/env python -# Last modified: Time-stamp: <2010-04-29 09:44:03 haines> +# Last modified: Time-stamp: <2010-07-30 14:10:56 haines> """Utilities to help data processing @@ -458,5 +458,5 @@ ifn = os.path.join(si['proc_dir'], si['proc_filename']) # output file - si['latest_filename'] = 'nccoos_%s_%s_latest.nc' % (platform, package) + si['latest_filename'] = 'nccoos-%s-%s-latest.nc' % (platform, package) ofn = os.path.join(si['latest_dir'], si['latest_filename']) if os.path.exists(ifn): Index: raw2proc/trunk/raw2proc/stones_config_20080701.py =================================================================== --- raw2proc/trunk/raw2proc/stones_config_20080701.py (revision 221) +++ raw2proc/trunk/raw2proc/stones_config_20080701.py (revision 336) @@ -24,6 +24,6 @@ 'bin_size' : 0.1, # meters 'nbins' : 150, # max number of samples in profile - # 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', - # 'latest_vars' : ('time','lat','lon','z','wtemp','salin'), + 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + 'latest_vars' : ('time','lat','lon','z','wtemp','salin'), }, 'met' : { 'id' : 'met', @@ -35,6 +35,6 @@ 'utc_offset' : 5., # hours offset to Eastern Standard 'anemometer_height' : 2., # meters - # 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', - # 'latest_vars' : ('time','lat','lon','z','u', 'v', 'wspd', 'wdir'), + 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + 'latest_vars' : ('time','lat','lon','z','u', 'v', 'wspd', 'wdir'), }, }