Index: raw2proc/trunk/raw2proc/jpier_config_20080722.py =================================================================== --- raw2proc/trunk/raw2proc/jpier_config_20080722.py (revision 180) +++ raw2proc/trunk/raw2proc/jpier_config_20080722.py (revision 208) @@ -21,4 +21,6 @@ 'proc_dir' : '/seacoos/data/nccoos/level1/jpier/met/', 'process_module' : 'proc_jpier_ascii_met', + 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + 'latest_vars' : ('time','lat','lon','z','wspd','wdir', 'air_temp', 'humidity', 'air_pressure'), }, 'adcp' : { 'id' : 'adcp', Index: raw2proc/trunk/raw2proc/morgan_config_20080701.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/morgan_config_20080701.py (revision 208) @@ -1,0 +1,39 @@ +platform_info = { + 'id' : 'morgan', + 'location' : 'Morgan Bay, New River, NC', + 'lat' : 34.7037, # degrees true (-) south, (+) north + 'lon' : 77.4022, # degrees true (-) west, (+) east + 'mvar' : -9.42, # degrees (-) west, (+) east + 'water_depth': 4.0, # nominal depth in meters (should be MSL) + 'institution' : 'nccoos', + # + 'config_start_date' : '2008-07-01 00:00:00', + 'config_end_date' : None, # None or yyyy-mm-dd HH:MM:SS + 'packages' : ('avp', 'met'), + } + +sensor_info = { + 'avp' : { 'id' : 'avp', + 'description' : 'Automated profiler data ctd and water quality', + 'raw_dir' : '/seacoos/data/nccoos/level0/morgan/avp/', + 'raw_file_glob' : '*.dat', + 'proc_dir' : '/seacoos/data/nccoos/level1/morgan/avp/', + 'process_module' : 'proc_avp_ysi_6600_v2', + 'utc_offset' : 5., # hours offset to Eastern Standard + 'bin_size' : 0.1, # meters + 'nbins' : 40, # for now, water_depth (MSL) divided by bin_size + # 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + # 'latest_vars' : ('time','lat','lon','z','wtemp','salin', 'turb', 'ph', 'chl', 'do'), + }, + 'met' : { 'id' : 'met', + 'description' : 'Wind Data at Automated Vertical Profiler Station', + 'raw_dir' : '/seacoos/data/nccoos/level0/morgan/met/', + 'raw_file_glob' : '*.wnd', + 'proc_dir' : '/seacoos/data/nccoos/level1/morgan/met/', + 'process_module' : 'proc_avp_ascii_met', + 'utc_offset' : 5., # hours offset to Eastern Standard + 'anemometer_height' : 2., # meters + # 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + # 'latest_vars' : ('time','lat','lon','z','u','v'), + }, + } Index: raw2proc/trunk/raw2proc/proc_avp_ascii_met.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/proc_avp_ascii_met.py (revision 208) @@ -1,0 +1,335 @@ +#!/usr/bin/env python +# Last modified: Time-stamp: <2008-09-10 16:44:18 haines> +""" +how to parse data, and assert what data and info goes into +creating and updating monthly netcdf files + + +parser : output delimited ASCII file from onsite perl script +creator : lat, lon, z, time, wspd, wdir, cdir, u, v, nwnd + +updater : time, wspd, wdir, cdir, u, v, nwnd + + +Examples +-------- + +>> (parse, create, update) = load_processors('proc_avp_ascii_wnd') +or +>> si = get_config(cn+'.sensor_info') +>> (parse, create, update) = load_processors(si['met']['proc_module']) + +>> lines = load_data(filename) +>> data = parse(platform_info, sensor_info, lines) +>> create(platform_info, sensor_info, data) or +>> update(platform_info, sensor_info, data) + +""" + +from raw2proc import * +from procutil import * +from ncutil import * +import time + +now_dt = datetime.utcnow() +now_dt.replace(microsecond=0) + +def parser(platform_info, sensor_info, lines): + """ + parse Automated Vertical Profile Station (AVP) Wind data + + Notes + ----- + 1. Wind: + + Date, time, speed, dir, compass dir, North , East, n-samples + (m/s) (magN) (magN) (m/s) (m/s) + + 08/11/2008 00:00:00 5.881 197 197 -5.638 -1.674 696 + 08/11/2008 00:30:00 5.506 216 197 -4.448 -3.246 699 + 08/11/2008 01:00:00 7.233 329 159 6.183 -3.754 705 + + """ + import numpy + from datetime import datetime + from time import strptime + + # get sample datetime from filename + fn = sensor_info['fn'] + sample_dt_start = filt_datetime(fn)[0] + N = len(lines) + data = { + 'dt' : numpy.array(numpy.ones((N,), dtype=object)*numpy.nan), + 'time' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan), + 'wspd' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'wdir' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'cdir' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'v' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'u' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'nwnd' : numpy.array(numpy.ones((N,), dtype=int)*numpy.nan), + } + + i = 0 + lines.sort() + + mvar = platform_info['mvar'] # Magnetic Variation at station + # mvar = -8.0 # (??????) + + wnd = [] + for line in lines: + wnd = [] + # split line and parse float and integers + sw = re.split('[\s/\:]*', line) + for s in sw: + m = re.search(REAL_RE_STR, s) + if m: + wnd.append(float(m.groups()[0])) + + if len(wnd)==12: + # get sample datetime from data + sample_str = '%02d-%02d-%4d %02d:%02d:%02d' % tuple(wnd[0:6]) + if sensor_info['utc_offset']: + sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%Y %H:%M:%S') + \ + timedelta(hours=sensor_info['utc_offset']) + else: + sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%Y %H:%M:%S') + + wspd = int(wnd[6]) # wind speed (m/s) + wdir = int(wnd[7]) # wind dir (mag N) + cdir = wnd[8] # compass dir (mag N) + u = wnd[9] # Easterly (?) Component (m/s) (mag or true??) + v = wnd[10] # Northerly (?) Component (m/s) (mag or true??) + nwnd = int(wnd[11]) # Number of samples in wind average + + # combine wind dir and buoy compass direction + # correct direction from magnetic N to true N + # rotate u, v to true N + # or + # recompute u, v from direction and speed + + data['dt'][i] = sample_dt # sample datetime + data['time'][i] = dt2es(sample_dt) # sample time in epoch seconds + data['wspd'][i] = wspd + data['wdir'][i] = wdir + data['cdir'][i] = cdir + data['u'][i] = u + data['v'][i] = v + data['nwnd'][i] = nwnd + + i=i+1 + + # if len(wnd)==12 + # for line + + return data + +def creator(platform_info, sensor_info, data): + # + # + title_str = sensor_info['description']+' at '+ platform_info['location'] + global_atts = { + 'title' : title_str, + 'institution' : 'Unversity of North Carolina at Chapel Hill (UNC-CH)', + 'institution_url' : 'http://nccoos.org', + 'institution_dods_url' : 'http://nccoos.org', + 'metadata_url' : 'http://nccoos.org', + 'references' : 'http://nccoos.org', + 'contact' : 'Sara Haines (haines@email.unc.edu)', + # + 'source' : 'AVP Wind Observations', + 'history' : 'raw2proc using ' + sensor_info['process_module'], + 'comment' : 'File created using pycdf'+pycdfVersion()+' and numpy '+pycdfArrayPkg(), + # conventions + 'Conventions' : 'CF-1.0; SEACOOS-CDL-v2.0', + # SEACOOS CDL codes + 'format_category_code' : 'fixed-point', + 'institution_code' : platform_info['institution'], + 'platform_code' : platform_info['id'], + 'package_code' : sensor_info['id'], + # institution specific + 'project' : 'North Carolina Coastal Ocean Observing System (NCCOOS)', + 'project_url' : 'http://nccoos.org', + # timeframe of data contained in file yyyy-mm-dd HH:MM:SS + 'start_date' : data['dt'][0].strftime("%Y-%m-%d %H:%M:%S"), + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + # + 'creation_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + 'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + 'process_level' : 'level1', + # + # must type match to data (e.g. fillvalue is real if data is real) + '_FillValue' : -99999., + } + + var_atts = { + # coordinate variables + 'time' : {'short_name': 'time', + 'long_name': 'Sample Time', + 'standard_name': 'time', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + 'axis': 'T', + }, + 'lat' : {'short_name': 'lat', + 'long_name': 'Latitude in Decimal Degrees', + 'standard_name': 'latitude', + 'reference':'geographic coordinates', + 'units': 'degrees_north', + 'valid_range':(-90.,90.), + 'axis': 'Y', + }, + 'lon' : {'short_name': 'lon', + 'long_name': 'Longtitude in Decimal Degrees', + 'standard_name': 'longtitude', + 'reference':'geographic coordinates', + 'units': 'degrees_east', + 'valid_range':(-180.,180.), + 'axis': 'Y', + }, + 'z' : {'short_name': 'z', + 'long_name': 'Height', + 'standard_name': 'height', + 'reference':'zero at sea-surface', + 'positive': 'up', + 'units': 'm', + 'axis': 'Z', + }, + # data variables + 'wspd' : {'short_name': 'wspd', + 'long_name': 'Wind Speed', + 'standard_name': 'wind_speed', + 'units': 'm s-1', + 'can_be_normalized': 'yes', + 'z' : sensor_info['anemometer_height'], + }, + 'wdir' : {'short_name': 'wdir', + 'long_name': 'Wind Direction from', + 'standard_name': 'wind_from_direction', + 'reference': 'clockwise from Magnetic North', + 'valid_range': (0., 360), + 'units': 'degrees', + 'z' : sensor_info['anemometer_height'], + }, + 'cdir' : {'short_name': 'cdir', + 'long_name': 'Buoy Orientation', + 'standard_name': 'compass_direction', + 'reference': 'clockwise from Magnetic North', + 'valid_range': (0., 360), + 'units': 'degrees', + }, + 'u' : {'short_name': 'u', + 'long_name': 'East/West Component of Wind', + 'standard_name': 'eastward_wind', + 'reference': 'relative to True East (?)', + 'units': 'm s-1', + 'can_be_normalized': 'yes', + 'z' : sensor_info['anemometer_height'], + }, + 'v' : {'short_name': 'v', + 'long_name': 'North/South Component of Wind', + 'standard_name': 'northward_wind', + 'reference': 'relative to True North (?)', + 'units': 'm s-1', + 'can_be_normalized': 'yes', + 'z' : sensor_info['anemometer_height'], + }, + 'nwnd' : {'short_name': 'nwnd', + 'long_name': 'Number of wind samples in sample period', + 'standard_name': 'number_of_samples', + 'units': 'm s-1', + }, + + } + # integer values + ntime=NC.UNLIMITED + nlat=1 + nlon=1 + nz=1 + + # dimension names use tuple so order of initialization is maintained + dim_inits = ( + ('ntime', NC.UNLIMITED), + ('nlat', 1), + ('nlon', 1), + ('nz', 1) + ) + + # using tuple of tuples so order of initialization is maintained + # using dict for attributes order of init not important + # use dimension names not values + # (varName, varType, (dimName1, [dimName2], ...)) + var_inits = ( + # coordinate variables + ('time', NC.INT, ('ntime',)), + ('lat', NC.FLOAT, ('nlat',)), + ('lon', NC.FLOAT, ('nlon',)), + ('z', NC.FLOAT, ('nz',)), + # data variables + ('wspd', NC.FLOAT, ('ntime',)), + ('wdir', NC.FLOAT, ('ntime',)), + ('cdir', NC.FLOAT, ('ntime',)), + ('u', NC.FLOAT, ('ntime',)), + ('v', NC.FLOAT, ('ntime',)), + ('nwnd', NC.FLOAT, ('ntime',)), + ) + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # var data + var_data = ( + ('lat', platform_info['lat']), + ('lon', platform_info['lon']), + ('z', 1), + # + ('time', data['time'][i]), + ('wspd', data['wspd'][i]), + ('wdir', data['wdir'][i]), + ('cdir', data['cdir'][i]), + ('u', data['u'][i]), + ('v', data['v'][i]), + ('nwnd', data['nwnd'][i]), + ) + + return (global_atts, var_atts, dim_inits, var_inits, var_data) + +def updater(platform_info, sensor_info, data): + # + global_atts = { + # update times of data contained in file (yyyy-mm-dd HH:MM:SS) + # last date in monthly file + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + # + 'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + } + + # data variables + # update any variable attributes like range, min, max + var_atts = {} + # var_atts = { + # 'u': {'max': max(data.u), + # 'min': min(data.v), + # }, + # 'v': {'max': max(data.u), + # 'min': min(data.v), + # }, + # } + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # data + var_data = ( + ('time', data['time'][i]), + ('wspd', data['wspd'][i]), + ('wdir', data['wdir'][i]), + ('cdir', data['cdir'][i]), + ('u', data['u'][i]), + ('v', data['v'][i]), + ('nwnd', data['nwnd'][i]), + ) + + return (global_atts, var_atts, var_data) + +# Index: raw2proc/trunk/raw2proc/proc_avp_ysi_6600_v2.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/proc_avp_ysi_6600_v2.py (revision 208) @@ -1,0 +1,420 @@ +#!/usr/bin/env python +# Last modified: Time-stamp: <2008-09-09 15:09:47 haines> +""" +how to parse data, and assert what data and info goes into +creating and updating monthly netcdf files + +parse data from YSI 6600 V2-2 on an automated veritical profiler (avp) + +parser : sample date and time, water_depth for each profile + water temperature, conductivity, pressure (depth), salinity, pH, dissolved oxygen, turbidity, and chlorophyll + raw data averaged to 10 cm bins + +creator : lat, lon, z, time, water_depth, water_temp, cond, salin, ph, turb, chl, do +updator : time, water_depth, water_temp, cond, salin, ph, turb, chl, do + + +Examples +-------- + +>> (parse, create, update) = load_processors('proc_avp_ysi_6600_v2') +or +>> si = get_config(cn+'.sensor_info') +>> (parse, create, update) = load_processors(si['adcp']['proc_module']) + +>> lines = load_data(filename) +>> data = parse(platform_info, sensor_info, lines) +>> create(platform_info, sensor_info, data) or +>> update(platform_info, sensor_info, data) + +""" + + +from raw2proc import * +from procutil import * +from ncutil import * + +now_dt = datetime.utcnow() +now_dt.replace(microsecond=0) + +def parser(platform_info, sensor_info, lines): + """ + parse Automated Vertical Profile Station (AVP) Water Quality Data + + month, day, year, hour, min, sec, temp (deg. C), conductivity + (mS/cm), salinity (ppt or PSU), depth (meters), pH, turbidity (NTU), + chlorophyll (micrograms per liter), DO (micrograms per liter) + + Notes + ----- + 1. Column Format + + temp, cond, salin, depth, pH, turb, chl, DO + (C), (mS/cm), (ppt), (m), pH, (NTU), (ug/l), (ug/l) + + Profile Time: 00:30:00 + Profile Date: 08/18/2008 + Profile Depth: 255.0 cm + Profile Location: Stones Bay Serial No: 00016B79, ID: AVP1_SERDP + 08/18/08 00:30:06 26.94 41.87 26.81 0.134 8.00 3.4 4.5 6.60 + 08/18/08 00:30:07 26.94 41.87 26.81 0.143 8.00 3.4 4.8 6.59 + 08/18/08 00:30:08 26.94 41.87 26.81 0.160 8.00 3.4 4.8 6.62 + 08/18/08 00:30:09 26.94 41.87 26.81 0.183 8.00 3.4 4.8 6.66 + + 2. While each parameter is measured uniquely with time and depth such that, temp(t) and z(t) + match up with time, we want to grid depth every 1 cm and make each param as temp(t,z). + + Tony Whipple at IMS says 'The AVPs sample at one second intervals. + Between the waves and the instrument descending from a spool of + line with variable radius it works out to about 3-5 cm between + observations on average. When I process the data to make the + images, I bin the data every 10 cm and take the average of however + many observations fell within that bin.' + + Do we interpolate or average samples in bin? + + """ + import numpy + from datetime import datetime + from time import strptime + + # get sample datetime from filename + fn = sensor_info['fn'] + sample_dt_start = filt_datetime(fn)[0] + + # how many profiles in one file, count number of "Profile Time:" in lines + nprof = 0 + for line in lines: + m=re.search("Profile Time:", line) + if m: + nprof=nprof+1 + + # remove first occurrence of blank line if within first 10-40 lines + # and put it on the end to signal end of profile after last profile + for i in range(len(lines[0:40])): + if re.search("^ \r\n", lines[i]): + # print str(i) + " " + lines[i] + " " + lines[i+1] + blank_line = lines.pop(i) + lines.append(blank_line) + + bin_size = sensor_info['bin_size'] # Bin Size (meters) + nominal_depth = platform_info['water_depth'] # Mean sea level at station (meters) or nominal water depth + z = numpy.arange(0, -1*nominal_depth, -1*bin_size, dtype=float) + + N = nprof + nbins = len(z) + + if nbins != sensor_info['nbins']: + print 'Number of bins computed from water_depth and bin_size ('+ \ + str(nbins)+') does not match config number ('+ \ + str(sensor_info['nbins'])+')' + + data = { + 'dt' : numpy.array(numpy.ones((N,), dtype=object)*numpy.nan), + 'time' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan), + 'z' : numpy.array(numpy.ones((nbins,), dtype=float)*numpy.nan), + # + 'wd' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan), + 'wtemp' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + 'cond' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + 'salin' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + 'turb' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + 'ph' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + 'chl' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + 'do' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + } + + # current profile count + i = 0 + + for line in lines: + ysi = [] + # split line and parse float and integers + sw = re.split('[\s/\:]*', line) + for s in sw: + m = re.search(REAL_RE_STR, s) + if m: + ysi.append(float(m.groups()[0])) + + if re.search("Profile Time:", line): + HH=ysi[0] + MM=ysi[1] + SS=ysi[2] + elif re.search("Profile Date:", line): + mm=ysi[0] + dd=ysi[1] + yyyy=ysi[2] + elif re.search("Profile Depth:", line): + wd = ysi[0]/100. # cm to meters + sample_str = '%02d-%02d-%4d %02d:%02d:%02d' % (mm,dd,yyyy,HH,MM,SS) + if sensor_info['utc_offset']: + sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%Y %H:%M:%S') + \ + timedelta(hours=sensor_info['utc_offset']) + else: + sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%Y %H:%M:%S') + + # initialize for new profile at zero for averaging samples within each bin + wtemp = numpy.zeros(nbins) + cond = numpy.zeros(nbins) + salin = numpy.zeros(nbins) + turb = numpy.zeros(nbins) + ph = numpy.zeros(nbins) + chl = numpy.zeros(nbins) + do = numpy.zeros(nbins) + Ns = numpy.zeros(nbins) # count samples per bin for averaging + elif len(ysi)==14: + # get sample datetime from data + # sample_str = '%02d-%02d-%2d %02d:%02d:%02d' % tuple(ysi[0:6]) + # if sensor_info['utc_offset']: + # sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%Y %H:%M:%S') + \ + # timedelta(hours=sensor_info['utc_offset']) + # else: + # sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%y %H:%M:%S') + + depth = -1*ysi[9] # depth (m, positive up) + ibin = ((z)<=depth)*(depth<(z+bin_size)) + + Ns[ibin] = Ns[ibin]+1 + wtemp[ibin] = wtemp[ibin]+ysi[6] # water temperature (C) + cond[ibin] = cond[ibin]+ysi[7] # conductivity (mS/cm) + salin[ibin] = salin[ibin]+ysi[8] # salinity (ppt or PSU??) + # + ph[ibin] = ph[ibin]+ysi[10] # ph + turb[ibin] = turb[ibin]+ysi[11] # turbidity (NTU) + chl[ibin] = chl[ibin]+ysi[12] # chlorophyll (ug/l) + do[ibin] = do[ibin]+ysi[13] # dissolved oxygen (mg/l) + + elif (len(ysi)==0): # each profile separated by empty line + # average summations by sample count per bin + # where count is zero make it NaN so average is not divide by zero + Ns[Ns==0]=numpy.nan*Ns[Ns==0] + + data['dt'][i] = sample_dt # sample datetime + data['time'][i] = dt2es(sample_dt) # sample time in epoch seconds + data['wd'][i] = wd + data['z'] = z + # divide by counts + data['wtemp'][i] = wtemp/Ns + data['cond'][i] = cond/Ns + data['salin'][i] = salin/Ns + data['turb'][i] = turb/Ns + data['ph'][i] = ph/Ns + data['chl'][i] = chl/Ns + data['do'][i] = do/Ns + + i=i+1 + + # if-elif + # for line + + return data + + +def creator(platform_info, sensor_info, data): + # + # + title_str = sensor_info['description']+' at '+ platform_info['location'] + global_atts = { + 'title' : title_str, + 'institution' : 'Unversity of North Carolina at Chapel Hill (UNC-CH)', + 'institution_url' : 'http://nccoos.unc.edu', + 'institution_dods_url' : 'http://nccoos.unc.edu', + 'metadata_url' : 'http://nccoos.unc.edu', + 'references' : 'http://nccoos.unc.edu', + 'contact' : 'Sara Haines (haines@email.unc.edu)', + # + 'source' : 'fixed-automated-profiler observation', + 'history' : 'raw2proc using ' + sensor_info['process_module'], + 'comment' : 'File created using pycdf'+pycdfVersion()+' and numpy '+pycdfArrayPkg(), + # conventions + 'Conventions' : 'CF-1.0; SEACOOS-CDL-v2.0', + # SEACOOS CDL codes + 'format_category_code' : 'fixed-profiler', + 'institution_code' : platform_info['institution'], + 'platform_code' : platform_info['id'], + 'package_code' : sensor_info['id'], + # institution specific + 'project' : 'North Carolina Coastal Ocean Observing System (NCCOOS)', + 'project_url' : 'http://nccoos.unc.edu', + # timeframe of data contained in file yyyy-mm-dd HH:MM:SS + # first date in monthly file + 'start_date' : data['dt'][0].strftime("%Y-%m-%d %H:%M:%S"), + # last date in monthly file + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + # + 'creation_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + 'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + 'process_level' : 'level1', + # + # must type match to data (e.g. fillvalue is real if data is real) + '_FillValue' : -99999., + } + + var_atts = { + # coordinate variables + 'time' : {'short_name': 'time', + 'long_name': 'Time', + 'standard_name': 'time', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + 'axis': 'T', + }, + 'lat' : {'short_name': 'lat', + 'long_name': 'Latitude', + 'standard_name': 'latitude', + 'reference':'geographic coordinates', + 'units': 'degrees_north', + 'valid_range':(-90.,90.), + 'axis': 'Y', + }, + 'lon' : {'short_name': 'lon', + 'long_name': 'Longtitude', + 'standard_name': 'longtitude', + 'reference':'geographic coordinates', + 'units': 'degrees_east', + 'valid_range':(-180.,180.), + 'axis': 'Y', + }, + 'z' : {'short_name': 'z', + 'long_name': 'Height', + 'standard_name': 'height', + 'reference':'zero at sea-surface', + 'positive' : 'up', + 'units': 'm', + 'axis': 'Z', + }, + # data variables + 'wd': {'short_name': 'wd', + 'long_name': 'Water Depth', + 'standard_name': 'water_depth', + 'units': 'm', + }, + 'wtemp': {'short_name': 'wtemp', + 'long_name': 'Water Temperature', + 'standard_name': 'water_temperature', + 'units': 'degrees Celsius', + }, + 'cond': {'short_name': 'cond', + 'long_name': 'Conductivity', + 'standard_name': 'conductivity', + 'units': 'mS cm-1', + }, + 'salin': {'short_name': 'salin', + 'long_name': 'Salinity', + 'standard_name': 'salinity', + 'units': 'PSU', + }, + 'turb': {'short_name': 'turb', + 'long_name': 'Turbidity', + 'standard_name': 'turbidity', + 'units': 'NTU', + }, + 'ph': {'short_name': 'ph', + 'long_name': 'pH', + 'standard_name': 'ph', + 'units': '', + }, + 'chl': {'short_name': 'chl', + 'long_name': 'Chlorophyll', + 'standard_name': 'chlorophyll', + 'units': 'ug l-1', + }, + 'do': {'short_name': 'do', + 'long_name': 'Dissolved Oxygen', + 'standard_name': 'dissolved_oxygen', + 'units': 'mg l-1', + }, + } + + # dimension names use tuple so order of initialization is maintained + dim_inits = ( + ('ntime', NC.UNLIMITED), + ('nlat', 1), + ('nlon', 1), + ('nz', sensor_info['nbins']) + ) + + # using tuple of tuples so order of initialization is maintained + # using dict for attributes order of init not important + # use dimension names not values + # (varName, varType, (dimName1, [dimName2], ...)) + var_inits = ( + # coordinate variables + ('time', NC.INT, ('ntime',)), + ('lat', NC.FLOAT, ('nlat',)), + ('lon', NC.FLOAT, ('nlon',)), + ('z', NC.FLOAT, ('nz',)), + # data variables + ('wd', NC.FLOAT, ('ntime',)), + ('wtemp', NC.FLOAT, ('ntime', 'nz')), + ('cond', NC.FLOAT, ('ntime', 'nz')), + ('salin', NC.FLOAT, ('ntime', 'nz')), + ('turb', NC.FLOAT, ('ntime', 'nz')), + ('ph', NC.FLOAT, ('ntime', 'nz')), + ('chl', NC.FLOAT, ('ntime', 'nz')), + ('do', NC.FLOAT, ('ntime', 'nz')), + ) + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # var data + var_data = ( + ('lat', platform_info['lat']), + ('lon', platform_info['lon']), + ('z', data['z']), + # + ('time', data['time'][i]), + ('wd', data['wd'][i]), + ('wtemp', data['wtemp'][i]), + ('cond', data['cond'][i]), + ('salin', data['salin'][i]), + ('turb', data['turb'][i]), + ('ph', data['ph'][i]), + ('chl', data['chl'][i]), + ('do', data['do'][i]), + ) + + return (global_atts, var_atts, dim_inits, var_inits, var_data) + +def updater(platform_info, sensor_info, data): + # + global_atts = { + # update times of data contained in file (yyyy-mm-dd HH:MM:SS) + # last date in monthly file + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + # + 'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + } + + # data variables + # update any variable attributes like range, min, max + var_atts = {} + # var_atts = { + # 'wtemp': {'max': max(data.u), + # 'min': min(data.v), + # }, + # 'cond': {'max': max(data.u), + # 'min': min(data.v), + # }, + # } + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # data + var_data = ( + ('time', data['time'][i]), + ('wd', data['wd'][i]), + ('wtemp', data['wtemp'][i]), + ('cond', data['cond'][i]), + ('salin', data['salin'][i]), + ('turb', data['turb'][i]), + ('ph', data['ph'][i]), + ('chl', data['chl'][i]), + ('do', data['do'][i]), + ) + + return (global_atts, var_atts, var_data) +# Index: raw2proc/trunk/raw2proc/proc_rdi_rawdata_adcp.py =================================================================== --- raw2proc/trunk/raw2proc/proc_rdi_rawdata_adcp.py (revision 101) +++ (revision ) @@ -1,278 +1,0 @@ -#!/usr/bin/env python -# Last modified: Time-stamp: <2007-12-27 15:16:33 haines> -""" -how to parse data, and assert what data and info goes into -creating and updating monthly netcdf files - -RDI/Wavesmon processed adcp current profile data - -parser : sample date and time, ensemble number, currents - and wave summary output from WavesMon software -nc_creator : -nc_updator : - -Examples --------- - ->> (parse, create, update) = load_processors('proc_rdi_logdata') ->> data = parse(lines) ->> create(platform_info, sensor_info, data) ->> update(platform_info, sensor_info, data) - -""" - -def parser(lines): - """ - parse and assign currents data from RDI ADCP Log Data - - """ - i = 0 - - for line in lines: - # split line and parse float and integers - rdi = [] - sw = re.split(',', line) - for s in sw: - m = re.search(REAL_RE_STR, s) - if m: - rdi.append(float(m.groups()[0])) - - # assign specific fields - n = len(rdi) - burst_num = int(rdi[0]) # Ensemble Number - - # get sample datetime from data - sample_str = '%02d-%02d-%02d %02d:%02d:%02d' % tuple(rdi[1:7]) - sample_dt = datetime(*strptime(sample_str, "%y-%m-%d %H:%M:%S")[0:6]) - - # get sample datetime from filename - # compare with datetime from filename - - sig_wave_ht = rdi[8] # Significant Wave Height (Hs, meters) - peak_wave_period = rdi[9] # Peak Wave Period (Tp, sec) - peak_wave_dir = rdi[10] # Peak Wave Direction (deg N) - max_wave_ht = rdi[12] # Maximum Wave Height (Hmax, meters) - max_wave_period = rdi[13] # Maximum Wave Period (Tmax, sec) - - water_depth = rdi[11]/1000 # Water Depth (meters) (based on ADCP backscatter or input config??) - nbins = int(rdi[14]) # Number of bins - - current_spd = numpy.array(rdi[15::2]) # starting at idx=15 skip=2 to end - current_dir = numpy.array(rdi[16::2]) # starting at idx=16 skip=2 to end - - if nbins!=sensor_info['adcp']['nbins']: - print 'Number of bins reported in data ('+ \ - str(nbins)+') does not match config number ('+ \ - str(sensor_info['adcp']['nbins'])+')' - - if len(current_spd)!=nbins or len(current_dir)!=nbins: - print 'Data length does not match number of bins in data' - - ibad = (current_spd==-32768) | (current_dir==-32768) - current_spd[ibad] = numpy.nan - current_dir[ibad] = numpy.nan - - # these items can also be teased out of raw adcp but for now get from config file - th = sensor_info['adcp']['transducer_ht'] # Transducer height above bottom (meters) - bh = sensor_info['adcp']['blanking_ht'] # Blanking height above Transducer (meters) - bin_size = sensor_info['adcp']['bin_size'] # Bin Size (meters) - - # compute height for each bin above the bottom - bins = numpy.arange(1,nbins+1) - bin_habs = (bins*bin_size+bin_size/2)+th+bh - - # compute water mask - # Using George Voulgaris' method based on water depth - # minus half of the significant wave height (Hs) - # and computed habs - # if positive is up, what's less than zero depth? - bin_depths = bin_habs-(water_depth-sig_wave_ht/2) - iwater = bin_depths+bin_size/2 < 0 - - z = bin_habs - # check that length of bin_depths is equal to nbins - u = numpy.ones(nbins)*numpy.nan - v = numpy.ones(nbins)*numpy.nan - - u[iwater] = current_spd[iwater]*numpy.sin(current_dir[iwater]*numpy.pi/180) - v[iwater] = current_spd[iwater]*numpy.cos(current_dir[iwater]*numpy.pi/180) - - # set up dict of data if first line - if i==0: - data = { - 'en' : numpy.array(numpy.ones((len(lines),), dtype=float)*numpy.nan), - 'dt' : numpy.array(numpy.ones((len(lines),), dtype=object)*numpy.nan), - 'nbins' : numpy.array(numpy.zeros((len(lines),), dtype=int)), - 'z' : numpy.array(numpy.ones((len(lines),nbins), dtype=float)*numpy.nan), - 'u' : numpy.array(numpy.ones((len(lines),nbins), dtype=float)*numpy.nan), - 'v' : numpy.array(numpy.ones((len(lines),nbins), dtype=float)*numpy.nan), - } - - data['en'][i] = burst_num - data['dt'][i] = sample_dt - data['nbins'][i] = nbins - data['z'][i] = z - data['u'][i] = u - data['v'][i] = v - i = i+1 - - return data - -def creator(platform_info, sensor_info, data): - # - # - title_str = sensor_info['description']+' at '+ platform_info['location'] - global_atts = { - 'title' : title_str, - 'institution' : 'Unversity of North Carolina at Chapel Hill (UNC-CH)', - 'institution_url' : 'http://nccoos.unc.edu', - 'institution_dods_url' : 'http://nccoos.unc.edu', - 'metadata_url' : 'http://nccoos.unc.edu', - 'references' : 'http://nccoos.unc.edu', - 'contact' : 'Sara Haines (haines@email.unc.edu)', - # - 'source' : 'fixed-profiler (acoustic doppler) observation', - 'history' : 'Data processed by NCCOOS', - 'comment' : 'File created using pycdf'+pycdfVersion()+' and numpy '+pycdfArrayPkg(), - # conventions - 'Conventions' : 'CF-1.0; SEACOOS-CDL-v2.0', - # SEACOOS CDL codes - 'format_category_code' : 'fixed-profiler', - 'institution_code' : platform_info['instituion'], - 'platform_code' : platform_info['id'], - 'package_code' : sensor_info['id'], - # institution specific - 'project' : 'North Carolina Coastal Ocean Observing System (NCCOOS)', - 'project_url' : 'http://nccoos.unc.edu', - # timeframe of data contained in file yyyy-mm-dd HH:MM:SS - 'start_date' : data['sample_dt'].strftime("%Y-%m-%d %H:%M:%S"), - 'end_date' : data['sample_dt'].strftime("%Y-%m-%d %H:%M:%S"), - 'release_date' : now.strftime("%Y-%m-%d %H:%M:%S"), - # - 'creation_date' : now.strftime("%Y-%m-%d %H:%M:%S"), - 'modification_date' : now.strftime("%Y-%m-%d %H:%M:%S"), - 'process_level' : 'level1', - # - # must type match to data (e.g. fillvalue is real if data is real) - '_FillValue' : -99999., - } - - var_atts = { - # coordinate variables - 'time' : {'short_name': 'time', - 'long_name': 'Time', - 'standard_name': 'time', - 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC - 'axis': 'T', - }, - 'lat' : {'short_name': 'lat', - 'long_name': 'Latitude', - 'standard_name': 'latitude', - 'reference':'geographic coordinates', - 'units': 'degrees_north', - 'valid_range':(-90.,90.), - 'axis': 'Y', - }, - 'lon' : {'short_name': 'lon', - 'long_name': 'Longtitude', - 'standard_name': 'longtitude', - 'reference':'geographic coordinates', - 'units': 'degrees_east', - 'valid_range':(-180.,180.), - 'axis': 'Y', - }, - 'z' : {'short_name': 'z', - 'long_name': 'Height', - 'standard_name': 'height', - 'reference':'zero at sea-surface', - 'units': 'm', - 'axis': 'Z', - }, - # data variables - 'u': {'long_name': 'East/West Component of Current', - 'standard_name': 'eastward_current', - 'units': 'm s-1', - 'reference': 'clockwise from True East', - }, - 'v': {'long_name': 'North/South Component of Current', - 'standard_name': 'northward_current', - 'units': 'm s-1', - 'reference': 'clockwise from True North', - }, - 'w': {'long_name': 'Upward/Downward Component of Current', - 'standard_name': 'upward_current', - 'units': 'm s-1', - 'positive': 'up', - }, - 'back_scatter':{'long_name': 'Backscatter', - 'standard_name': 'back_scatter', - 'units': 'decibels', - }, - 'wtemp': {'long_name': 'Water Temperature', - 'standard_name': 'water_temperature', - 'units': 'degrees Celsius', - }, - } - - - # integer values - ntime=NC.UNLIMITED - nlat=1 - nlon=1 - nz=sensor_info['nbins'] - - # dimension names use tuple so order of initialization is maintained - dimensions = ('ntime', 'nlat', 'nlon', 'nz') - - # using tuple of tuples so order of initialization is maintained - # using dict for attributes order of init not important - # use dimension names not values - # (varName, varType, (dimName1, [dimName2], ...)) - var_inits = ( - # coordinate variables - ('time', NC.INT, ('ntime',)), - ('lat', NC.FLOAT, ('nlat',)), - ('lon', NC.FLOAT, ('nlon',)), - ('z', NC.FLOAT, ('nz',)), - # data variables - ('u', NC.FLOAT, ('ntime', 'nz')), - ('v', NC.FLOAT, ('ntime', 'nz')), - ('w', NC.FLOAT, ('ntime', 'nz')), - ('back_scatter', NC.FLOAT, ('ntime', 'nz')), - ('wtemp', NC.FLOAT, ('ntime',)), - ) - - # var data - var_data = ( - ('lat', platform_info['lat']), - ('lon', platform_info['lon']), - ('z', []), - ('u', []), - ('v', []), - ('w', []), - ('back_scatter', []), - ('wtemp', []), - ) - - return (global_atts, dimensions, var_inits, var_data) - -def updater(platform_info, sensor_info, data): - # - global_atts = { - # timeframe of data contained in file yyyy-mm-dd HH:MM:SS - 'end_date' : data['sample_dt'].strftime("%Y-%m-%d %H:%M:%S"), - 'release_date' : now.strftime("%Y-%m-%d %H:%M:%S"), - # - 'creation_date' : now.strftime("%Y-%m-%d %H:%M:%S"), - 'modification_date' : now.strftime("%Y-%m-%d %H:%M:%S"), - } - # var data - var_data = ( - ('u', data['u']), - ('v', data['v']), - ('w', data['w']), - ('back_scatter', data['back_scatter']), - ('wtemp', data['wtemp']), - ) - return (global_atts, var_data) -# Index: raw2proc/trunk/raw2proc/raw2proc.py =================================================================== --- raw2proc/trunk/raw2proc/raw2proc.py (revision 180) +++ raw2proc/trunk/raw2proc/raw2proc.py (revision 208) @@ -1,4 +1,4 @@ #!/usr/bin/env python -# Last modified: Time-stamp: <2008-08-09 17:14:39 haines> +# Last modified: Time-stamp: <2008-09-08 11:38:54 haines> """Process raw data to monthly netCDF data files @@ -36,5 +36,5 @@ # define config file location to run under cron -defconfigs='/home/haines/nccoos/test/r2p' +defconfigs='/home/haines/nccoos/raw2proc' import numpy Index: raw2proc/trunk/raw2proc/stones_config_20080701.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/stones_config_20080701.py (revision 208) @@ -1,0 +1,39 @@ +platform_info = { + 'id' : 'stones', + 'location' : 'Stones Bay, New River, NC', + 'lat' : 34.5962, # degrees true (-) south, (+) north + 'lon' : 77.4120, # degrees true (-) west, (+) east + 'mvar' : -9.38, # degrees (-) west, (+) east + 'water_depth': 4.0, # nominal depth in meters (should be MSL) + 'institution' : 'nccoos', + # + 'config_start_date' : '2008-07-01 00:00:00', + 'config_end_date' : None, # None or yyyy-mm-dd HH:MM:SS + 'packages' : ('avp', 'met'), + } + +sensor_info = { + 'avp' : { 'id' : 'avp', + 'description' : 'Automated profiler data ctd and water quality', + 'raw_dir' : '/seacoos/data/nccoos/level0/stones/avp', + 'raw_file_glob' : '*.dat', + 'proc_dir' : '/seacoos/data/nccoos/level1/stones/avp', + 'process_module' : 'proc_avp_ysi_6600_v2', + 'utc_offset' : 5., # hours offset to Eastern Standard + 'bin_size' : 0.1, # meters + 'nbins' : 40, # for now, water_depth (MSL) divided by bin_size + # 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + # 'latest_vars' : ('time','lat','lon','z','wtemp','salin', 'turb', 'ph', 'chl', 'do'), + }, + 'met' : { 'id' : 'met', + 'description' : 'Wind Data at Automated Vertical Profiler Station', + 'raw_dir' : '/seacoos/data/nccoos/level0/stones/met/', + 'raw_file_glob' : '*.wnd', + 'proc_dir' : '/seacoos/data/nccoos/level1/stones/met/', + 'process_module' : 'proc_avp_ascii_met', + 'utc_offset' : 5., # hours offset to Eastern Standard + 'anemometer_height' : 2., # meters + # 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + # 'latest_vars' : ('time','lat','lon','z','u', 'v'), + }, + }