#!/usr/bin/env python # Last modified: Time-stamp: <2010-12-09 16:13:21 haines> """ how to parse data, and assert what data and info goes into creating and updating monthly netcdf files RDI/Wavesmon processed adcp 2-D power spectrum (Dspec) as function of frequency and direction parser : sample date and time from file name, compute wave summary based on George Voulgaris' matlab script (version 8, Feb 14, 2005, polar_waves_cur_rdi.m) and additional parameters. creator : lat, lon, z, time, freq, dir, Sxx(time, freq, dir), Sf(time, freq), Stheta(time, dir), Stheta_swell(time, dir), Stheta_wind(time, dir), Hs, Hs_swell, Hs_wind, Tp, Tp_swell, Tp_wind, Tm, Tm_swell, Tm_wind, Dp, Dp_swell, Dp_wind, Dm, Dm_swell, Dm_wind, updater : time, Sxx(time, freq, dir), Sf(time, freq), Stheta(time, dir), Stheta_swell(time, dir), Stheta_wind(time, dir), Hs, Hs_swell, Hs_wind, Tp, Tp_swell, Tp_wind, Tm, Tm_swell, Tm_wind, Dp, Dp_swell, Dp_wind, Dm, Dm_swell, Dm_wind, check that freq and dir have not changed from what is in current NetCDF file Examples -------- >> (parse, create, update) = load_processors(module_name_without_dot_py) For example, >> (parse, create, update) = load_processors('proc_rdi_logdata_adcp') or >> si = get_config(cn+'.sensor_info') >> (parse, create, update) = load_processors(si['adcp']['proc_module']) Then use the generic name of processor to parse data, create or update monthly output file >> lines = load_data(filename) >> data = parse(platform_info, sensor_info, lines) >> create(platform_info, sensor_info, data) or >> update(platform_info, sensor_info, data) """ from raw2proc import * from procutil import * from ncutil import * now_dt = datetime.utcnow() now_dt.replace(microsecond=0) def parser(platform_info, sensor_info, lines): """ parse and assign wave spectra data from RDI ADCP Dspec and compute wave statistics and parameters Notes ----- 1. adapted from polar_waves_cur_rdi.m (Version 8 - February 14, 2005) by George Voulgaris Coastal Processes & Sediment Dynamics Lab Department of Geological Sciences University of South Carolina, Columbia, SC 29205 Email: gvoulgaris@geol.sc.edu 1. should only be one line in each file of comma-delimited data """ import numpy from datetime import datetime from time import strptime # get sample datetime from filename fn = sensor_info['fn'] # print " ... %s" % (fn,) if sensor_info['utc_offset']: sample_dt = filt_datetime(fn) + \ timedelta(hours=sensor_info['utc_offset']) else: sample_dt = filt_datetime(fn) # extract header (first 6 lines) rdi = [] for line in [lines[2], lines[4], lines[5]]: # split line and parse float and integers sw = re.split(' ', line) for s in sw: m = re.search(REAL_RE_STR, s) if m: rdi.append(float(m.groups()[0])) # assign specific fields n = len(rdi) ndir = float(rdi[0]) # Number of directions (no units) nfreq = float(rdi[1]) # Number of frequencies (no units) freq_bw = float(rdi[2]) # Frequency bandwidth (Hz) Do = float(rdi[3]) # Starting direction (degrees from True North) if ndir!=sensor_info['ndir']: print 'Number of direction bins reported in data ('+ \ str(ndir)+') does not match config number ('+ \ str(sensor_info['ndir'])+'). \nCheck for change at sensor.' if nfreq!=sensor_info['nfreq']: print 'Number of frequencies reported in data ('+ \ str(nfreq)+') does not match config number ('+ \ str(sensor_info['nfreq'])+'). \nCheck for change at sensor. ' Dtheta = 360./ndir D = Do + numpy.arange(ndir)*Dtheta D = numpy.mod(D,360) Df = 1./nfreq f = numpy.arange(1,nfreq+1)*Df # some data checks if Df != freq_bw: # frequency resolution should be the same as freq_bw print "Df (%f) not equal to freq_bw (%f)" % (Df, freq_bw) # set up dict of data data = { 'dt' : numpy.array(numpy.ones((1,), dtype=object)*numpy.nan), 'time' : numpy.array(numpy.ones((1,), dtype=long)*numpy.nan), 'dirs' : numpy.array(numpy.ones((ndir,), dtype=float)*numpy.nan), 'freqs' : numpy.array(numpy.ones((nfreq,), dtype=float)*numpy.nan), 'Sxx' : numpy.array(numpy.ones((1,nfreq,ndir), dtype=float)*numpy.nan), 'Sf' : numpy.array(numpy.ones((1,nfreq), dtype=float)*numpy.nan), 'Stheta' : numpy.array(numpy.ones((1,ndir), dtype=float)*numpy.nan), 'Stheta_swell' : numpy.array(numpy.ones((1,ndir), dtype=float)*numpy.nan), 'Stheta_wind' : numpy.array(numpy.ones((1,ndir), dtype=float)*numpy.nan), 'Hs' : numpy.array(numpy.ones((1,), dtype=float)*numpy.nan), 'Hs_swell' : numpy.array(numpy.ones((1,), dtype=float)*numpy.nan), 'Hs_wind' : numpy.array(numpy.ones((1,), dtype=float)*numpy.nan), 'Tm' : numpy.array(numpy.ones((1,), dtype=float)*numpy.nan), 'Tm_swell' : numpy.array(numpy.ones((1,), dtype=float)*numpy.nan), 'Tm_wind' : numpy.array(numpy.ones((1,), dtype=float)*numpy.nan), 'Tp' : numpy.array(numpy.ones((1,), dtype=float)*numpy.nan), 'Tp_swell' : numpy.array(numpy.ones((1,), dtype=float)*numpy.nan), 'Tp_wind' : numpy.array(numpy.ones((1,), dtype=float)*numpy.nan), 'Dm' : numpy.array(numpy.ones((1,), dtype=float)*numpy.nan), 'Dm_swell' : numpy.array(numpy.ones((1,), dtype=float)*numpy.nan), 'Dm_wind' : numpy.array(numpy.ones((1,), dtype=float)*numpy.nan), 'Dp' : numpy.array(numpy.ones((1,), dtype=float)*numpy.nan), 'Dp_swell' : numpy.array(numpy.ones((1,), dtype=float)*numpy.nan), 'Dp_wind' : numpy.array(numpy.ones((1,), dtype=float)*numpy.nan), } # throw a dummy datetime into dt so we can return data with no data # if we encounter an unrecoverable error while parsing the file data['dt'][:] = datetime(1970,1,1,0,0,0) j = 0 Sxx = numpy.array(numpy.ones((nfreq,ndir), dtype=float)*numpy.nan) # each line is a freq, each column is a direction for line in lines[6:]: rdi = [] # split line and parse float and integers sw = re.split(' ', line) for s in sw: m = re.search(REAL_RE_STR, s) if m: rdi.append(float(m.groups()[0])) if len(rdi) == ndir: Sxx[j][:] = numpy.array(rdi[:]) # cross spectrum as mm^2/Hz/deg j = j+1 # Did we get the number of data rows that we expected? Should equal nfreq if j != nfreq: print "Number of data rows %d does not match expected number %d" % (j, nfreq) print " .... skipping %s" % (fn,) return data Sxx = Sxx/360./1000./1000. # convert cross spectrum to units of m^2/Hz/deg # NOTE make fupper location dependent?? (add to config_files??) fupper = 0.65 # upper freq limit 0.65 Hz or wave periods less than T~1.538s iswell = f<=1/10. # swell band for T>10s iwind = (f>1/10.) * (f<=fupper) # wind band 1/fupper