Index: raw2proc/trunk/raw2proc/jpier_config_20050425.py =================================================================== --- raw2proc/trunk/raw2proc/jpier_config_20050425.py (revision 179) +++ raw2proc/trunk/raw2proc/jpier_config_20050425.py (revision 180) @@ -8,5 +8,5 @@ # 'config_start_date' : '2005-04-25 00:00:00', - 'config_end_date' : '2008-04-15 00:00:00', # None or yyyy-mm-dd HH:MM:SS + 'config_end_date' : '2008-04-11 00:00:00', # None or yyyy-mm-dd HH:MM:SS 'packages' : ('met'), } Index: raw2proc/trunk/raw2proc/jpier_config_20080411.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/jpier_config_20080411.py (revision 180) @@ -1,0 +1,41 @@ +platform_info = { + 'id' : 'jpier', + 'location' : 'Jennettes Pier, Nags Head, NC', + 'lat' : 35.9101, # degrees true (-) south, (+) north + 'lon' : 75.5958, # degrees true (-) west, (+) east + 'mvar' : -10.83333, # degrees (-) west, (+) east + 'institution' : 'nccoos', + # + 'config_start_date' : '2008-04-11 00:00:00', + 'config_end_date' : '2008-07-21 23:00:00', # None or yyyy-mm-dd HH:MM:SS + 'packages' : ('adcp', 'adcpwaves'), + } +sensor_info = { + 'adcp' : { 'id' : 'adcp', + 'description' : 'Current profile data', + 'raw_dir' : '/seacoos/data/nccoos/level0/jpier/adcp_ascii', + 'raw_file_glob' : '*.wpa', + 'proc_dir' : '/seacoos/data/nccoos/level1/jpier/adcp', + 'process_module' : 'proc_nortek_wpa_adcp', + 'utc_offset' : 4, # hours offset to utc + 'nbins' : 34, + 'bin_size' : 0.5, # meters + 'transducer_ht' : 0.5, # meters above the bottom + 'blanking_ht' : 0.41, # meters above transducer + 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + 'latest_vars' : ('time','lat','lon','z','u','v'), + }, + 'adcpwaves' : {'id' : 'adcpwaves', + 'description' : 'Directional wave data', + 'raw_dir' : '/seacoos/data/nccoos/level0/jpier/adcp_ascii', + 'raw_file_glob' : '*.wds', + 'proc_dir' : '/seacoos/data/nccoos/level1/jpier/adcpwaves', + 'process_module' : 'proc_nortek_wds_dw', + 'utc_offset' : 4, # hours offset to utc + 'ndir' : 90., + 'nfreq' : 97., + 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + 'latest_vars' : ('time','lat','lon','z','Tp','Hs'), + }, + + } Index: raw2proc/trunk/raw2proc/jpier_config_20080722.py =================================================================== --- raw2proc/trunk/raw2proc/jpier_config_20080722.py (revision 179) +++ raw2proc/trunk/raw2proc/jpier_config_20080722.py (revision 180) @@ -9,14 +9,42 @@ 'config_start_date' : '2008-07-22 00:00:00', 'config_end_date' : None, # None or yyyy-mm-dd HH:MM:SS - 'packages' : ('met'), + 'packages' : ('met', 'adcp', 'adcpwaves'), } +# met back online and calibrated today, not sure about met/linux computer clock setting +# Nortek AWAC computer clock set to UTC sensor_info = { - 'met' : { 'id' : 'met', - 'description' : 'Met data', - 'raw_dir' : '/seacoos/data/nccoos/level0/jpier/met/', - 'utc_offset' : 4, # hours offset to utc - 'raw_file_glob' : '*.jpierMet.stats', - 'proc_dir' : '/seacoos/data/nccoos/level1/jpier/met/', - 'process_module' : 'proc_jpier_ascii_met', + 'met' : { 'id' : 'met', + 'description' : 'Met data', + 'raw_dir' : '/seacoos/data/nccoos/level0/jpier/met/', + 'utc_offset' : 4, # hours offset to utc + 'raw_file_glob' : '*.jpierMet.stats', + 'proc_dir' : '/seacoos/data/nccoos/level1/jpier/met/', + 'process_module' : 'proc_jpier_ascii_met', }, + 'adcp' : { 'id' : 'adcp', + 'description' : 'Current profile data', + 'raw_dir' : '/seacoos/data/nccoos/level0/jpier/adcp_ascii', + 'raw_file_glob' : '*.wpa', + 'proc_dir' : '/seacoos/data/nccoos/level1/jpier/adcp', + 'process_module' : 'proc_nortek_wpa_adcp', + 'utc_offset' : 0, # hours offset to utc + 'nbins' : 34, + 'bin_size' : 0.5, # meters + 'transducer_ht' : 0.5, # meters above the bottom + 'blanking_ht' : 0.41, # meters above transducer + 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + 'latest_vars' : ('time','lat','lon','z','u','v'), + }, + 'adcpwaves' : {'id' : 'adcpwaves', + 'description' : 'Directional wave data', + 'raw_dir' : '/seacoos/data/nccoos/level0/jpier/adcp_ascii', + 'raw_file_glob' : '*.wds', + 'proc_dir' : '/seacoos/data/nccoos/level1/jpier/adcpwaves', + 'process_module' : 'proc_nortek_wds_dw', + 'utc_offset' : 0., # hours offset to utc + 'ndir' : 90., + 'nfreq' : 97., + 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + 'latest_vars' : ('time','lat','lon','z','Tp','Hs'), + }, } Index: raw2proc/trunk/raw2proc/proc_nortek_wds_dw.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/proc_nortek_wds_dw.py (revision 180) @@ -1,0 +1,743 @@ +#!/usr/bin/env python +# Last modified: Time-stamp: <2008-07-31 17:18:05 haines> +""" +how to parse data, and assert what data and info goes into +creating and updating monthly netcdf files + +Nortek/AWAC processed adcp 2-D power spectrum (wds) as function of +frequency and direction + +parser : sample date and time and pressure from .wap, + energy spectrum m^2/Hz from .was, + normalized energy/deg from .wds + + based on George Voulgaris' matlab script (version 8, Feb 14, 2005, + polar_waves_cur_rdi.m) and additional parameters. +creator : lat, lon, z, time, freq, dir, Sxx(time, freq, dir), Sf(time, freq), + Stheta(time, dir), Stheta_swell(time, dir), Stheta_wind(time, dir), + Hs, Hs_swell, Hs_wind, + Tp, Tp_swell, Tp_wind, Tm, Tm_swell, Tm_wind, + Dp, Dp_swell, Dp_wind, Dm, Dm_swell, Dm_wind, + +updater : time, Sxx(time, freq, dir), Sf(time, freq), + Stheta(time, dir), Stheta_swell(time, dir), Stheta_wind(time, dir), + Hs, Hs_swell, Hs_wind, + Tp, Tp_swell, Tp_wind, Tm, Tm_swell, Tm_wind, + Dp, Dp_swell, Dp_wind, Dm, Dm_swell, Dm_wind, + + check that freq and dir have not changed from what is in current + NetCDF file + +Examples +-------- + +>> (parse, create, update) = load_processors(module_name_without_dot_py) +For example, +>> (parse, create, update) = load_processors('proc_rdi_logdata_adcp') +or +>> si = get_config(cn+'.sensor_info') +>> (parse, create, update) = load_processors(si['adcp']['proc_module']) + +Then use the generic name of processor to parse data, create or update +monthly output file + +>> lines = load_data(filename) +>> data = parse(platform_info, sensor_info, lines) +>> create(platform_info, sensor_info, data) +or +>> update(platform_info, sensor_info, data) + +""" + +from raw2proc import * +from procutil import * +from ncutil import * + +now_dt = datetime.utcnow() +now_dt.replace(microsecond=0) + +def parser(platform_info, sensor_info, lines): + """ + parse and assign wave spectra data from RDI ADCP Dspec + and compute wave statistics and parameters + + Notes + ----- + 1. adapted from polar_waves_cur_rdi.m (Version 8 - February 14, 2005) + by George Voulgaris + Coastal Processes & Sediment Dynamics Lab + Department of Geological Sciences + University of South Carolina, Columbia, SC 29205 + Email: gvoulgaris@geol.sc.edu + 2. This parser requires date/time be parsed from .wap for each + spectum sample in .wds, strip .wds on input filename and load and + parse .wap here. If .wap with same name not available, then use sample + per hour starting at time parsed from filename. + + 3. The .wds contains several bursts of full directional wave + spectrum. One for each hour in .wap. Each directional burst is + formatted in .wds file as each row is a one frequency, default + [0.02:0.01:0.99] or [0.02:0.01:0.49]. Each column is descretized + by 4 degrees 0:4:356 as degrees + + (faxed doc also states that freq's could be reported to .was file, + but I didn't find this so to be true) + + """ + + import numpy + from datetime import datetime + from time import strptime + + # get sample datetime from filename + fn = sensor_info['fn'] + sample_dt_start = filt_datetime(fn)[0] + + # try getting sample date/times from .wap + wap_fn = os.path.splitext(fn)[0] + ".wap" + if os.path.exists(wap_fn): + wap_lines = load_data(wap_fn) + + data = { + 'dt' : numpy.array(numpy.ones((len(wap_lines),), dtype=object)*numpy.nan), + 'time' : numpy.array(numpy.ones((len(wap_lines),), dtype=long)*numpy.nan), + 'press' : numpy.array(numpy.ones((len(wap_lines),), dtype=float)*numpy.nan), + } + i=0 + + for line in wap_lines: + # split line and parse float and integers + wap = [] + sw = re.split(' ', line) + for s in sw: + m = re.search(REAL_RE_STR, s) + if m: + wap.append(float(m.groups()[0])) + + # get sample datetime from data + sample_str = '%02d-%02d-%4d %02d:%02d:%02d' % tuple(wap[0:6]) + if sensor_info['utc_offset']: + sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%Y %H:%M:%S') + \ + timedelta(hours=sensor_info['utc_offset']) + else: + sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%Y %H:%M:%S') + + # these items can also be teased out of raw adcp but for now get from config file + # th = sensor_info['transducer_ht'] # Transducer height above bottom (meters) + + # pressure (dbar) converted to water depth + pressure = wap[17] # pressure (dbar) at tranducer height (?) + # water_depth = th + sw_dpth(pressure, lat) + + data['dt'][i] = sample_dt + data['time'][i] = dt2es(sample_dt) + data['press'][i] = pressure # dbar + i=i+1 + + else: + print "error: No corresponding .wap file" + print " .... skipping %s" % (fn,) + return data + + # assign specific fields + nbursts = len(data['dt']) + Df = 0.01 # (Hz) + f = numpy.arange(0.02, 0.99, Df) + nfreq = len(f) # Number of frequencies (no units) + + # Did we get the number of data rows that we expected? Should equal nfreq + n = int(len(lines)/nfreq) + if n != nbursts: + print "Number of data rows %d does not match expected number %d" % (n, nbursts) + print " .... skipping %s" % (fn,) + return data + + Dtheta = 1.0 # degrees + D = numpy.arange(0.0, 360.0, 4) + D = numpy.mod(D,360) + ndir = len(D) # Number of directions (no units) + + # now get power spectra from .was + was_fn = os.path.splitext(fn)[0] + ".was" + was_Sf = numpy.array(numpy.ones((nbursts,nfreq), dtype=float)*numpy.nan) + if os.path.exists(was_fn): + was_lines = load_data(was_fn) + + i=0 + # first line is freq label for each column start at [1] + for line in was_lines[1:]: + # split line and parse float and integers + was = [] + sw = re.split(' ', line) + for s in sw: + m = re.search(REAL_RE_STR, s) + if m: + was.append(float(m.groups()[0])) + + # just the frequencies we have in directional spectra [0:nfreq] + was_Sf[i] = was[0:nfreq] # (m^2/Hz) non-directional power spectrum for each sample time + + i=i+1 + + else: + print "error: No corresponding .was file" + print " .... skipping %s" % (fn,) + return data + + # add these keys, value pairs to dictionary "data" already setup after reading .wap + data['dirs'] = numpy.array(numpy.ones((ndir,), dtype=float)*numpy.nan) + data['freqs'] = numpy.array(numpy.ones((nfreq,), dtype=float)*numpy.nan) + data['Sxx'] = numpy.array(numpy.ones((nbursts,nfreq,ndir), dtype=float)*numpy.nan) + data['Sf'] = numpy.array(numpy.ones((nbursts,nfreq), dtype=float)*numpy.nan) + data['Stheta'] = numpy.array(numpy.ones((nbursts,ndir), dtype=float)*numpy.nan) + data['Stheta_swell'] = numpy.array(numpy.ones((nbursts,ndir), dtype=float)*numpy.nan) + data['Stheta_wind'] = numpy.array(numpy.ones((nbursts,ndir), dtype=float)*numpy.nan) + data['Hs'] = numpy.array(numpy.ones((nbursts,), dtype=float)*numpy.nan) + data['Hs_swell'] = numpy.array(numpy.ones((nbursts,), dtype=float)*numpy.nan) + data['Hs_wind'] = numpy.array(numpy.ones((nbursts,), dtype=float)*numpy.nan) + data['Tm'] = numpy.array(numpy.ones((nbursts,), dtype=float)*numpy.nan) + data['Tm_swell'] = numpy.array(numpy.ones((nbursts,), dtype=float)*numpy.nan) + data['Tm_wind'] = numpy.array(numpy.ones((nbursts,), dtype=float)*numpy.nan) + data['Tp'] = numpy.array(numpy.ones((nbursts,), dtype=float)*numpy.nan) + data['Tp_swell'] = numpy.array(numpy.ones((nbursts,), dtype=float)*numpy.nan) + data['Tp_wind'] = numpy.array(numpy.ones((nbursts,), dtype=float)*numpy.nan) + data['Dm'] = numpy.array(numpy.ones((nbursts,), dtype=float)*numpy.nan) + data['Dm_swell'] = numpy.array(numpy.ones((nbursts,), dtype=float)*numpy.nan) + data['Dm_wind'] = numpy.array(numpy.ones((nbursts,), dtype=float)*numpy.nan) + data['Dp'] = numpy.array(numpy.ones((nbursts,), dtype=float)*numpy.nan) + data['Dp_swell'] = numpy.array(numpy.ones((nbursts,), dtype=float)*numpy.nan) + data['Dp_wind'] = numpy.array(numpy.ones((nbursts,), dtype=float)*numpy.nan) + + # for each burst read nfreq lines + for j in range(nbursts): + + i = 0 + Sxx = numpy.array(numpy.ones((nfreq,ndir), dtype=float)*numpy.nan) + # each line is a freq, each column is a direction + for line in lines[j*nfreq:nfreq*(j+1)]: + wds = [] + # split line and parse float and integers + sw = re.split(' ', line) + for s in sw: + m = re.search(REAL_RE_STR, s) + if m: + wds.append(float(m.groups()[0])) + + # wds[] is in units of Normalized-Energy/degree from .wds + # use power (m^2/Hz) at same time and freq from .was to get units of m^2/Hz/deg + if len(wds) == ndir: + Sxx[i,:] = numpy.array(wds[:])*was_Sf[j,i] # cross spectrum as m^2/Hz/deg + i = i+1 + + # Did we get the number of data rows that we expected? Should equal nfreq + if i != nfreq: + print "Number of data rows %d does not match expected number %d" % (i, nfreq) + + + # NOTE make fupper location dependent?? (add to config_files??) + fupper = 0.65 # upper freq limit 0.65 Hz or wave periods less than T~1.538s + iswell = f<=1/10. # swell band for T>10s + iwind = (f>1/10.) * (f<=fupper) # wind band 1/fupper +""" +how to parse data, and assert what data and info goes into +creating and updating monthly netcdf files + +RDI/Wavesmon processed adcp current profile data + +parser : sample date and time, currents, water temperature, pressure and water_depth + +creator : lat, lon, z, time, ens, u, v, w, water_depth, water_temp (at tranducer depth), pressure +updator : time, ens, u, v, w, water_depth, water_temp (at tranducer depth), pressure + + +Examples +-------- + +>> (parse, create, update) = load_processors('proc_rdi_logdata_adcp') +or +>> si = get_config(cn+'.sensor_info') +>> (parse, create, update) = load_processors(si['adcp']['proc_module']) + +>> lines = load_data(filename) +>> data = parse(platform_info, sensor_info, lines) +>> create(platform_info, sensor_info, data) or +>> update(platform_info, sensor_info, data) + +""" + + +from raw2proc import * +from procutil import * +from ncutil import * + +import seawater + +now_dt = datetime.utcnow() +now_dt.replace(microsecond=0) + +def parser(platform_info, sensor_info, lines): + """ + parse and assign ocean profile current data from Nortek AWAC ADCP Data + + Notes + ----- + 1. This parser requires date/time be parsed from .wap for each to + get sig_wave_ht for determining depth of each bin and surface mask + and check time same as in .wpa file. + + 2. multiple profiles in one file separated by header w/time, + pitch, roll, heading, ducer pressure, bottom temp, top bin# + bottom bin# (??). The profile data is several lines one for + each bin. + + MM DD YYYY HH MM SS ERR STATUS BATT SNDSPD HDG PITCH ROLL PRESS WTEMP ?? ?? TBIN BBIN + 07 31 2008 23 54 00 0 48 18.2 1525.8 270.1 -2.4 0.2 10.503 21.64 0 0 3 34 + 1 0.9 0.071 24.04 0.029 0.065 -0.058 123 126 124 + 2 1.4 0.089 342.38 -0.027 0.085 -0.057 110 111 113 + 3 1.9 0.065 310.03 -0.050 0.042 -0.063 102 104 104 + 4 2.4 0.063 46.93 0.046 0.043 -0.045 93 95 99 + 5 2.9 0.049 355.33 -0.004 0.049 -0.047 87 89 92 + ... + NBIN DEPTH SPEED DIR U V W E1? E2? E3? + 32 16.4 0.184 331.76 -0.087 0.162 -0.162 26 25 27 + 33 16.9 0.137 288.70 -0.130 0.044 -0.181 26 24 26 + 34 17.4 0.070 32.78 0.038 0.059 -0.248 25 25 26 + + 3. not sure if depth column is hab or down from surface? + + + """ + + # get sample datetime from filename + fn = sensor_info['fn'] + sample_dt_start = filt_datetime(fn)[0] + + nbins = sensor_info['nbins'] # Number of bins in data + nbursts = len(lines)/(nbins+1) + + data = { + 'dt' : numpy.array(numpy.ones((nbursts,), dtype=object)*numpy.nan), + 'time' : numpy.array(numpy.ones((nbursts,), dtype=long)*numpy.nan), + 'z' : numpy.array(numpy.ones((nbins,), dtype=float)*numpy.nan), + 'u' : numpy.array(numpy.ones((nbursts,nbins), dtype=float)*numpy.nan), + 'v' : numpy.array(numpy.ones((nbursts,nbins), dtype=float)*numpy.nan), + 'w' : numpy.array(numpy.ones((nbursts,nbins), dtype=float)*numpy.nan), + 'e1' : numpy.array(numpy.ones((nbursts,nbins), dtype=int)*numpy.nan), + 'e2' : numpy.array(numpy.ones((nbursts,nbins), dtype=int)*numpy.nan), + 'e3' : numpy.array(numpy.ones((nbursts,nbins), dtype=int)*numpy.nan), + 'water_depth' : numpy.array(numpy.ones((nbursts), dtype=float)*numpy.nan), + 'water_temp' : numpy.array(numpy.ones((nbursts), dtype=float)*numpy.nan), + 'pressure' : numpy.array(numpy.ones((nbursts), dtype=float)*numpy.nan), + } + + # these items can also be teased out of raw adcp but for now get from config file + th = sensor_info['transducer_ht'] # Transducer height above bottom (meters) + bh = sensor_info['blanking_ht'] # Blanking height above Transducer (meters) + bin_size = sensor_info['bin_size'] # Bin Size (meters) + + # compute height for each bin above the bottom + bins = numpy.arange(1,nbins+1) + # bin_habs = (bins*bin_size+bin_size/2)+th+bh + bin_habs = (bins*bin_size+bin_size/2) + iaboveblank = bin_habs > th+bh+(bin_size) + + # current profile count + i = 0 + + wpa = [] + for line in lines: + wpa = [] + # split line and parse float and integers + sw = re.split(' ', line) + for s in sw: + m = re.search(REAL_RE_STR, s) + if m: + wpa.append(float(m.groups()[0])) + + if len(wpa)==19: + # get sample datetime from data + sample_str = '%02d-%02d-%4d %02d:%02d:%02d' % tuple(wpa[0:6]) + if sensor_info['utc_offset']: + sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%Y %H:%M:%S') + \ + timedelta(hours=sensor_info['utc_offset']) + else: + sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%Y %H:%M:%S') + + # these items can also be teased out of raw adcp but for now get from config file + # th = sensor_info['transducer_ht'] # Transducer height above bottom (meters) + + error_code = int(wpa[6]) + status_code = int(wpa[7]) + battery_voltage = wpa[8] # volts + sound_speed = wpa[9] # m/s + heading = wpa[10] # deg + pitch = wpa[11] # deg + roll = wpa[12] # deg + + pressure = wpa[13] # dbar + # pressure (dbar) converted to water depth + water_depth = th + seawater.depth(pressure, platform_info['lat']) # m + temperature = wpa[14] # deg C + + start_bin = int(wpa[17]) # first good bin from transducer (?) + wpa_nbins = int(wpa[18]) # Number of bins + # check this is same as in sensor_info + + # initialize for new profile + bin_hab = numpy.ones(nbins)*numpy.nan + spd = numpy.ones(nbins)*numpy.nan + dir = numpy.ones(nbins)*numpy.nan + u = numpy.ones(nbins)*numpy.nan + v = numpy.ones(nbins)*numpy.nan + w = numpy.ones(nbins)*numpy.nan + e1 = numpy.array(numpy.ones((nbins), dtype=int)*numpy.nan) + e2 = numpy.array(numpy.ones((nbins), dtype=int)*numpy.nan) + e3 = numpy.array(numpy.ones((nbins), dtype=int)*numpy.nan) + + elif len(wpa)==10: + # current profile data at each bin + bin_number = wpa[0] + j = wpa[0]-1 + bin_hab[j] = wpa[1] + + spd[j] = wpa[2] # m/s + dir[j] = wpa[3] # deg N + + u[j] = wpa[4] # m/s + v[j] = wpa[5] # m/s + w[j] = wpa[6] # m/s + + e1[j] = int(wpa[7]) # echo dB ?? + e2[j] = int(wpa[8]) # + e3[j] = int(wpa[9]) # + + # ibad = (current_spd==-32768) | (current_dir==-32768) + # current_spd[ibad] = numpy.nan + # current_dir[ibad] = numpy.nan + + # if done reading profile, just read data for last bin + if bin_number==nbins: + # compute water mask + # Using George Voulgaris' method based on water depth + # minus half of the significant wave height (Hs) + # and computed habs + # if positive is up, what's less than zero depth? + bin_depths = bin_habs-(water_depth) + iwater = bin_depths+bin_size/2 < 0 + iwater = iwater*iaboveblank + + # alternatively use nominal water depth (MSL) averaged from full pressure record + # this should be checked/recalulated every so often + # MSL = sensor_info['mean_sea_level'] # Mean sea level at station (meters) or nominal water depth + # MSL = 12. # **** first guess is 12 meters for jpier + # z = bin_habs-MSL + z = bin_habs + + data['dt'][i] = sample_dt # sample datetime + data['time'][i] = dt2es(sample_dt) # sample time in epoch seconds + data['z'] = z + data['water_depth'][i] = water_depth + data['water_temp'][i] = temperature + data['pressure'][i] = pressure + + data['u'][i][iwater] = u[iwater] + data['v'][i][iwater] = v[iwater] + data['w'][i][iwater] = w[iwater] + + data['e1'][i] = e1 + data['e2'][i] = e2 + data['e3'][i] = e3 + + # ready for next burst + i = i+1 + # if j+1==nbins + # if len(wpa)==19 elif ==10 + # for line + + return data + + +def creator(platform_info, sensor_info, data): + # + # + title_str = sensor_info['description']+' at '+ platform_info['location'] + global_atts = { + 'title' : title_str, + 'institution' : 'Unversity of North Carolina at Chapel Hill (UNC-CH)', + 'institution_url' : 'http://nccoos.unc.edu', + 'institution_dods_url' : 'http://nccoos.unc.edu', + 'metadata_url' : 'http://nccoos.unc.edu', + 'references' : 'http://nccoos.unc.edu', + 'contact' : 'Sara Haines (haines@email.unc.edu)', + # + 'source' : 'fixed-profiler (acoustic doppler) observation', + 'history' : 'raw2proc using ' + sensor_info['process_module'], + 'comment' : 'File created using pycdf'+pycdfVersion()+' and numpy '+pycdfArrayPkg(), + # conventions + 'Conventions' : 'CF-1.0; SEACOOS-CDL-v2.0', + # SEACOOS CDL codes + 'format_category_code' : 'fixed-profiler', + 'institution_code' : platform_info['institution'], + 'platform_code' : platform_info['id'], + 'package_code' : sensor_info['id'], + # institution specific + 'project' : 'North Carolina Coastal Ocean Observing System (NCCOOS)', + 'project_url' : 'http://nccoos.unc.edu', + # timeframe of data contained in file yyyy-mm-dd HH:MM:SS + # first date in monthly file + 'start_date' : data['dt'][0].strftime("%Y-%m-%d %H:%M:%S"), + # last date in monthly file + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + # + 'creation_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + 'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + 'process_level' : 'level1', + # + # must type match to data (e.g. fillvalue is real if data is real) + '_FillValue' : -99999., + } + + var_atts = { + # coordinate variables + 'time' : {'short_name': 'time', + 'long_name': 'Time', + 'standard_name': 'time', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + 'axis': 'T', + }, + 'lat' : {'short_name': 'lat', + 'long_name': 'Latitude', + 'standard_name': 'latitude', + 'reference':'geographic coordinates', + 'units': 'degrees_north', + 'valid_range':(-90.,90.), + 'axis': 'Y', + }, + 'lon' : {'short_name': 'lon', + 'long_name': 'Longtitude', + 'standard_name': 'longtitude', + 'reference':'geographic coordinates', + 'units': 'degrees_east', + 'valid_range':(-180.,180.), + 'axis': 'Y', + }, + 'z' : {'short_name': 'z', + 'long_name': 'Height', + 'standard_name': 'height', + 'reference':'zero at sea-surface', + 'positive' : 'up', + 'units': 'm', + 'axis': 'Z', + }, + # data variables + 'u': {'short_name' : 'u', + 'long_name': 'East/West Component of Current', + 'standard_name': 'eastward_current', + 'units': 'm s-1', + 'reference': 'clockwise from True East', + }, + 'v': {'short_name' : 'v', + 'long_name': 'North/South Component of Current', + 'standard_name': 'northward_current', + 'units': 'm s-1', + 'reference': 'clockwise from True North', + }, + 'w': {'short_name' : 'w', + 'long_name': 'Vertical Component of Current', + 'standard_name': 'upward_current', + 'units': 'm s-1', + 'reference': 'clockwise from True North', + }, + 'e1': {'short_name' : 'e1', + 'long_name': 'Echo Beam 1 (??)', + 'standard_name': 'beam_echo', + 'units': 'dB', + }, + 'e2': {'short_name' : 'e2', + 'long_name': 'Echo Beam 2 (??)', + 'standard_name': 'beam_echo', + 'units': 'dB', + }, + 'e3': {'short_name' : 'e3', + 'long_name': 'Echo Beam 3 (??)', + 'standard_name': 'beam_echo', + 'units': 'dB', + }, + 'water_depth': {'short_name': 'wd', + 'long_name': 'Water Depth', + 'standard_name': 'water_depth', + 'units': 'm', + }, + 'pressure': {'short_name': 'p', + 'long_name': 'Pressure', + 'standard_name': 'pressure', + 'units': 'dbar', + }, + 'water_temp': {'short_name': 'wtemp', + 'long_name': 'Water Temperature at Transducer', + 'standard_name': 'water_temperature', + 'units': 'deg C', + }, + } + + + # dimension names use tuple so order of initialization is maintained + dim_inits = ( + ('ntime', NC.UNLIMITED), + ('nlat', 1), + ('nlon', 1), + ('nz', sensor_info['nbins']) + ) + + # using tuple of tuples so order of initialization is maintained + # using dict for attributes order of init not important + # use dimension names not values + # (varName, varType, (dimName1, [dimName2], ...)) + var_inits = ( + # coordinate variables + ('time', NC.INT, ('ntime',)), + ('lat', NC.FLOAT, ('nlat',)), + ('lon', NC.FLOAT, ('nlon',)), + ('z', NC.FLOAT, ('nz',)), + # data variables + ('u', NC.FLOAT, ('ntime', 'nz')), + ('v', NC.FLOAT, ('ntime', 'nz')), + ('w', NC.FLOAT, ('ntime', 'nz')), + ('e1', NC.INT, ('ntime', 'nz')), + ('e2', NC.INT, ('ntime', 'nz')), + ('e3', NC.INT, ('ntime', 'nz')), + ('water_depth', NC.FLOAT, ('ntime',)), + ('pressure', NC.FLOAT, ('ntime',)), + ('water_temp', NC.FLOAT, ('ntime',)), + ) + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # var data + var_data = ( + ('lat', platform_info['lat']), + ('lon', platform_info['lon']), + ('z', data['z']), + # + ('time', data['time'][i]), + ('u', data['u'][i]), + ('v', data['v'][i]), + ('w', data['w'][i]), + ('e1', data['e1'][i]), + ('e2', data['e2'][i]), + ('e3', data['e3'][i]), + ('water_depth', data['water_depth'][i]), + ('pressure', data['pressure'][i]), + ('water_temp', data['water_temp'][i]), + ) + + return (global_atts, var_atts, dim_inits, var_inits, var_data) + +def updater(platform_info, sensor_info, data): + # + global_atts = { + # update times of data contained in file (yyyy-mm-dd HH:MM:SS) + # last date in monthly file + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + # + 'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + } + + # data variables + # update any variable attributes like range, min, max + var_atts = {} + # var_atts = { + # 'u': {'max': max(data.u), + # 'min': min(data.v), + # }, + # 'v': {'max': max(data.u), + # 'min': min(data.v), + # }, + # } + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # data + var_data = ( + ('time', data['time'][i]), + ('u', data['u'][i]), + ('v', data['v'][i]), + ('w', data['w'][i]), + ('e1', data['e1'][i]), + ('e2', data['e2'][i]), + ('e3', data['e3'][i]), + ('water_depth', data['water_depth'][i]), + ('pressure', data['pressure'][i]), + ('water_temp', data['water_temp'][i]), + ) + + return (global_atts, var_atts, var_data) +# Index: raw2proc/trunk/raw2proc/raw2proc.py =================================================================== --- raw2proc/trunk/raw2proc/raw2proc.py (revision 179) +++ raw2proc/trunk/raw2proc/raw2proc.py (revision 180) @@ -1,4 +1,4 @@ #!/usr/bin/env python -# Last modified: Time-stamp: <2008-07-23 15:43:33 haines> +# Last modified: Time-stamp: <2008-08-09 17:14:39 haines> """Process raw data to monthly netCDF data files @@ -30,6 +30,11 @@ import re +# for production use: +# defconfigs='/home/haines/nccoos/raw2proc' +# for testing use: +# defconfigs='/home/haines/nccoos/test/r2p' + # define config file location to run under cron -defconfigs='/home/haines/nccoos/raw2proc' +defconfigs='/home/haines/nccoos/test/r2p' import numpy @@ -90,4 +95,5 @@ # list of config files based on platform configs = glob.glob(os.path.join(config_dir, platform + '_config_*.py')) + configs.sort() now_dt = datetime.utcnow() now_dt.replace(microsecond=0) @@ -358,5 +364,13 @@ if index==0 and os.path.exists(ofn): os.remove(ofn) - # + # this added just in case data repeated in data files + if os.path.exists(ofn): + # get last dt from current month file + (es, units) = nc_get_time(ofn) + last_dt = es2dt(es[-1]) + # if older than month_start_dt use it instead to only process newest data + if last_dt>=month_start_dt: + si['proc_start_dt'] = last_dt + if raw_files: process(pi, si, raw_files, yyyy_mm)