Index: raw2proc/trunk/raw2proc/bogue_config_20060918.py =================================================================== --- raw2proc/trunk/raw2proc/bogue_config_20060918.py (revision 212) +++ raw2proc/trunk/raw2proc/bogue_config_20060918.py (revision 213) @@ -5,5 +5,6 @@ 'lon' : -77.034131, # degrees true (-) west, (+) east 'mvar' : -9.7, # degrees (-) west, (+) east - 'water_depth' : 8.14, # meters + 'mean_water_depth' : -8.14, # meters (-) down, (+) up + 'mean_water_depth_time_period' : 'June 2006 - June 2008', 'institution' : 'nccoos', # Index: raw2proc/trunk/raw2proc/bogue_config_20070224.py =================================================================== --- raw2proc/trunk/raw2proc/bogue_config_20070224.py (revision 212) +++ raw2proc/trunk/raw2proc/bogue_config_20070224.py (revision 213) @@ -5,5 +5,6 @@ 'lon' : -77.034131, # degrees true (-) west, (+) east 'mvar' : -9.7, # degrees (-) west, (+) east - 'water_depth' : 8.14, # meters + 'mean_water_depth' : -8.14, # meters (-) down, (+) up + 'mean_water_depth_time_period' : 'June 2006 - June 2008', 'institution' : 'nccoos', # Index: raw2proc/trunk/raw2proc/bogue_config_20080430.py =================================================================== --- raw2proc/trunk/raw2proc/bogue_config_20080430.py (revision 212) +++ raw2proc/trunk/raw2proc/bogue_config_20080430.py (revision 213) @@ -5,5 +5,6 @@ 'lon' : -77.034131, # degrees true (-) west, (+) east 'mvar' : -9.7, # degrees (-) west, (+) east - 'water_depth' : 8.14, # meters + 'mean_water_depth' : -8.14, # meters (-) down, (+) up + 'mean_water_depth_time_period' : 'June 2006 - June 2008', 'institution' : 'nccoos', # Index: raw2proc/trunk/raw2proc/jpier_config_20050425.py =================================================================== --- raw2proc/trunk/raw2proc/jpier_config_20050425.py (revision 212) +++ raw2proc/trunk/raw2proc/jpier_config_20050425.py (revision 213) @@ -5,5 +5,6 @@ 'lon' : -75.5958, # degrees true (-) west, (+) east 'mvar' : -10.83333, # degrees (-) west, (+) east - 'water_depth' : 11.38, # meters + 'mean_water_depth' : -11.38, # meters (-) down, (+) up + 'mean_water_depth_time_period' : 'May 2008 - Oct 2008', 'institution' : 'nccoos', # Index: raw2proc/trunk/raw2proc/jpier_config_20080411.py =================================================================== --- raw2proc/trunk/raw2proc/jpier_config_20080411.py (revision 212) +++ raw2proc/trunk/raw2proc/jpier_config_20080411.py (revision 213) @@ -5,5 +5,6 @@ 'lon' : -75.5958, # degrees true (-) west, (+) east 'mvar' : -10.83333, # degrees (-) west, (+) east - 'water_depth' : 11.38, # meters + 'mean_water_depth' : -11.38, # meters (-) down, (+) up + 'mean_water_depth_time_period' : 'May 2008 - Oct 2008', 'institution' : 'nccoos', # @@ -23,5 +24,6 @@ 'bin_size' : 0.5, # meters 'transducer_ht' : 0.5, # meters above the bottom - 'blanking_ht' : 0.41, # meters above transducer + # 'blanking_ht' : 0.41, # meters above transducer + 'blanking_ht' : 1.9, # meters above transducer 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', 'latest_vars' : ('time','lat','lon','z','u','v'), Index: raw2proc/trunk/raw2proc/jpier_config_20080722.py =================================================================== --- raw2proc/trunk/raw2proc/jpier_config_20080722.py (revision 212) +++ raw2proc/trunk/raw2proc/jpier_config_20080722.py (revision 213) @@ -5,5 +5,6 @@ 'lon' : -75.5958, # degrees true (-) west, (+) east 'mvar' : -10.83333, # degrees (-) west, (+) east - 'water_depth' : 11.38, # meters + 'mean_water_depth' : -11.38, # meters (-) down, (+) up + 'mean_water_depth_time_period' : 'May 2008 - Oct 2008', 'institution' : 'nccoos', # @@ -24,5 +25,5 @@ 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', 'latest_vars' : ('time','lat','lon','z','wspd','wdir', 'air_temp', 'humidity', 'air_pressure'), - }, + }, 'adcp' : { 'id' : 'adcp', 'description' : 'Current profile data', @@ -35,7 +36,8 @@ 'bin_size' : 0.5, # meters 'transducer_ht' : 0.5, # meters above the bottom - 'blanking_ht' : 0.41, # meters above transducer + # 'blanking_ht' : 0.41, # meters above transducer + 'blanking_ht' : 1.9, # meters above transducer 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', - 'latest_vars' : ('time','lat','lon','z','u','v'), + 'latest_vars' : ('time','lat','lon','z','u','v','wd','wl'), }, 'adcpwaves' : {'id' : 'adcpwaves', Index: raw2proc/trunk/raw2proc/lsrb_config_20050623.py =================================================================== --- raw2proc/trunk/raw2proc/lsrb_config_20050623.py (revision 212) +++ raw2proc/trunk/raw2proc/lsrb_config_20050623.py (revision 213) @@ -5,5 +5,6 @@ 'lon' : -76.42, # degrees true (-) west, (+) east 'mvar' : -9.967, # degrees (-) west, (+) east - 'water_depth' : 25.17, # meters (ave from start to end of deployment) + 'mean_water_depth' : -25.17, # meters (-) down, (+) up + 'mean_water_depth_time_period' : 'June 2005 - Sept 2005', 'institution' : 'nccoos', # @@ -27,8 +28,8 @@ 'adcpwaves' : {'id' : 'adcpwaves', 'description' : 'Directional wave data', - 'raw_dir' : '/seacoos/data/nccoos/level0/lsrb/adcp_cSpecData', - 'raw_file_glob' : 'DSpec*', + 'raw_dir' : '/seacoos/data/nccoos/level0/lsrb/adcp_bLogData/2005_06', + 'raw_file_glob' : 'lsrb_LogData_20050623*', 'proc_dir' : '/seacoos/data/nccoos/level1/lsrb/adcpwaves', - 'process_module' : 'proc_rdi_dspec_dw', + 'process_module' : 'proc_rdi_logdata_dw', 'utc_offset' : 4, # hours offset to utc 'ndir' : 90., Index: raw2proc/trunk/raw2proc/lsrb_config_20051031.py =================================================================== --- raw2proc/trunk/raw2proc/lsrb_config_20051031.py (revision 212) +++ raw2proc/trunk/raw2proc/lsrb_config_20051031.py (revision 213) @@ -5,5 +5,6 @@ 'lon' : -76.42, # degrees true (-) west, (+) east 'mvar' : -9.967, # degrees (-) west, (+) east - 'water_depth' : 25.86, # meters (ave from start to end of deployment) + 'mean_water_depth' : -25.86, # meters (-) down, (+) up + 'mean_water_depth_time_period' : 'Oct 2005 - Feb 2006', 'institution' : 'nccoos', # @@ -27,8 +28,8 @@ 'adcpwaves' : {'id' : 'adcpwaves', 'description' : 'Directional wave data', - 'raw_dir' : '/seacoos/data/nccoos/level0/lsrb/adcp_cSpecData', - 'raw_file_glob' : 'DSpec*', + 'raw_dir' : '/seacoos/data/nccoos/level0/lsrb/adcp_bLogData/2005_10', + 'raw_file_glob' : 'lsrb_LogData*', 'proc_dir' : '/seacoos/data/nccoos/level1/lsrb/adcpwaves', - 'process_module' : 'proc_rdi_dspec_dw', + 'process_module' : 'proc_rdi_logdata_dw', 'utc_offset' : 4, # hours offset to utc 'ndir' : 90., Index: raw2proc/trunk/raw2proc/lsrb_config_20060622.py =================================================================== --- raw2proc/trunk/raw2proc/lsrb_config_20060622.py (revision 212) +++ raw2proc/trunk/raw2proc/lsrb_config_20060622.py (revision 213) @@ -5,5 +5,6 @@ 'lon' : -76.42, # degrees true (-) west, (+) east 'mvar' : -9.967, # degrees (-) west, (+) east - 'water_depth' : 26.26, # meters (ave from start to end of deployment) + 'mean_water_depth' : -26.26, # meters (-) down, (+) up + 'mean_water_depth_time_period' : 'June 2006 - Oct 2006', 'institution' : 'nccoos', # @@ -27,8 +28,8 @@ 'adcpwaves' : {'id' : 'adcpwaves', 'description' : 'Directional wave data', - 'raw_dir' : '/seacoos/data/nccoos/level0/lsrb/adcp_cSpecData', - 'raw_file_glob' : 'DSpec*', + 'raw_dir' : '/seacoos/data/nccoos/level0/lsrb/adcp_bLogData/2006_06', + 'raw_file_glob' : 'lsrb_LogData*', 'proc_dir' : '/seacoos/data/nccoos/level1/lsrb/adcpwaves', - 'process_module' : 'proc_rdi_dspec_dw', + 'process_module' : 'proc_rdi_logdata_dw', 'utc_offset' : 4, # hours offset to utc 'ndir' : 90., Index: raw2proc/trunk/raw2proc/lsrb_config_20070612.py =================================================================== --- raw2proc/trunk/raw2proc/lsrb_config_20070612.py (revision 212) +++ raw2proc/trunk/raw2proc/lsrb_config_20070612.py (revision 213) @@ -5,5 +5,6 @@ 'lon' : -76.42, # degrees true (-) west, (+) east 'mvar' : -9.967, # degrees (-) west, (+) east - 'water_depth' : 26.5, # meters (ave from start to end of deployment) + 'mean_water_depth' : -26.98, # meters (-) down, (+) up + 'mean_water_depth_time_period' : 'Aug 2007 - Sep 2007', 'institution' : 'nccoos', # @@ -27,8 +28,8 @@ 'adcpwaves' : {'id' : 'adcpwaves', 'description' : 'Directional wave data', - 'raw_dir' : '/seacoos/data/nccoos/level0/lsrb/adcp_cSpecData', - 'raw_file_glob' : 'DSpec*', + 'raw_dir' : '/seacoos/data/nccoos/level0/lsrb/adcp_bLogData/2007_06', + 'raw_file_glob' : 'lsrb_LogData*', 'proc_dir' : '/seacoos/data/nccoos/level1/lsrb/adcpwaves', - 'process_module' : 'proc_rdi_dspec_dw', + 'process_module' : 'proc_rdi_logdata_dw', 'utc_offset' : 4, # hours offset to utc 'ndir' : 90., Index: raw2proc/trunk/raw2proc/ncutil.py =================================================================== --- raw2proc/trunk/raw2proc/ncutil.py (revision 210) +++ raw2proc/trunk/raw2proc/ncutil.py (revision 213) @@ -1,4 +1,4 @@ #!/usr/bin/env python -# Last modified: Time-stamp: <2008-09-25 14:01:50 haines> +# Last modified: Time-stamp: <2008-10-23 11:31:24 haines> """ Create, update and load utilities for netcdf files @@ -124,5 +124,5 @@ def nc_get_time(ncFile): - """get time array from file """ + """Get time array from file """ try: nc = CDF(ncFile) @@ -140,5 +140,5 @@ def nc_find_record_vars(ncFile): - """find which variable are record variables""" + """Find which variable are record variables""" try: nc = CDF(ncFile) @@ -151,4 +151,51 @@ print "CDFError:", msg + +def nc_replace_fillvalue(ncFile, newfillvalue=-99999.0): + """ + Replaces any occurrence of old _FillValue with new one + + This function is useful for replacing the _FillValue global + attribute and then searching the data for the old value and + replacing it with the new one. + + :Parameters: + ncFile : string + Path and name of file to create + + :Other Parameters: + newfillvalue : type match to data (generally float) + By default is -99999.0 + + """ + try: + nc = CDF(ncFile, NC.WRITE) + nc.automode() + oldfillvalue = nc._FillValue + nc._FillValue = newfillvalue + for v in nc.variables().keys(): + vd = nc.var(v)[:] + if numpy.isnan(oldfillvalue): + idx = numpy.isnan(vd) + else: + idx = vd == oldfillvalue + if idx.any(): + vd[idx] = nc._FillValue + nc.var(v)[:] = vd + nc.close() + except CDFError, msg: + print "CDFError:", msg + +def nc_rename_dimension(ncFile, oldname, newname): + """ Rename dimension name """ + try: + nc = CDF(ncFile, NC.WRITE) + nc.definemode() + for d in nc.dimensions().keys(): + if d==oldname: nc.dim(d).rename(newname) + nc.close() + except CDFError, msg: + print "CDFError:", msg + def nc_load(ncFile, varsLoad='all', nameType='variable_name', Index: raw2proc/trunk/raw2proc/proc_nortek_wpa_adcp.py =================================================================== --- raw2proc/trunk/raw2proc/proc_nortek_wpa_adcp.py (revision 211) +++ raw2proc/trunk/raw2proc/proc_nortek_wpa_adcp.py (revision 213) @@ -1,4 +1,4 @@ #!/usr/bin/env python -# Last modified: Time-stamp: <2008-10-01 12:47:16 haines> +# Last modified: Time-stamp: <2008-10-16 17:02:54 haines> """ how to parse data, and assert what data and info goes into @@ -88,5 +88,6 @@ 'e2' : numpy.array(numpy.ones((nbursts,nbins), dtype=int)*numpy.nan), 'e3' : numpy.array(numpy.ones((nbursts,nbins), dtype=int)*numpy.nan), - 'water_depth' : numpy.array(numpy.ones((nbursts), dtype=float)*numpy.nan), + 'wd' : numpy.array(numpy.ones((nbursts), dtype=float)*numpy.nan), + 'wl' : numpy.array(numpy.ones((nbursts), dtype=float)*numpy.nan), 'water_temp' : numpy.array(numpy.ones((nbursts), dtype=float)*numpy.nan), 'pressure' : numpy.array(numpy.ones((nbursts), dtype=float)*numpy.nan), @@ -101,5 +102,11 @@ bins = numpy.arange(1,nbins+1) # bin_habs = (bins*bin_size+bin_size/2)+th+bh - bin_habs = (bins*bin_size+bin_size/2) + bin_habs = (bins*bin_size+bin_size/2)+th+bh + + # added by SH -- 15 Oct 2008 + # raw2proc:ticket:27 adjust bin_habs along beam to nadir + # Nortek awac beam angle is fixed at 25 deg + # adjustment is cos(25 deg) (which is approx .90*height) + bin_habs = (bin_habs*numpy.cos(25.*numpy.pi/180)) iaboveblank = bin_habs > th+bh+(bin_size) @@ -139,5 +146,5 @@ pressure = wpa[13] # dbar # pressure (dbar) converted to water depth - water_depth = th + seawater.depth(pressure, platform_info['lat']) # m + wd = th + seawater.depth(pressure, platform_info['lat']) # m temperature = wpa[14] # deg C @@ -147,5 +154,5 @@ # initialize for new profile - bin_hab = numpy.ones(nbins)*numpy.nan + hab = numpy.ones(nbins)*numpy.nan spd = numpy.ones(nbins)*numpy.nan dir = numpy.ones(nbins)*numpy.nan @@ -161,5 +168,5 @@ bin_number = wpa[0] j = wpa[0]-1 - bin_hab[j] = wpa[1] + hab[j] = wpa[1] spd[j] = wpa[2] # m/s @@ -180,24 +187,19 @@ # if done reading profile, just read data for last bin if bin_number==nbins: - # compute water mask - # Using George Voulgaris' method based on water depth - # minus half of the significant wave height (Hs) - # and computed habs - # if positive is up, what's less than zero depth? - bin_depths = bin_habs-(water_depth) + # compute water mask + # if positive is up, in water is less than zero depth + bin_depths = (bin_habs)-(wd) iwater = bin_depths+bin_size/2 < 0 iwater = iwater*iaboveblank - # alternatively use nominal water depth (MSL) averaged from full pressure record + # use nominal water depth (MSL) averaged from full pressure record # this should be checked/recalulated every so often - # MSL = sensor_info['mean_sea_level'] # Mean sea level at station (meters) or nominal water depth - # MSL = 12. # **** first guess is 12 meters for jpier - # z = bin_habs-MSL - z = bin_habs + z = bin_habs+platform_info['mean_water_depth'] data['dt'][i] = sample_dt # sample datetime data['time'][i] = dt2es(sample_dt) # sample time in epoch seconds data['z'] = z - data['water_depth'][i] = water_depth + data['wd'][i] = -1*wd + data['wl'][i] = platform_info['mean_water_depth'] - (-1*wd) data['water_temp'][i] = temperature data['pressure'][i] = pressure @@ -224,4 +226,14 @@ # title_str = sensor_info['description']+' at '+ platform_info['location'] + + if 'mean_water_depth' in platform_info.keys(): + msl_str = platform_info['mean_water_depth'] + else: + msl_str = 'None' + if 'mean_water_depth_time_period' in platform_info.keys(): + msl_tp_str = platform_info['mean_water_depth_time_period'] + else: + msl_tp_str = 'None' + global_atts = { 'title' : title_str, @@ -252,4 +264,7 @@ 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), 'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + # + 'mean_water_depth' : msl_str, + 'mean_water_depth_time_period' : msl_tp_str, # 'creation_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), @@ -258,5 +273,5 @@ # # must type match to data (e.g. fillvalue is real if data is real) - '_FillValue' : -99999., + '_FillValue' : numpy.nan, } @@ -288,5 +303,5 @@ 'long_name': 'Height', 'standard_name': 'height', - 'reference':'zero at sea-surface', + 'reference':'zero at mean-sea-level', 'positive' : 'up', 'units': 'm', @@ -327,14 +342,27 @@ 'units': 'dB', }, - 'water_depth': {'short_name': 'wd', - 'long_name': 'Water Depth', - 'standard_name': 'water_depth', - 'units': 'm', - }, + 'wd': {'short_name': 'wd', + 'long_name': 'Water Depth', + 'standard_name': 'water_depth', + 'reference':'zero at surface', + 'positive' : 'up', + 'units': 'm', + }, + 'wl': {'short_name': 'wl', + 'long_name': 'Water Level', + 'standard_name': 'water_level', + 'reference':'MSL', + 'reference_to_MSL' : 0., + 'reference_MSL_datum' : platform_info['mean_water_depth'], + 'reference_MSL_datum_time_period' : platform_info['mean_water_depth_time_period'], + 'positive' : 'up', + 'z' : 0., + 'units': 'm', + }, 'pressure': {'short_name': 'p', - 'long_name': 'Pressure', - 'standard_name': 'pressure', - 'units': 'dbar', - }, + 'long_name': 'Pressure', + 'standard_name': 'pressure', + 'units': 'dbar', + }, 'water_temp': {'short_name': 'wtemp', 'long_name': 'Water Temperature at Transducer', @@ -370,5 +398,6 @@ ('e2', NC.INT, ('ntime', 'nz')), ('e3', NC.INT, ('ntime', 'nz')), - ('water_depth', NC.FLOAT, ('ntime',)), + ('wd', NC.FLOAT, ('ntime',)), + ('wl', NC.FLOAT, ('ntime',)), ('pressure', NC.FLOAT, ('ntime',)), ('water_temp', NC.FLOAT, ('ntime',)), @@ -391,5 +420,6 @@ ('e2', data['e2'][i]), ('e3', data['e3'][i]), - ('water_depth', data['water_depth'][i]), + ('wd', data['wd'][i]), + ('wl', data['wl'][i]), ('pressure', data['pressure'][i]), ('water_temp', data['water_temp'][i]), @@ -433,5 +463,6 @@ ('e2', data['e2'][i]), ('e3', data['e3'][i]), - ('water_depth', data['water_depth'][i]), + ('wd', data['wd'][i]), + ('wl', data['wl'][i]), ('pressure', data['pressure'][i]), ('water_temp', data['water_temp'][i]), Index: raw2proc/trunk/raw2proc/proc_rdi_logdata_adcp.py =================================================================== --- raw2proc/trunk/raw2proc/proc_rdi_logdata_adcp.py (revision 211) +++ raw2proc/trunk/raw2proc/proc_rdi_logdata_adcp.py (revision 213) @@ -1,4 +1,4 @@ #!/usr/bin/env python -# Last modified: Time-stamp: <2008-10-01 12:47:42 haines> +# Last modified: Time-stamp: <2008-10-16 14:06:06 haines> """ how to parse data, and assert what data and info goes into @@ -75,5 +75,6 @@ max_wave_period = rdi[13] # Maximum Wave Period (Tmax, sec) - water_depth = rdi[11]/1000 # Water Depth (meters) (based on ADCP backscatter or input config??) + wd = rdi[11]/1000 # Water Depth (meters) (based on ADCP backscatter or input config??) + # This includes height of transducer nbins = int(rdi[14]) # Number of bins @@ -107,8 +108,16 @@ # and computed habs # if positive is up, what's less than zero depth? - bin_depths = bin_habs-(water_depth-sig_wave_ht/2) + + # added by SH -- 15 Oct 2008 + # raw2proc:ticket:27 adjust bin_habs along beam to nadir + # adjustment is cos(20 deg) (which is approx .95*height) assuming fixed 20 deg + bin_habs = bin_habs*numpy.cos(20.*numpy.pi/180) + bin_depths = bin_habs-(wd) iwater = bin_depths+bin_size/2 < 0 - z = bin_habs + # use nominal water depth (MSL) averaged from full pressure record + # this should be checked/recalulated every so often + z = bin_habs + platform_info['mean_water_depth'] # meters, (+) up, (-) down + # check that length of bin_depths is equal to nbins u = numpy.ones(nbins)*numpy.nan @@ -127,5 +136,6 @@ 'u' : numpy.array(numpy.ones((len(lines),nbins), dtype=float)*numpy.nan), 'v' : numpy.array(numpy.ones((len(lines),nbins), dtype=float)*numpy.nan), - 'water_depth' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan), + 'wd' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan), + 'wl' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan), } @@ -136,5 +146,6 @@ data['u'][i] = u data['v'][i] = v - data['water_depth'][i] = water_depth + data['wd'][i] = -1*wd + data['wl'][i] = platform_info['mean_water_depth'] - (-1*wd) i = i+1 @@ -145,4 +156,14 @@ # title_str = sensor_info['description']+' at '+ platform_info['location'] + + if 'mean_water_depth' in platform_info.keys(): + msl_str = platform_info['mean_water_depth'] + else: + msl_str = 'None' + if 'mean_water_depth_time_period' in platform_info.keys(): + msl_tp_str = platform_info['mean_water_depth_time_period'] + else: + msl_tp_str = 'None' + global_atts = { 'title' : title_str, @@ -173,4 +194,7 @@ 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), 'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + # + 'mean_water_depth' : msl_str, + 'mean_water_depth_time_period' : msl_tp_str, # 'creation_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), @@ -231,9 +255,22 @@ 'reference': 'clockwise from True North', }, - 'water_depth': {'short_name': 'wd', - 'long_name': 'Water Depth', - 'standard_name': 'water_depth', - 'units': 'm', - }, + 'wd': {'short_name': 'wd', + 'long_name': 'Water Depth', + 'standard_name': 'water_depth', + 'reference':'zero at surface', + 'positive' : 'up', + 'units': 'm', + }, + 'wl': {'short_name': 'wl', + 'long_name': 'Water Level', + 'standard_name': 'water_level', + 'reference':'MSL', + 'reference_to_MSL' : 0., + 'reference_MSL_datum' : platform_info['mean_water_depth'], + 'reference_MSL_datum_time_period' : platform_info['mean_water_depth_time_period'], + 'positive' : 'up', + 'z' : 0., + 'units': 'm', + }, } @@ -261,5 +298,6 @@ ('u', NC.FLOAT, ('ntime', 'nz')), ('v', NC.FLOAT, ('ntime', 'nz')), - ('water_depth', NC.FLOAT, ('ntime',)), + ('wd', NC.FLOAT, ('ntime',)), + ('wl', NC.FLOAT, ('ntime',)), ) @@ -277,5 +315,6 @@ ('u', data['u'][i]), ('v', data['v'][i]), - ('water_depth', data['water_depth'][i]), + ('wd', data['wd'][i]), + ('wl', data['wl'][i]), ) @@ -314,5 +353,6 @@ ('u', data['u'][i]), ('v', data['v'][i]), - ('water_depth', data['water_depth'][i]), + ('wd', data['wd'][i]), + ('wl', data['wl'][i]), ) Index: raw2proc/trunk/raw2proc/procutil.py =================================================================== --- raw2proc/trunk/raw2proc/procutil.py (revision 167) +++ raw2proc/trunk/raw2proc/procutil.py (revision 213) @@ -1,4 +1,4 @@ #!/usr/bin/env python -# Last modified: Time-stamp: <2008-05-08 16:11:36 haines> +# Last modified: Time-stamp: <2008-10-23 11:30:48 haines> """Utilities to help data processing @@ -13,8 +13,11 @@ __author__ = "Sara Haines " +import os.path from datetime import datetime, timedelta, tzinfo from dateutil.tz import tzlocal, tzutc import time import math + +from ncutil import * def check_configs(): @@ -321,2 +324,76 @@ return wind_speed*math.cos(wind_from_direction*math.pi/180) + +def proc2latest(pi, si, yyyy_mm): + """Select specific variables and times from current monthly netCDF + and post as latest data. TEST MODE. + + For each active config file, load specific variables from NCCOOS + monthly netCDF, make any necessary changes to data or attributes + conform to SEACOOS Data Model, subset data (last 48 hours), and + create new netCDF file in latest netCDF directory. + + NOTE: In test mode right now. See auto() function for similar action. + + """ + + platform = pi['id'] + package = si['id'] + si['proc_filename'] = '%s_%s_%s.nc' % (platform, package, yyyy_mm) + ifn = os.path.join(si['proc_dir'], si['proc_filename']) + if os.path.exists(ifn): + # get dt from current month file + (es, units) = nc_get_time(ifn) + dt = [es2dt(e) for e in es] + last_dt = dt[-1] + + # determine which index of data is within the specified timeframe (last 2 days) + n = len(dt) + idx = numpy.array([False for i in range(n)]) + for i, val in enumerate(dt): + if val>last_dt-timedelta(days=2) and val<=last_dt+timedelta(seconds=360): + idx[i] = True + dt = numpy.array(dt) + dt = dt[idx] + + # read in data and unpack tuple + d = nc_load(ifn, si['latest_vars']) + global_atts, var_atts, dim_inits, var_inits, var_data = d + list_of_record_vars = nc_find_record_vars(ifn) + + # turn off unlimited dimension (SH NOTE: As of pycdf-0.6-3b cannot + # delete a dimension or reset unlimited to limited within either + # CDF or CDFDim class, so doing it manually here by setting list + # 'dim_init' before creating of new netcdf.) + dim_inits = list(dim_inits) + for i in range(len(dim_inits)): + if dim_inits[i][1]==0: + dim_inits[i] = ('ntime', len(dt)) + dim_inits = tuple(dim_inits) + + # subset data + varNames = [vn for vn, vt, vd in var_inits] + var_data = list(var_data) + for i in range(len(varNames)): + vn, vd = var_data[i] + + if vn in list_of_record_vars: + var_data[i]=(vn, vd[idx]) + var_data = tuple(var_data) + + global_atts['start_date'] = dt[0].strftime('%Y-%m-%d %H:%M:%S') + si['latest_filename'] = 'nccoos_%s_%s_latest.nc' % (platform, package) + ofn = os.path.join(si['latest_dir'], si['latest_filename']) + d = (global_atts, var_atts, dim_inits, var_inits, var_data) + + # write latest data + nc_create(ofn, d) + + # quick way to rename dimensions + nc_rename_dimension(ofn, 'ntime', 'time') + nc_rename_dimension(ofn, 'nlat', 'lat') + nc_rename_dimension(ofn, 'nlon', 'lon') + nc_rename_dimension(ofn, 'nz', 'z') + + # global replace _FillValue + nc_replace_fillvalue(ofn, -99999.0) Index: raw2proc/trunk/raw2proc/raw2proc.py =================================================================== --- raw2proc/trunk/raw2proc/raw2proc.py (revision 211) +++ raw2proc/trunk/raw2proc/raw2proc.py (revision 213) @@ -1,4 +1,4 @@ #!/usr/bin/env python -# Last modified: Time-stamp: <2008-09-30 17:40:58 haines> +# Last modified: Time-stamp: <2008-10-23 11:05:43 haines> """Process raw data to monthly netCDF data files @@ -153,20 +153,24 @@ """Determine which list of raw files to process for month """ import glob - # determine when month starts and ends - # + months = find_months(yyyy_mm) # list all the raw files in prev-month, this-month, and next-month all_raw_files = [] - for mon in months: - mstr = mon.strftime('%Y_%m') - gs = os.path.join(si['raw_dir'], mstr, si['raw_file_glob']) + m = re.search('\d{4}_\d{2}$', si['raw_dir']) + if m: + # look for raw_file_glob in specific directory ending in YYYY_MM + # but look no further. + gs = os.path.join(si['raw_dir'], si['raw_file_glob']) all_raw_files.extend(glob.glob(gs)) - + else: + # no YYYY_MM at end of raw_dir then look for files + # in prev-month, this-month, and next-month + for mon in months: + mstr = mon.strftime('%Y_%m') + gs = os.path.join(si['raw_dir'], mstr, si['raw_file_glob']) + all_raw_files.extend(glob.glob(gs)) + all_raw_files.sort() - # ****** ((SMH) NOTE: Will need to override looking in specific - # subdirs of months if all data is contained in one file for long - # deployment, such as with adcp binary data. - # dt_start = si['proc_start_dt']-timedelta(days=1) @@ -187,5 +191,5 @@ # end JC changes if fndt: - if dt_start <= fndt <= dt_end: + if dt_start <= fndt <= dt_end or m: raw_files.append(fn) raw_dts.append(fndt) @@ -418,87 +422,4 @@ -def proc2latest(pi, si, yyyy_mm): - """Select specific variables and times from current monthly netCDF - and post as latest data. TEST MODE. - - For each active config file, load specific variables from NCCOOS - monthly netCDF, make any necessary changes to data or attributes - conform to SEACOOS Data Model, subset data (last 48 hours), and - create new netCDF file in latest netCDF directory. - - NOTE: In test mode right now. See auto() function for similar action. - - """ - platform = pi['id'] - package = si['id'] - si['proc_filename'] = '%s_%s_%s.nc' % (platform, package, yyyy_mm) - ifn = os.path.join(si['proc_dir'], si['proc_filename']) - if os.path.exists(ifn): - # get dt from current month file - (es, units) = nc_get_time(ifn) - dt = [es2dt(e) for e in es] - last_dt = dt[-1] - - # determine which index of data is within the specified timeframe (last 2 days) - n = len(dt) - idx = numpy.array([False for i in range(n)]) - for i, val in enumerate(dt): - if val>last_dt-timedelta(days=2) and val<=last_dt+timedelta(seconds=360): - idx[i] = True - dt = numpy.array(dt) - dt = dt[idx] - - # read in data and unpack tuple - d = nc_load(ifn, si['latest_vars']) - global_atts, var_atts, dim_inits, var_inits, var_data = d - list_of_record_vars = nc_find_record_vars(ifn) - - # find unlimited dimension (a dimension length of zero) and set to length of dt - # SECOORA scout does not understand of zero (0) also replace dim names with following names - # e.g. ntime for time - dees = [['ntime','time'], ['nlat','lat'], ['nlon', 'lon'], ['nz', 'z'],['ndir','dir'],['nfreq','freq']] - dim_inits = list(dim_inits) - for j in range(len(dees)): - for i in range(len(dim_inits)): - if dim_inits[i][0]==dees[j][0]: # 'ntime' - dim_inits[i] = (dees[j][1], dim_inits[i][1]) - for i in range(len(dim_inits)): - if dim_inits[i][1]==0: - dim_inits[i] = ('time', len(dt)) - dim_inits = tuple(dim_inits) - # print dim_inits - - # replace name of dependent dimension names with speificied names for CDL v2.0 in variables - # e.g. ntime for time - var_inits = list(var_inits) - for j in range(len(dees)): - for i in range(len(var_inits)): - v = list(var_inits[i]) - dl = list(v[2]) - for k in range(len(dl)): - if dl[k]==dees[j][0]: - dl[k]=dees[j][1] # 'time' - var_inits[i]=(v[0], v[1], tuple(dl)) - var_inits = tuple(var_inits) - # print var_inits - - # subset data - varNames = [vn for vn, vt, vd in var_inits] - var_data = list(var_data) - for i in range(len(varNames)): - vn, vd = var_data[i] - if vn in list_of_record_vars: - var_data[i]=(vn, vd[idx]) - var_data = tuple(var_data) - global_atts['start_date'] = dt[0].strftime('%Y-%m-%d %H:%M:%S') - - # write latest data - si['latest_filename'] = 'nccoos_%s_%s_latest.nc' % (platform, package) - ofn = os.path.join(si['latest_dir'], si['latest_filename']) - d = (global_atts, var_atts, dim_inits, var_inits, var_data) - - nc_create(ofn, d) - - # globals start_dt = datetime.utcnow()