Changeset 213
- Timestamp:
- 10/23/08 11:37:09
- Files:
-
- raw2proc/trunk/raw2proc/bogue_config_20060918.py (modified) (1 diff)
- raw2proc/trunk/raw2proc/bogue_config_20070224.py (modified) (1 diff)
- raw2proc/trunk/raw2proc/bogue_config_20080430.py (modified) (1 diff)
- raw2proc/trunk/raw2proc/jpier_config_20050425.py (modified) (1 diff)
- raw2proc/trunk/raw2proc/jpier_config_20080411.py (modified) (2 diffs)
- raw2proc/trunk/raw2proc/jpier_config_20080722.py (modified) (3 diffs)
- raw2proc/trunk/raw2proc/lsrb_config_20050623.py (modified) (2 diffs)
- raw2proc/trunk/raw2proc/lsrb_config_20051031.py (modified) (2 diffs)
- raw2proc/trunk/raw2proc/lsrb_config_20060622.py (modified) (2 diffs)
- raw2proc/trunk/raw2proc/lsrb_config_20070612.py (modified) (2 diffs)
- raw2proc/trunk/raw2proc/ncutil.py (modified) (4 diffs)
- raw2proc/trunk/raw2proc/proc_nortek_wpa_adcp.py (modified) (15 diffs)
- raw2proc/trunk/raw2proc/proc_rdi_logdata_adcp.py (modified) (11 diffs)
- raw2proc/trunk/raw2proc/procutil.py (modified) (3 diffs)
- raw2proc/trunk/raw2proc/raw2proc.py (modified) (4 diffs)
Legend:
- Unmodified
- Added
- Removed
- Modified
- Copied
- Moved
raw2proc/trunk/raw2proc/bogue_config_20060918.py
r212 r213 5 5 'lon' : -77.034131, # degrees true (-) west, (+) east 6 6 'mvar' : -9.7, # degrees (-) west, (+) east 7 'water_depth' : 8.14, # meters 7 'mean_water_depth' : -8.14, # meters (-) down, (+) up 8 'mean_water_depth_time_period' : 'June 2006 - June 2008', 8 9 'institution' : 'nccoos', 9 10 # raw2proc/trunk/raw2proc/bogue_config_20070224.py
r212 r213 5 5 'lon' : -77.034131, # degrees true (-) west, (+) east 6 6 'mvar' : -9.7, # degrees (-) west, (+) east 7 'water_depth' : 8.14, # meters 7 'mean_water_depth' : -8.14, # meters (-) down, (+) up 8 'mean_water_depth_time_period' : 'June 2006 - June 2008', 8 9 'institution' : 'nccoos', 9 10 # raw2proc/trunk/raw2proc/bogue_config_20080430.py
r212 r213 5 5 'lon' : -77.034131, # degrees true (-) west, (+) east 6 6 'mvar' : -9.7, # degrees (-) west, (+) east 7 'water_depth' : 8.14, # meters 7 'mean_water_depth' : -8.14, # meters (-) down, (+) up 8 'mean_water_depth_time_period' : 'June 2006 - June 2008', 8 9 'institution' : 'nccoos', 9 10 # raw2proc/trunk/raw2proc/jpier_config_20050425.py
r212 r213 5 5 'lon' : -75.5958, # degrees true (-) west, (+) east 6 6 'mvar' : -10.83333, # degrees (-) west, (+) east 7 'water_depth' : 11.38, # meters 7 'mean_water_depth' : -11.38, # meters (-) down, (+) up 8 'mean_water_depth_time_period' : 'May 2008 - Oct 2008', 8 9 'institution' : 'nccoos', 9 10 # raw2proc/trunk/raw2proc/jpier_config_20080411.py
r212 r213 5 5 'lon' : -75.5958, # degrees true (-) west, (+) east 6 6 'mvar' : -10.83333, # degrees (-) west, (+) east 7 'water_depth' : 11.38, # meters 7 'mean_water_depth' : -11.38, # meters (-) down, (+) up 8 'mean_water_depth_time_period' : 'May 2008 - Oct 2008', 8 9 'institution' : 'nccoos', 9 10 # … … 23 24 'bin_size' : 0.5, # meters 24 25 'transducer_ht' : 0.5, # meters above the bottom 25 'blanking_ht' : 0.41, # meters above transducer 26 # 'blanking_ht' : 0.41, # meters above transducer 27 'blanking_ht' : 1.9, # meters above transducer 26 28 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', 27 29 'latest_vars' : ('time','lat','lon','z','u','v'), raw2proc/trunk/raw2proc/jpier_config_20080722.py
r212 r213 5 5 'lon' : -75.5958, # degrees true (-) west, (+) east 6 6 'mvar' : -10.83333, # degrees (-) west, (+) east 7 'water_depth' : 11.38, # meters 7 'mean_water_depth' : -11.38, # meters (-) down, (+) up 8 'mean_water_depth_time_period' : 'May 2008 - Oct 2008', 8 9 'institution' : 'nccoos', 9 10 # … … 24 25 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', 25 26 'latest_vars' : ('time','lat','lon','z','wspd','wdir', 'air_temp', 'humidity', 'air_pressure'), 26 27 }, 27 28 'adcp' : { 'id' : 'adcp', 28 29 'description' : 'Current profile data', … … 35 36 'bin_size' : 0.5, # meters 36 37 'transducer_ht' : 0.5, # meters above the bottom 37 'blanking_ht' : 0.41, # meters above transducer 38 # 'blanking_ht' : 0.41, # meters above transducer 39 'blanking_ht' : 1.9, # meters above transducer 38 40 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', 39 'latest_vars' : ('time','lat','lon','z','u','v' ),41 'latest_vars' : ('time','lat','lon','z','u','v','wd','wl'), 40 42 }, 41 43 'adcpwaves' : {'id' : 'adcpwaves', raw2proc/trunk/raw2proc/lsrb_config_20050623.py
r212 r213 5 5 'lon' : -76.42, # degrees true (-) west, (+) east 6 6 'mvar' : -9.967, # degrees (-) west, (+) east 7 'water_depth' : 25.17, # meters (ave from start to end of deployment) 7 'mean_water_depth' : -25.17, # meters (-) down, (+) up 8 'mean_water_depth_time_period' : 'June 2005 - Sept 2005', 8 9 'institution' : 'nccoos', 9 10 # … … 27 28 'adcpwaves' : {'id' : 'adcpwaves', 28 29 'description' : 'Directional wave data', 29 'raw_dir' : '/seacoos/data/nccoos/level0/lsrb/adcp_ cSpecData',30 'raw_file_glob' : ' DSpec*',30 'raw_dir' : '/seacoos/data/nccoos/level0/lsrb/adcp_bLogData/2005_06', 31 'raw_file_glob' : 'lsrb_LogData_20050623*', 31 32 'proc_dir' : '/seacoos/data/nccoos/level1/lsrb/adcpwaves', 32 'process_module' : 'proc_rdi_ dspec_dw',33 'process_module' : 'proc_rdi_logdata_dw', 33 34 'utc_offset' : 4, # hours offset to utc 34 35 'ndir' : 90., raw2proc/trunk/raw2proc/lsrb_config_20051031.py
r212 r213 5 5 'lon' : -76.42, # degrees true (-) west, (+) east 6 6 'mvar' : -9.967, # degrees (-) west, (+) east 7 'water_depth' : 25.86, # meters (ave from start to end of deployment) 7 'mean_water_depth' : -25.86, # meters (-) down, (+) up 8 'mean_water_depth_time_period' : 'Oct 2005 - Feb 2006', 8 9 'institution' : 'nccoos', 9 10 # … … 27 28 'adcpwaves' : {'id' : 'adcpwaves', 28 29 'description' : 'Directional wave data', 29 'raw_dir' : '/seacoos/data/nccoos/level0/lsrb/adcp_ cSpecData',30 'raw_file_glob' : ' DSpec*',30 'raw_dir' : '/seacoos/data/nccoos/level0/lsrb/adcp_bLogData/2005_10', 31 'raw_file_glob' : 'lsrb_LogData*', 31 32 'proc_dir' : '/seacoos/data/nccoos/level1/lsrb/adcpwaves', 32 'process_module' : 'proc_rdi_ dspec_dw',33 'process_module' : 'proc_rdi_logdata_dw', 33 34 'utc_offset' : 4, # hours offset to utc 34 35 'ndir' : 90., raw2proc/trunk/raw2proc/lsrb_config_20060622.py
r212 r213 5 5 'lon' : -76.42, # degrees true (-) west, (+) east 6 6 'mvar' : -9.967, # degrees (-) west, (+) east 7 'water_depth' : 26.26, # meters (ave from start to end of deployment) 7 'mean_water_depth' : -26.26, # meters (-) down, (+) up 8 'mean_water_depth_time_period' : 'June 2006 - Oct 2006', 8 9 'institution' : 'nccoos', 9 10 # … … 27 28 'adcpwaves' : {'id' : 'adcpwaves', 28 29 'description' : 'Directional wave data', 29 'raw_dir' : '/seacoos/data/nccoos/level0/lsrb/adcp_ cSpecData',30 'raw_file_glob' : ' DSpec*',30 'raw_dir' : '/seacoos/data/nccoos/level0/lsrb/adcp_bLogData/2006_06', 31 'raw_file_glob' : 'lsrb_LogData*', 31 32 'proc_dir' : '/seacoos/data/nccoos/level1/lsrb/adcpwaves', 32 'process_module' : 'proc_rdi_ dspec_dw',33 'process_module' : 'proc_rdi_logdata_dw', 33 34 'utc_offset' : 4, # hours offset to utc 34 35 'ndir' : 90., raw2proc/trunk/raw2proc/lsrb_config_20070612.py
r212 r213 5 5 'lon' : -76.42, # degrees true (-) west, (+) east 6 6 'mvar' : -9.967, # degrees (-) west, (+) east 7 'water_depth' : 26.5, # meters (ave from start to end of deployment) 7 'mean_water_depth' : -26.98, # meters (-) down, (+) up 8 'mean_water_depth_time_period' : 'Aug 2007 - Sep 2007', 8 9 'institution' : 'nccoos', 9 10 # … … 27 28 'adcpwaves' : {'id' : 'adcpwaves', 28 29 'description' : 'Directional wave data', 29 'raw_dir' : '/seacoos/data/nccoos/level0/lsrb/adcp_ cSpecData',30 'raw_file_glob' : ' DSpec*',30 'raw_dir' : '/seacoos/data/nccoos/level0/lsrb/adcp_bLogData/2007_06', 31 'raw_file_glob' : 'lsrb_LogData*', 31 32 'proc_dir' : '/seacoos/data/nccoos/level1/lsrb/adcpwaves', 32 'process_module' : 'proc_rdi_ dspec_dw',33 'process_module' : 'proc_rdi_logdata_dw', 33 34 'utc_offset' : 4, # hours offset to utc 34 35 'ndir' : 90., raw2proc/trunk/raw2proc/ncutil.py
r210 r213 1 1 #!/usr/bin/env python 2 # Last modified: Time-stamp: <2008- 09-25 14:01:50haines>2 # Last modified: Time-stamp: <2008-10-23 11:31:24 haines> 3 3 """ 4 4 Create, update and load utilities for netcdf files … … 124 124 125 125 def nc_get_time(ncFile): 126 """ get time array from file """126 """Get time array from file """ 127 127 try: 128 128 nc = CDF(ncFile) … … 140 140 141 141 def nc_find_record_vars(ncFile): 142 """ find which variable are record variables"""142 """Find which variable are record variables""" 143 143 try: 144 144 nc = CDF(ncFile) … … 151 151 print "CDFError:", msg 152 152 153 154 def nc_replace_fillvalue(ncFile, newfillvalue=-99999.0): 155 """ 156 Replaces any occurrence of old _FillValue with new one 157 158 This function is useful for replacing the _FillValue global 159 attribute and then searching the data for the old value and 160 replacing it with the new one. 161 162 :Parameters: 163 ncFile : string 164 Path and name of file to create 165 166 :Other Parameters: 167 newfillvalue : type match to data (generally float) 168 By default is -99999.0 169 170 """ 171 try: 172 nc = CDF(ncFile, NC.WRITE) 173 nc.automode() 174 oldfillvalue = nc._FillValue 175 nc._FillValue = newfillvalue 176 for v in nc.variables().keys(): 177 vd = nc.var(v)[:] 178 if numpy.isnan(oldfillvalue): 179 idx = numpy.isnan(vd) 180 else: 181 idx = vd == oldfillvalue 182 if idx.any(): 183 vd[idx] = nc._FillValue 184 nc.var(v)[:] = vd 185 nc.close() 186 except CDFError, msg: 187 print "CDFError:", msg 188 189 def nc_rename_dimension(ncFile, oldname, newname): 190 """ Rename dimension name """ 191 try: 192 nc = CDF(ncFile, NC.WRITE) 193 nc.definemode() 194 for d in nc.dimensions().keys(): 195 if d==oldname: nc.dim(d).rename(newname) 196 nc.close() 197 except CDFError, msg: 198 print "CDFError:", msg 199 153 200 154 201 def nc_load(ncFile, varsLoad='all', nameType='variable_name', raw2proc/trunk/raw2proc/proc_nortek_wpa_adcp.py
r211 r213 1 1 #!/usr/bin/env python 2 # Last modified: Time-stamp: <2008-10- 01 12:47:16haines>2 # Last modified: Time-stamp: <2008-10-16 17:02:54 haines> 3 3 """ 4 4 how to parse data, and assert what data and info goes into … … 88 88 'e2' : numpy.array(numpy.ones((nbursts,nbins), dtype=int)*numpy.nan), 89 89 'e3' : numpy.array(numpy.ones((nbursts,nbins), dtype=int)*numpy.nan), 90 'water_depth' : numpy.array(numpy.ones((nbursts), dtype=float)*numpy.nan), 90 'wd' : numpy.array(numpy.ones((nbursts), dtype=float)*numpy.nan), 91 'wl' : numpy.array(numpy.ones((nbursts), dtype=float)*numpy.nan), 91 92 'water_temp' : numpy.array(numpy.ones((nbursts), dtype=float)*numpy.nan), 92 93 'pressure' : numpy.array(numpy.ones((nbursts), dtype=float)*numpy.nan), … … 101 102 bins = numpy.arange(1,nbins+1) 102 103 # bin_habs = (bins*bin_size+bin_size/2)+th+bh 103 bin_habs = (bins*bin_size+bin_size/2) 104 bin_habs = (bins*bin_size+bin_size/2)+th+bh 105 106 # added by SH -- 15 Oct 2008 107 # raw2proc:ticket:27 adjust bin_habs along beam to nadir 108 # Nortek awac beam angle is fixed at 25 deg 109 # adjustment is cos(25 deg) (which is approx .90*height) 110 bin_habs = (bin_habs*numpy.cos(25.*numpy.pi/180)) 104 111 iaboveblank = bin_habs > th+bh+(bin_size) 105 112 … … 139 146 pressure = wpa[13] # dbar 140 147 # pressure (dbar) converted to water depth 141 w ater_depth= th + seawater.depth(pressure, platform_info['lat']) # m148 wd = th + seawater.depth(pressure, platform_info['lat']) # m 142 149 temperature = wpa[14] # deg C 143 150 … … 147 154 148 155 # initialize for new profile 149 bin_hab = numpy.ones(nbins)*numpy.nan156 hab = numpy.ones(nbins)*numpy.nan 150 157 spd = numpy.ones(nbins)*numpy.nan 151 158 dir = numpy.ones(nbins)*numpy.nan … … 161 168 bin_number = wpa[0] 162 169 j = wpa[0]-1 163 bin_hab[j] = wpa[1]170 hab[j] = wpa[1] 164 171 165 172 spd[j] = wpa[2] # m/s … … 180 187 # if done reading profile, just read data for last bin 181 188 if bin_number==nbins: 182 # compute water mask 183 # Using George Voulgaris' method based on water depth 184 # minus half of the significant wave height (Hs) 185 # and computed habs 186 # if positive is up, what's less than zero depth? 187 bin_depths = bin_habs-(water_depth) 189 # compute water mask 190 # if positive is up, in water is less than zero depth 191 bin_depths = (bin_habs)-(wd) 188 192 iwater = bin_depths+bin_size/2 < 0 189 193 iwater = iwater*iaboveblank 190 194 191 # alternativelyuse nominal water depth (MSL) averaged from full pressure record195 # use nominal water depth (MSL) averaged from full pressure record 192 196 # this should be checked/recalulated every so often 193 # MSL = sensor_info['mean_sea_level'] # Mean sea level at station (meters) or nominal water depth 194 # MSL = 12. # **** first guess is 12 meters for jpier 195 # z = bin_habs-MSL 196 z = bin_habs 197 z = bin_habs+platform_info['mean_water_depth'] 197 198 198 199 data['dt'][i] = sample_dt # sample datetime 199 200 data['time'][i] = dt2es(sample_dt) # sample time in epoch seconds 200 201 data['z'] = z 201 data['water_depth'][i] = water_depth 202 data['wd'][i] = -1*wd 203 data['wl'][i] = platform_info['mean_water_depth'] - (-1*wd) 202 204 data['water_temp'][i] = temperature 203 205 data['pressure'][i] = pressure … … 224 226 # 225 227 title_str = sensor_info['description']+' at '+ platform_info['location'] 228 229 if 'mean_water_depth' in platform_info.keys(): 230 msl_str = platform_info['mean_water_depth'] 231 else: 232 msl_str = 'None' 233 if 'mean_water_depth_time_period' in platform_info.keys(): 234 msl_tp_str = platform_info['mean_water_depth_time_period'] 235 else: 236 msl_tp_str = 'None' 237 226 238 global_atts = { 227 239 'title' : title_str, … … 252 264 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), 253 265 'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), 266 # 267 'mean_water_depth' : msl_str, 268 'mean_water_depth_time_period' : msl_tp_str, 254 269 # 255 270 'creation_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), … … 258 273 # 259 274 # must type match to data (e.g. fillvalue is real if data is real) 260 '_FillValue' : -99999.,275 '_FillValue' : numpy.nan, 261 276 } 262 277 … … 288 303 'long_name': 'Height', 289 304 'standard_name': 'height', 290 'reference':'zero at sea-surface',305 'reference':'zero at mean-sea-level', 291 306 'positive' : 'up', 292 307 'units': 'm', … … 327 342 'units': 'dB', 328 343 }, 329 'water_depth': {'short_name': 'wd', 330 'long_name': 'Water Depth', 331 'standard_name': 'water_depth', 332 'units': 'm', 333 }, 344 'wd': {'short_name': 'wd', 345 'long_name': 'Water Depth', 346 'standard_name': 'water_depth', 347 'reference':'zero at surface', 348 'positive' : 'up', 349 'units': 'm', 350 }, 351 'wl': {'short_name': 'wl', 352 'long_name': 'Water Level', 353 'standard_name': 'water_level', 354 'reference':'MSL', 355 'reference_to_MSL' : 0., 356 'reference_MSL_datum' : platform_info['mean_water_depth'], 357 'reference_MSL_datum_time_period' : platform_info['mean_water_depth_time_period'], 358 'positive' : 'up', 359 'z' : 0., 360 'units': 'm', 361 }, 334 362 'pressure': {'short_name': 'p', 335 336 337 338 363 'long_name': 'Pressure', 364 'standard_name': 'pressure', 365 'units': 'dbar', 366 }, 339 367 'water_temp': {'short_name': 'wtemp', 340 368 'long_name': 'Water Temperature at Transducer', … … 370 398 ('e2', NC.INT, ('ntime', 'nz')), 371 399 ('e3', NC.INT, ('ntime', 'nz')), 372 ('water_depth', NC.FLOAT, ('ntime',)), 400 ('wd', NC.FLOAT, ('ntime',)), 401 ('wl', NC.FLOAT, ('ntime',)), 373 402 ('pressure', NC.FLOAT, ('ntime',)), 374 403 ('water_temp', NC.FLOAT, ('ntime',)), … … 391 420 ('e2', data['e2'][i]), 392 421 ('e3', data['e3'][i]), 393 ('water_depth', data['water_depth'][i]), 422 ('wd', data['wd'][i]), 423 ('wl', data['wl'][i]), 394 424 ('pressure', data['pressure'][i]), 395 425 ('water_temp', data['water_temp'][i]), … … 433 463 ('e2', data['e2'][i]), 434 464 ('e3', data['e3'][i]), 435 ('water_depth', data['water_depth'][i]), 465 ('wd', data['wd'][i]), 466 ('wl', data['wl'][i]), 436 467 ('pressure', data['pressure'][i]), 437 468 ('water_temp', data['water_temp'][i]), raw2proc/trunk/raw2proc/proc_rdi_logdata_adcp.py
r211 r213 1 1 #!/usr/bin/env python 2 # Last modified: Time-stamp: <2008-10- 01 12:47:42haines>2 # Last modified: Time-stamp: <2008-10-16 14:06:06 haines> 3 3 """ 4 4 how to parse data, and assert what data and info goes into … … 75 75 max_wave_period = rdi[13] # Maximum Wave Period (Tmax, sec) 76 76 77 water_depth = rdi[11]/1000 # Water Depth (meters) (based on ADCP backscatter or input config??) 77 wd = rdi[11]/1000 # Water Depth (meters) (based on ADCP backscatter or input config??) 78 # This includes height of transducer 78 79 nbins = int(rdi[14]) # Number of bins 79 80 … … 107 108 # and computed habs 108 109 # if positive is up, what's less than zero depth? 109 bin_depths = bin_habs-(water_depth-sig_wave_ht/2) 110 111 # added by SH -- 15 Oct 2008 112 # raw2proc:ticket:27 adjust bin_habs along beam to nadir 113 # adjustment is cos(20 deg) (which is approx .95*height) assuming fixed 20 deg 114 bin_habs = bin_habs*numpy.cos(20.*numpy.pi/180) 115 bin_depths = bin_habs-(wd) 110 116 iwater = bin_depths+bin_size/2 < 0 111 117 112 z = bin_habs 118 # use nominal water depth (MSL) averaged from full pressure record 119 # this should be checked/recalulated every so often 120 z = bin_habs + platform_info['mean_water_depth'] # meters, (+) up, (-) down 121 113 122 # check that length of bin_depths is equal to nbins 114 123 u = numpy.ones(nbins)*numpy.nan … … 127 136 'u' : numpy.array(numpy.ones((len(lines),nbins), dtype=float)*numpy.nan), 128 137 'v' : numpy.array(numpy.ones((len(lines),nbins), dtype=float)*numpy.nan), 129 'water_depth' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan), 138 'wd' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan), 139 'wl' : numpy.array(numpy.ones((len(lines)), dtype=float)*numpy.nan), 130 140 } 131 141 … … 136 146 data['u'][i] = u 137 147 data['v'][i] = v 138 data['water_depth'][i] = water_depth 148 data['wd'][i] = -1*wd 149 data['wl'][i] = platform_info['mean_water_depth'] - (-1*wd) 139 150 i = i+1 140 151 … … 145 156 # 146 157 title_str = sensor_info['description']+' at '+ platform_info['location'] 158 159 if 'mean_water_depth' in platform_info.keys(): 160 msl_str = platform_info['mean_water_depth'] 161 else: 162 msl_str = 'None' 163 if 'mean_water_depth_time_period' in platform_info.keys(): 164 msl_tp_str = platform_info['mean_water_depth_time_period'] 165 else: 166 msl_tp_str = 'None' 167 147 168 global_atts = { 148 169 'title' : title_str, … … 173 194 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), 174 195 'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), 196 # 197 'mean_water_depth' : msl_str, 198 'mean_water_depth_time_period' : msl_tp_str, 175 199 # 176 200 'creation_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), … … 231 255 'reference': 'clockwise from True North', 232 256 }, 233 'water_depth': {'short_name': 'wd', 234 'long_name': 'Water Depth', 235 'standard_name': 'water_depth', 236 'units': 'm', 237 }, 257 'wd': {'short_name': 'wd', 258 'long_name': 'Water Depth', 259 'standard_name': 'water_depth', 260 'reference':'zero at surface', 261 'positive' : 'up', 262 'units': 'm', 263 }, 264 'wl': {'short_name': 'wl', 265 'long_name': 'Water Level', 266 'standard_name': 'water_level', 267 'reference':'MSL', 268 'reference_to_MSL' : 0., 269 'reference_MSL_datum' : platform_info['mean_water_depth'], 270 'reference_MSL_datum_time_period' : platform_info['mean_water_depth_time_period'], 271 'positive' : 'up', 272 'z' : 0., 273 'units': 'm', 274 }, 238 275 } 239 276 … … 261 298 ('u', NC.FLOAT, ('ntime', 'nz')), 262 299 ('v', NC.FLOAT, ('ntime', 'nz')), 263 ('water_depth', NC.FLOAT, ('ntime',)), 300 ('wd', NC.FLOAT, ('ntime',)), 301 ('wl', NC.FLOAT, ('ntime',)), 264 302 ) 265 303 … … 277 315 ('u', data['u'][i]), 278 316 ('v', data['v'][i]), 279 ('water_depth', data['water_depth'][i]), 317 ('wd', data['wd'][i]), 318 ('wl', data['wl'][i]), 280 319 ) 281 320 … … 314 353 ('u', data['u'][i]), 315 354 ('v', data['v'][i]), 316 ('water_depth', data['water_depth'][i]), 355 ('wd', data['wd'][i]), 356 ('wl', data['wl'][i]), 317 357 ) 318 358 raw2proc/trunk/raw2proc/procutil.py
r167 r213 1 1 #!/usr/bin/env python 2 # Last modified: Time-stamp: <2008- 05-08 16:11:36haines>2 # Last modified: Time-stamp: <2008-10-23 11:30:48 haines> 3 3 """Utilities to help data processing 4 4 … … 13 13 __author__ = "Sara Haines <sara_haines@unc.edu>" 14 14 15 import os.path 15 16 from datetime import datetime, timedelta, tzinfo 16 17 from dateutil.tz import tzlocal, tzutc 17 18 import time 18 19 import math 20 21 from ncutil import * 19 22 20 23 def check_configs(): … … 321 324 return wind_speed*math.cos(wind_from_direction*math.pi/180) 322 325 326 327 def proc2latest(pi, si, yyyy_mm): 328 """Select specific variables and times from current monthly netCDF 329 and post as latest data. TEST MODE. 330 331 For each active config file, load specific variables from NCCOOS 332 monthly netCDF, make any necessary changes to data or attributes 333 conform to SEACOOS Data Model, subset data (last 48 hours), and 334 create new netCDF file in latest netCDF directory. 335 336 NOTE: In test mode right now. See auto() function for similar action. 337 338 """ 339 340 platform = pi['id'] 341 package = si['id'] 342 si['proc_filename'] = '%s_%s_%s.nc' % (platform, package, yyyy_mm) 343 ifn = os.path.join(si['proc_dir'], si['proc_filename']) 344 if os.path.exists(ifn): 345 # get dt from current month file 346 (es, units) = nc_get_time(ifn) 347 dt = [es2dt(e) for e in es] 348 last_dt = dt[-1] 349 350 # determine which index of data is within the specified timeframe (last 2 days) 351 n = len(dt) 352 idx = numpy.array([False for i in range(n)]) 353 for i, val in enumerate(dt): 354 if val>last_dt-timedelta(days=2) and val<=last_dt+timedelta(seconds=360): 355 idx[i] = True 356 dt = numpy.array(dt) 357 dt = dt[idx] 358 359 # read in data and unpack tuple 360 d = nc_load(ifn, si['latest_vars']) 361 global_atts, var_atts, dim_inits, var_inits, var_data = d 362 list_of_record_vars = nc_find_record_vars(ifn) 363 364 # turn off unlimited dimension (SH NOTE: As of pycdf-0.6-3b cannot 365 # delete a dimension or reset unlimited to limited within either 366 # CDF or CDFDim class, so doing it manually here by setting list 367 # 'dim_init' before creating of new netcdf.) 368 dim_inits = list(dim_inits) 369 for i in range(len(dim_inits)): 370 if dim_inits[i][1]==0: 371 dim_inits[i] = ('ntime', len(dt)) 372 dim_inits = tuple(dim_inits) 373 374 # subset data 375 varNames = [vn for vn, vt, vd in var_inits] 376 var_data = list(var_data) 377 for i in range(len(varNames)): 378 vn, vd = var_data[i] 379 380 if vn in list_of_record_vars: 381 var_data[i]=(vn, vd[idx]) 382 var_data = tuple(var_data) 383 384 global_atts['start_date'] = dt[0].strftime('%Y-%m-%d %H:%M:%S') 385 si['latest_filename'] = 'nccoos_%s_%s_latest.nc' % (platform, package) 386 ofn = os.path.join(si['latest_dir'], si['latest_filename']) 387 d = (global_atts, var_atts, dim_inits, var_inits, var_data) 388 389 # write latest data 390 nc_create(ofn, d) 391 392 # quick way to rename dimensions 393 nc_rename_dimension(ofn, 'ntime', 'time') 394 nc_rename_dimension(ofn, 'nlat', 'lat') 395 nc_rename_dimension(ofn, 'nlon', 'lon') 396 nc_rename_dimension(ofn, 'nz', 'z') 397 398 # global replace _FillValue 399 nc_replace_fillvalue(ofn, -99999.0) raw2proc/trunk/raw2proc/raw2proc.py
r211 r213 1 1 #!/usr/bin/env python 2 # Last modified: Time-stamp: <2008- 09-30 17:40:58haines>2 # Last modified: Time-stamp: <2008-10-23 11:05:43 haines> 3 3 """Process raw data to monthly netCDF data files 4 4 … … 153 153 """Determine which list of raw files to process for month """ 154 154 import glob 155 # determine when month starts and ends 156 # 155 157 156 months = find_months(yyyy_mm) 158 157 # list all the raw files in prev-month, this-month, and next-month 159 158 all_raw_files = [] 160 for mon in months: 161 mstr = mon.strftime('%Y_%m') 162 gs = os.path.join(si['raw_dir'], mstr, si['raw_file_glob']) 159 m = re.search('\d{4}_\d{2}$', si['raw_dir']) 160 if m: 161 # look for raw_file_glob in specific directory ending in YYYY_MM 162 # but look no further. 163 gs = os.path.join(si['raw_dir'], si['raw_file_glob']) 163 164 all_raw_files.extend(glob.glob(gs)) 164 165 else: 166 # no YYYY_MM at end of raw_dir then look for files 167 # in prev-month, this-month, and next-month 168 for mon in months: 169 mstr = mon.strftime('%Y_%m') 170 gs = os.path.join(si['raw_dir'], mstr, si['raw_file_glob']) 171 all_raw_files.extend(glob.glob(gs)) 172 165 173 all_raw_files.sort() 166 174 167 # ****** ((SMH) NOTE: Will need to override looking in specific168 # subdirs of months if all data is contained in one file for long169 # deployment, such as with adcp binary data.170 171 175 # 172 176 dt_start = si['proc_start_dt']-timedelta(days=1) … … 187 191 # end JC changes 188 192 if fndt: 189 if dt_start <= fndt <= dt_end :193 if dt_start <= fndt <= dt_end or m: 190 194 raw_files.append(fn) 191 195 raw_dts.append(fndt) … … 418 422 419 423 420 def proc2latest(pi, si, yyyy_mm):421 """Select specific variables and times from current monthly netCDF422 and post as latest data. TEST MODE.423 424 For each active config file, load specific variables from NCCOOS425 monthly netCDF, make any necessary changes to data or attributes426 conform to SEACOOS Data Model, subset data (last 48 hours), and427 create new netCDF file in latest netCDF directory.428 429 NOTE: In test mode right now. See auto() function for similar action.430 431 """432 platform = pi['id']433 package = si['id']434 si['proc_filename'] = '%s_%s_%s.nc' % (platform, package, yyyy_mm)435 ifn = os.path.join(si['proc_dir'], si['proc_filename'])436 if os.path.exists(ifn):437 # get dt from current month file438 (es, units) = nc_get_time(ifn)439 dt = [es2dt(e) for e in es]440 last_dt = dt[-1]441 442 # determine which index of data is within the specified timeframe (last 2 days)443 n = len(dt)444 idx = numpy.array([False for i in range(n)])445 for i, val in enumerate(dt):446 if val>last_dt-timedelta(days=2) and val<=last_dt+timedelta(seconds=360):447 idx[i] = True448 dt = numpy.array(dt)449 dt = dt[idx]450 451 # read in data and unpack tuple452 d = nc_load(ifn, si['latest_vars'])453 global_atts, var_atts, dim_inits, var_inits, var_data = d454 list_of_record_vars = nc_find_record_vars(ifn)455 456 # find unlimited dimension (a dimension length of zero) and set to length of dt457 # SECOORA scout does not understand of zero (0) also replace dim names with following names458 # e.g. ntime for time459 dees = [['ntime','time'], ['nlat','lat'], ['nlon', 'lon'], ['nz', 'z'],['ndir','dir'],['nfreq','freq']]460 dim_inits = list(dim_inits)461 for j in range(len(dees)):462 for i in range(len(dim_inits)):463 if dim_inits[i][0]==dees[j][0]: # 'ntime'464 dim_inits[i] = (dees[j][1], dim_inits[i][1])465 for i in range(len(dim_inits)):466 if dim_inits[i][1]==0:467 dim_inits[i] = ('time', len(dt))468 dim_inits = tuple(dim_inits)469 # print dim_inits470 471 # replace name of dependent dimension names with speificied names for CDL v2.0 in variables472 # e.g. ntime for time473 var_inits = list(var_inits)474 for j in range(len(dees)):475 for i in range(len(var_inits)):476 v = list(var_inits[i])477 dl = list(v[2])478 for k in range(len(dl)):479 if dl[k]==dees[j][0]:480 dl[k]=dees[j][1] # 'time'481 var_inits[i]=(v[0], v[1], tuple(dl))482 var_inits = tuple(var_inits)483 # print var_inits484 485 # subset data486 varNames = [vn for vn, vt, vd in var_inits]487 var_data = list(var_data)488 for i in range(len(varNames)):489 vn, vd = var_data[i]490 if vn in list_of_record_vars:491 var_data[i]=(vn, vd[idx])492 var_data = tuple(var_data)493 global_atts['start_date'] = dt[0].strftime('%Y-%m-%d %H:%M:%S')494 495 # write latest data496 si['latest_filename'] = 'nccoos_%s_%s_latest.nc' % (platform, package)497 ofn = os.path.join(si['latest_dir'], si['latest_filename'])498 d = (global_atts, var_atts, dim_inits, var_inits, var_data)499 500 nc_create(ofn, d)501 502 503 424 # globals 504 425 start_dt = datetime.utcnow()