Changeset 179
- Timestamp:
- 07/23/08 15:55:24
- Files:
-
- raw2proc/trunk/raw2proc/bogue_config_20080430.py (modified) (3 diffs)
- raw2proc/trunk/raw2proc/jpier_config_20050425.py (modified) (2 diffs)
- raw2proc/trunk/raw2proc/jpier_config_20080722.py (added)
- raw2proc/trunk/raw2proc/ncutil.py (modified) (7 diffs)
- raw2proc/trunk/raw2proc/proc_jpier_ascii_met.py (modified) (2 diffs)
- raw2proc/trunk/raw2proc/raw2proc.py (modified) (9 diffs)
Legend:
- Unmodified
- Added
- Removed
- Modified
- Copied
- Moved
raw2proc/trunk/raw2proc/bogue_config_20080430.py
r167 r179 9 9 # 10 10 'config_start_date' : '2008-04-30 16:00:00', 11 'config_end_date' : None, # None or yyyy-mm-dd HH:MM:SS11 'config_end_date' : '2008-07-02 00:00:00', # None or yyyy-mm-dd HH:MM:SS 12 12 'packages' : ('adcp', 'adcpwaves'), 13 13 } … … 24 24 'transducer_ht' : 0.5, # meters above the bottom 25 25 'blanking_ht' : 1.6, # meters above transducer 26 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', 27 'latest_vars' : ('time','lat','lon','z','u','v'), 26 28 }, 27 29 'adcpwaves' : {'id' : 'adcpwaves', … … 34 36 'ndir' : 90., 35 37 'nfreq' : 128., 38 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', 39 'latest_vars' : ('time','lat','lon','z','Tp','Hs'), 36 40 }, 37 41 } raw2proc/trunk/raw2proc/jpier_config_20050425.py
r147 r179 8 8 # 9 9 'config_start_date' : '2005-04-25 00:00:00', 10 'config_end_date' : None, # None or yyyy-mm-dd HH:MM:SS10 'config_end_date' : '2008-04-15 00:00:00', # None or yyyy-mm-dd HH:MM:SS 11 11 'packages' : ('met'), 12 12 } … … 16 16 'raw_dir' : '/seacoos/data/nccoos/level0/jpier/met/', 17 17 'utc_offset' : 4, # hours offset to utc 18 'raw_file_glob' : '* ',18 'raw_file_glob' : '*.jpierMet.stats', 19 19 'proc_dir' : '/seacoos/data/nccoos/level1/jpier/met/', 20 20 'process_module' : 'proc_jpier_ascii_met', raw2proc/trunk/raw2proc/ncutil.py
r101 r179 1 1 #!/usr/bin/env python 2 # Last modified: Time-stamp: <2008-0 1-08 16:10:22haines>2 # Last modified: Time-stamp: <2008-05-19 11:54:11 haines> 3 3 """ 4 4 Create, update and load utilities for netcdf files … … 15 15 :Parameters: 16 16 ncFile : string 17 Path and name of file to create 17 Path and name of file to create 18 (global_atts, var_atts, dim_inits, var_inits, var_data) : tuple 19 Global Attributes, Variable Attributes, Dimensions, Variable Dimensions, and Data 20 Everything you need to create a netCDF file. 18 21 """ 19 22 try: … … 74 77 ncFile : string 75 78 Path and name of file to create 79 (global_atts, var_atts, var_data) : tuple 80 Global Attributes, Variable Attributes and Data 81 Everything you need to update a netCDF file. 76 82 """ 77 83 try: … … 125 131 es = nc.var('time')[:] 126 132 units = nc.var('time').units 133 else: 134 print "time variable not found in ", ncFile 127 135 nc.close() 128 136 return (es, units) … … 131 139 132 140 133 134 135 def nc_load(ncFile, nameType='variable_name', 136 varNames='all', ga_flag=True, va_flag=True): 141 def nc_find_record_vars(ncFile): 142 """find which variable are record variables""" 143 try: 144 nc = CDF(ncFile) 145 ncvars = nc.variables() 146 # list which variables is a record variable 147 var_list = [varName for varName in ncvars.keys() if nc.var(varName).isrecord()] 148 nc.close() 149 return var_list 150 except CDFError, msg: 151 print "CDFError:", msg 152 153 154 def nc_load(ncFile, varsLoad='all', nameType='variable_name', 155 ga_flag=True, va_flag=True): 137 156 """ 138 157 Load netcdf file … … 148 167 in file. 'standard_name' means use variable name based on 149 168 variable attribute called 'standard_name' of netcdf variable. 150 var Names: string or tuple of strings169 varLoad : string or tuple of strings 151 170 specific variable names to be loaded into a sequence or scalar 152 171 in python following specification set in nameType … … 157 176 By default, load the variable file attributes 158 177 159 """ 160 try: 161 nc = CDF(ncFile) 162 attr = nc.attributes(full=1) 163 dims = nc.dimensions(full=1) 178 :Returns: 179 (global_atts, var_atts, dim_inits, var_inits, var_data) : tuple 180 Global Attributes, Variable Attributes, Dimensions, Variable Dimensions, and Variable Data 181 Everything you need to create a netCDF file. 182 183 """ 184 try: 185 nc = CDF(ncFile, NC.NOWRITE) 186 187 ncdims = nc.dimensions(full=1) 164 188 ncvars = nc.variables() 165 for var in ncvars.keys(): 166 # load each variable by name?? 167 pass 168 169 except CDFError, msg: 170 print "CDFError:", msg 171 172 189 190 # GLOBAL ATTRIBUTES (global_atts) 191 if ga_flag: 192 global_atts = nc.attributes() 193 else: 194 global_atts = {} 195 196 # DIMENSIONS (dim_inits) 197 dim_inits = [None for j in range(len(ncdims))] 198 if len(ncdims)>0: 199 for dimName,dimValue in ncdims.items(): 200 val,idx,isUN = dimValue 201 if isUN: 202 dim_inits[idx] = (dimName, NC.UNLIMITED) 203 else: 204 dim_inits[idx] = (dimName, val) 205 206 if varsLoad == 'all': 207 varNames = ncvars.keys() 208 else: 209 varNames = varsLoad 210 211 # VARIABLE DIMENSIONS (var_inits) 212 # gets init info for requested variables and original order 213 214 # initialize with same number of original variables 215 # so order can be preserved by idx 216 var_inits = [None for j in range(len(ncvars))] 217 if len(ncvars)>0: 218 for varName in varNames: 219 val,shape,typ,idx = ncvars[varName] 220 var_inits[idx] = (varName, typ, val) 221 222 # remove the None values from the list to preserve original order 223 var_inits = [v for v in var_inits if v != None] 224 225 # VARIABLE ATTRIBUTES (var_atts) 226 # gets attributes of requested variables 227 var_atts = {} 228 if len(ncvars)>0 and va_flag: 229 for var in varNames: 230 varAttrs = nc.var(var).attributes() 231 var_atts[var] = varAttrs 232 233 # VARIABLE DATA (var_data) 234 # loads requested variables, original order preserved as with var_inits 235 var_data = [None for j in range(len(ncvars))] 236 if len(ncvars)>0: 237 for varName in varNames: 238 val,shape,typ,idx = ncvars[varName] 239 var_data[idx] = (varName, nc.var(varName)[:]) 240 241 var_data = [v for v in var_data if v != None] 242 243 # type cast lists into tuples 244 dim_inits = tuple(dim_inits) 245 var_inits = tuple(var_inits) 246 var_data = tuple(var_data) 247 248 nc.close() 249 return (global_atts, var_atts, dim_inits, var_inits, var_data) 250 251 except CDFError, msg: 252 print "CDFError:", msg 253 254 raw2proc/trunk/raw2proc/proc_jpier_ascii_met.py
r144 r179 1 1 #!/usr/bin/env python 2 # Last modified: Time-stamp: <2008-0 2-15 10:51:00 jcleary>2 # Last modified: Time-stamp: <2008-07-23 13:38:53 haines> 3 3 """ 4 4 how to parse data, and assert what data and info goes into … … 48 48 # drop header row from incoming lines list 49 49 if lines[0].startswith('year',0,5): 50 print "... Header row present, skipping ..."50 # print "... Header row present, skipping ..." 51 51 del lines[0] 52 52 raw2proc/trunk/raw2proc/raw2proc.py
r167 r179 1 1 #!/usr/bin/env python 2 # Last modified: Time-stamp: <2008-0 5-07 13:03:19haines>2 # Last modified: Time-stamp: <2008-07-23 15:43:33 haines> 3 3 """Process raw data to monthly netCDF data files 4 4 … … 31 31 32 32 # define config file location to run under cron 33 defconfigs='/home/haines/nccoos/ test/r2p'33 defconfigs='/home/haines/nccoos/raw2proc' 34 34 35 35 import numpy … … 178 178 # change dt_start to before monthly filename filt_datetime() date 179 179 dt_start = si['proc_start_dt']-timedelta(days=31) 180 print dt_start180 # print dt_start 181 181 # end JC changes 182 182 if fndt: … … 200 200 elif pi['config_end_date'] == None: 201 201 config_end_dt = now_dt 202 202 203 203 new_list = [raw_files[i] for i in range(len(raw_files)) \ 204 204 if config_start_dt <= dts[i] <= config_end_dt] 205 206 if not new_list: 207 new_list = [raw_files[i] for i in range(len(raw_files)) \ 208 if dts[i] <= config_end_dt] 209 205 210 return new_list 206 211 … … 300 305 (raw_files, raw_dts) = find_raw(si, yyyy_mm) 301 306 raw_files = which_raw(pi, raw_files, raw_dts) 302 process(pi, si, raw_files, yyyy_mm) 307 if raw_files: 308 process(pi, si, raw_files, yyyy_mm) 309 else: 310 print ' ... ... \nNOTE: no raw files found for %s %s for %s\n' % (package, platform, yyyy_mm) 311 312 # update latest data for SECOORA commons 313 if 'latest_dir' in si.keys(): 314 print ' ... ... latest : %s ' % si['latest_dir'] 315 proc2latest(pi, si, yyyy_mm) 303 316 # 304 317 else: … … 341 354 (raw_files, raw_dts) = find_raw(si, yyyy_mm) 342 355 raw_files = which_raw(pi, raw_files, raw_dts) 356 # print raw_files 343 357 # remove any previous netcdf file (platform_package_yyyy_mm.nc) 344 358 if index==0 and os.path.exists(ofn): 345 359 os.remove(ofn) 346 360 # 347 process(pi, si, raw_files, yyyy_mm) 361 if raw_files: 362 process(pi, si, raw_files, yyyy_mm) 363 else: 364 print ' ... ... \nNOTE: no raw files found for %s %s for %s\n' % (package, platform, yyyy_mm) 365 348 366 else: 349 367 print ' ... ... \nNOTE: %s not operational on %s for %s\n' % (package, platform, yyyy_mm) … … 364 382 n = len(data['dt']) 365 383 data['in'] = numpy.array([False for i in range(n)]) 384 366 385 for index, val in enumerate(data['dt']): 367 386 if val>si['proc_start_dt'] and val<=si['proc_end_dt']: … … 371 390 if data['in'].any(): 372 391 sys.stdout.write('... %s ... ' % fn) 373 sys.stdout.write('%d\n' % len(data['in'] ))392 sys.stdout.write('%d\n' % len(data['in'].nonzero()[0])) 374 393 ofn = os.path.join(si['proc_dir'], si['proc_filename']) 375 394 # update or create netcdf … … 385 404 386 405 406 def proc2latest(pi, si, yyyy_mm): 407 """Select specific variables and times from current monthly netCDF 408 and post as latest data. TEST MODE. 409 410 For each active config file, load specific variables from NCCOOS 411 monthly netCDF, make any necessary changes to data or attributes 412 conform to SEACOOS Data Model, subset data (last 48 hours), and 413 create new netCDF file in latest netCDF directory. 414 415 NOTE: In test mode right now. See auto() function for similar action. 416 417 """ 418 platform = pi['id'] 419 package = si['id'] 420 si['proc_filename'] = '%s_%s_%s.nc' % (platform, package, yyyy_mm) 421 ifn = os.path.join(si['proc_dir'], si['proc_filename']) 422 if os.path.exists(ifn): 423 # get dt from current month file 424 (es, units) = nc_get_time(ifn) 425 dt = [es2dt(e) for e in es] 426 last_dt = dt[-1] 427 428 # determine which index of data is within the specified timeframe (last 2 days) 429 n = len(dt) 430 idx = numpy.array([False for i in range(n)]) 431 for i, val in enumerate(dt): 432 if val>last_dt-timedelta(days=2) and val<=last_dt+timedelta(seconds=360): 433 idx[i] = True 434 dt = numpy.array(dt) 435 dt = dt[idx] 436 437 # read in data and unpack tuple 438 d = nc_load(ifn, si['latest_vars']) 439 global_atts, var_atts, dim_inits, var_inits, var_data = d 440 list_of_record_vars = nc_find_record_vars(ifn) 441 # subset data 442 varNames = [vn for vn, vt, vd in var_inits] 443 var_data = list(var_data) 444 for i in range(len(varNames)): 445 vn, vd = var_data[i] 446 if vn in list_of_record_vars: 447 var_data[i]=(vn, vd[idx]) 448 var_data = tuple(var_data) 449 global_atts['start_date'] = dt[0].strftime('%Y-%m-%d %H:%M:%S') 450 451 # write latest data 452 si['latest_filename'] = 'nccoos_%s_%s_latest.nc' % (platform, package) 453 ofn = os.path.join(si['latest_dir'], si['latest_filename']) 454 d = (global_atts, var_atts, dim_inits, var_inits, var_data) 455 nc_create(ofn, d) 456 457 387 458 # globals 388 459 start_dt = datetime.utcnow()