Index: raw2proc/trunk/raw2proc/bogue_config_20080430.py =================================================================== --- raw2proc/trunk/raw2proc/bogue_config_20080430.py (revision 167) +++ raw2proc/trunk/raw2proc/bogue_config_20080430.py (revision 179) @@ -9,5 +9,5 @@ # 'config_start_date' : '2008-04-30 16:00:00', - 'config_end_date' : None, # None or yyyy-mm-dd HH:MM:SS + 'config_end_date' : '2008-07-02 00:00:00', # None or yyyy-mm-dd HH:MM:SS 'packages' : ('adcp', 'adcpwaves'), } @@ -24,4 +24,6 @@ 'transducer_ht' : 0.5, # meters above the bottom 'blanking_ht' : 1.6, # meters above transducer + 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + 'latest_vars' : ('time','lat','lon','z','u','v'), }, 'adcpwaves' : {'id' : 'adcpwaves', @@ -34,4 +36,6 @@ 'ndir' : 90., 'nfreq' : 128., + 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + 'latest_vars' : ('time','lat','lon','z','Tp','Hs'), }, } Index: raw2proc/trunk/raw2proc/jpier_config_20050425.py =================================================================== --- raw2proc/trunk/raw2proc/jpier_config_20050425.py (revision 147) +++ raw2proc/trunk/raw2proc/jpier_config_20050425.py (revision 179) @@ -8,5 +8,5 @@ # 'config_start_date' : '2005-04-25 00:00:00', - 'config_end_date' : None, # None or yyyy-mm-dd HH:MM:SS + 'config_end_date' : '2008-04-15 00:00:00', # None or yyyy-mm-dd HH:MM:SS 'packages' : ('met'), } @@ -16,5 +16,5 @@ 'raw_dir' : '/seacoos/data/nccoos/level0/jpier/met/', 'utc_offset' : 4, # hours offset to utc - 'raw_file_glob' : '*', + 'raw_file_glob' : '*.jpierMet.stats', 'proc_dir' : '/seacoos/data/nccoos/level1/jpier/met/', 'process_module' : 'proc_jpier_ascii_met', Index: raw2proc/trunk/raw2proc/jpier_config_20080722.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/jpier_config_20080722.py (revision 179) @@ -1,0 +1,22 @@ +platform_info = { + 'id' : 'jpier', + 'location' : 'Jennettes Pier, Nags Head, NC', + 'lat' : 35.9101, # degrees true (-) south, (+) north + 'lon' : 75.5958, # degrees true (-) west, (+) east + 'mvar' : -10.83333, # degrees (-) west, (+) east + 'institution' : 'nccoos', + # + 'config_start_date' : '2008-07-22 00:00:00', + 'config_end_date' : None, # None or yyyy-mm-dd HH:MM:SS + 'packages' : ('met'), + } +sensor_info = { + 'met' : { 'id' : 'met', + 'description' : 'Met data', + 'raw_dir' : '/seacoos/data/nccoos/level0/jpier/met/', + 'utc_offset' : 4, # hours offset to utc + 'raw_file_glob' : '*.jpierMet.stats', + 'proc_dir' : '/seacoos/data/nccoos/level1/jpier/met/', + 'process_module' : 'proc_jpier_ascii_met', + }, + } Index: raw2proc/trunk/raw2proc/ncutil.py =================================================================== --- raw2proc/trunk/raw2proc/ncutil.py (revision 101) +++ raw2proc/trunk/raw2proc/ncutil.py (revision 179) @@ -1,4 +1,4 @@ #!/usr/bin/env python -# Last modified: Time-stamp: <2008-01-08 16:10:22 haines> +# Last modified: Time-stamp: <2008-05-19 11:54:11 haines> """ Create, update and load utilities for netcdf files @@ -15,5 +15,8 @@ :Parameters: ncFile : string - Path and name of file to create + Path and name of file to create + (global_atts, var_atts, dim_inits, var_inits, var_data) : tuple + Global Attributes, Variable Attributes, Dimensions, Variable Dimensions, and Data + Everything you need to create a netCDF file. """ try: @@ -74,4 +77,7 @@ ncFile : string Path and name of file to create + (global_atts, var_atts, var_data) : tuple + Global Attributes, Variable Attributes and Data + Everything you need to update a netCDF file. """ try: @@ -125,4 +131,6 @@ es = nc.var('time')[:] units = nc.var('time').units + else: + print "time variable not found in ", ncFile nc.close() return (es, units) @@ -131,8 +139,19 @@ - - -def nc_load(ncFile, nameType='variable_name', - varNames='all', ga_flag=True, va_flag=True): +def nc_find_record_vars(ncFile): + """find which variable are record variables""" + try: + nc = CDF(ncFile) + ncvars = nc.variables() + # list which variables is a record variable + var_list = [varName for varName in ncvars.keys() if nc.var(varName).isrecord()] + nc.close() + return var_list + except CDFError, msg: + print "CDFError:", msg + + +def nc_load(ncFile, varsLoad='all', nameType='variable_name', + ga_flag=True, va_flag=True): """ Load netcdf file @@ -148,5 +167,5 @@ in file. 'standard_name' means use variable name based on variable attribute called 'standard_name' of netcdf variable. - varNames : string or tuple of strings + varLoad : string or tuple of strings specific variable names to be loaded into a sequence or scalar in python following specification set in nameType @@ -157,16 +176,79 @@ By default, load the variable file attributes - """ - try: - nc = CDF(ncFile) - attr = nc.attributes(full=1) - dims = nc.dimensions(full=1) + :Returns: + (global_atts, var_atts, dim_inits, var_inits, var_data) : tuple + Global Attributes, Variable Attributes, Dimensions, Variable Dimensions, and Variable Data + Everything you need to create a netCDF file. + + """ + try: + nc = CDF(ncFile, NC.NOWRITE) + + ncdims = nc.dimensions(full=1) ncvars = nc.variables() - for var in ncvars.keys(): - # load each variable by name?? - pass - - except CDFError, msg: - print "CDFError:", msg - - + + # GLOBAL ATTRIBUTES (global_atts) + if ga_flag: + global_atts = nc.attributes() + else: + global_atts = {} + + # DIMENSIONS (dim_inits) + dim_inits = [None for j in range(len(ncdims))] + if len(ncdims)>0: + for dimName,dimValue in ncdims.items(): + val,idx,isUN = dimValue + if isUN: + dim_inits[idx] = (dimName, NC.UNLIMITED) + else: + dim_inits[idx] = (dimName, val) + + if varsLoad == 'all': + varNames = ncvars.keys() + else: + varNames = varsLoad + + # VARIABLE DIMENSIONS (var_inits) + # gets init info for requested variables and original order + + # initialize with same number of original variables + # so order can be preserved by idx + var_inits = [None for j in range(len(ncvars))] + if len(ncvars)>0: + for varName in varNames: + val,shape,typ,idx = ncvars[varName] + var_inits[idx] = (varName, typ, val) + + # remove the None values from the list to preserve original order + var_inits = [v for v in var_inits if v != None] + + # VARIABLE ATTRIBUTES (var_atts) + # gets attributes of requested variables + var_atts = {} + if len(ncvars)>0 and va_flag: + for var in varNames: + varAttrs = nc.var(var).attributes() + var_atts[var] = varAttrs + + # VARIABLE DATA (var_data) + # loads requested variables, original order preserved as with var_inits + var_data = [None for j in range(len(ncvars))] + if len(ncvars)>0: + for varName in varNames: + val,shape,typ,idx = ncvars[varName] + var_data[idx] = (varName, nc.var(varName)[:]) + + var_data = [v for v in var_data if v != None] + + # type cast lists into tuples + dim_inits = tuple(dim_inits) + var_inits = tuple(var_inits) + var_data = tuple(var_data) + + nc.close() + return (global_atts, var_atts, dim_inits, var_inits, var_data) + + except CDFError, msg: + print "CDFError:", msg + + Index: raw2proc/trunk/raw2proc/proc_jpier_ascii_met.py =================================================================== --- raw2proc/trunk/raw2proc/proc_jpier_ascii_met.py (revision 144) +++ raw2proc/trunk/raw2proc/proc_jpier_ascii_met.py (revision 179) @@ -1,4 +1,4 @@ #!/usr/bin/env python -# Last modified: Time-stamp: <2008-02-15 10:51:00 jcleary> +# Last modified: Time-stamp: <2008-07-23 13:38:53 haines> """ how to parse data, and assert what data and info goes into @@ -48,5 +48,5 @@ # drop header row from incoming lines list if lines[0].startswith('year',0,5): - print "... Header row present, skipping ..." + # print "... Header row present, skipping ..." del lines[0] Index: raw2proc/trunk/raw2proc/raw2proc.py =================================================================== --- raw2proc/trunk/raw2proc/raw2proc.py (revision 167) +++ raw2proc/trunk/raw2proc/raw2proc.py (revision 179) @@ -1,4 +1,4 @@ #!/usr/bin/env python -# Last modified: Time-stamp: <2008-05-07 13:03:19 haines> +# Last modified: Time-stamp: <2008-07-23 15:43:33 haines> """Process raw data to monthly netCDF data files @@ -31,5 +31,5 @@ # define config file location to run under cron -defconfigs='/home/haines/nccoos/test/r2p' +defconfigs='/home/haines/nccoos/raw2proc' import numpy @@ -178,5 +178,5 @@ # change dt_start to before monthly filename filt_datetime() date dt_start = si['proc_start_dt']-timedelta(days=31) - print dt_start + # print dt_start # end JC changes if fndt: @@ -200,7 +200,12 @@ elif pi['config_end_date'] == None: config_end_dt = now_dt - + new_list = [raw_files[i] for i in range(len(raw_files)) \ if config_start_dt <= dts[i] <= config_end_dt] + + if not new_list: + new_list = [raw_files[i] for i in range(len(raw_files)) \ + if dts[i] <= config_end_dt] + return new_list @@ -300,5 +305,13 @@ (raw_files, raw_dts) = find_raw(si, yyyy_mm) raw_files = which_raw(pi, raw_files, raw_dts) - process(pi, si, raw_files, yyyy_mm) + if raw_files: + process(pi, si, raw_files, yyyy_mm) + else: + print ' ... ... \nNOTE: no raw files found for %s %s for %s\n' % (package, platform, yyyy_mm) + + # update latest data for SECOORA commons + if 'latest_dir' in si.keys(): + print ' ... ... latest : %s ' % si['latest_dir'] + proc2latest(pi, si, yyyy_mm) # else: @@ -341,9 +354,14 @@ (raw_files, raw_dts) = find_raw(si, yyyy_mm) raw_files = which_raw(pi, raw_files, raw_dts) + # print raw_files # remove any previous netcdf file (platform_package_yyyy_mm.nc) if index==0 and os.path.exists(ofn): os.remove(ofn) # - process(pi, si, raw_files, yyyy_mm) + if raw_files: + process(pi, si, raw_files, yyyy_mm) + else: + print ' ... ... \nNOTE: no raw files found for %s %s for %s\n' % (package, platform, yyyy_mm) + else: print ' ... ... \nNOTE: %s not operational on %s for %s\n' % (package, platform, yyyy_mm) @@ -364,4 +382,5 @@ n = len(data['dt']) data['in'] = numpy.array([False for i in range(n)]) + for index, val in enumerate(data['dt']): if val>si['proc_start_dt'] and val<=si['proc_end_dt']: @@ -371,5 +390,5 @@ if data['in'].any(): sys.stdout.write('... %s ... ' % fn) - sys.stdout.write('%d\n' % len(data['in'])) + sys.stdout.write('%d\n' % len(data['in'].nonzero()[0])) ofn = os.path.join(si['proc_dir'], si['proc_filename']) # update or create netcdf @@ -385,4 +404,56 @@ +def proc2latest(pi, si, yyyy_mm): + """Select specific variables and times from current monthly netCDF + and post as latest data. TEST MODE. + + For each active config file, load specific variables from NCCOOS + monthly netCDF, make any necessary changes to data or attributes + conform to SEACOOS Data Model, subset data (last 48 hours), and + create new netCDF file in latest netCDF directory. + + NOTE: In test mode right now. See auto() function for similar action. + + """ + platform = pi['id'] + package = si['id'] + si['proc_filename'] = '%s_%s_%s.nc' % (platform, package, yyyy_mm) + ifn = os.path.join(si['proc_dir'], si['proc_filename']) + if os.path.exists(ifn): + # get dt from current month file + (es, units) = nc_get_time(ifn) + dt = [es2dt(e) for e in es] + last_dt = dt[-1] + + # determine which index of data is within the specified timeframe (last 2 days) + n = len(dt) + idx = numpy.array([False for i in range(n)]) + for i, val in enumerate(dt): + if val>last_dt-timedelta(days=2) and val<=last_dt+timedelta(seconds=360): + idx[i] = True + dt = numpy.array(dt) + dt = dt[idx] + + # read in data and unpack tuple + d = nc_load(ifn, si['latest_vars']) + global_atts, var_atts, dim_inits, var_inits, var_data = d + list_of_record_vars = nc_find_record_vars(ifn) + # subset data + varNames = [vn for vn, vt, vd in var_inits] + var_data = list(var_data) + for i in range(len(varNames)): + vn, vd = var_data[i] + if vn in list_of_record_vars: + var_data[i]=(vn, vd[idx]) + var_data = tuple(var_data) + global_atts['start_date'] = dt[0].strftime('%Y-%m-%d %H:%M:%S') + + # write latest data + si['latest_filename'] = 'nccoos_%s_%s_latest.nc' % (platform, package) + ofn = os.path.join(si['latest_dir'], si['latest_filename']) + d = (global_atts, var_atts, dim_inits, var_inits, var_data) + nc_create(ofn, d) + + # globals start_dt = datetime.utcnow()