Changeset 489

Timestamp:

04/27/12 09:56:48

Author:

haines

Message:

added new 'spin' function in raw2proc()

Files:

• raw2proc/trunk/raw2proc/jpier_config_20050425.py (modified) (1 diff)
• raw2proc/trunk/raw2proc/raw2proc.py (modified) (19 diffs)

raw2proc/trunk/raw2proc/jpier_config_20050425.py

@@ -11 +11 @@
-            'config_start_date' : '2005-04-25 00:00:00',
-            'config_end_date' : '2008-04-11 00:00:00', # None or yyyy-mm-dd HH:MM:SS
-
+,
-            }
-sensor_info = {

raw2proc/trunk/raw2proc/raw2proc.py

@@ -1 +1 @@
-#!/usr/bin/env python
-17 13:22:05
+27 09:09:11
-"""Process raw data to monthly netCDF data files
-
@@ -80 +80 @@
-    return attr
-
+def get_config_dates(pi):
+    """ Get datetime of both start and end setting within config file
+
+    Example
+    -------
+        >>> pi = get_config(cn+'.platform_info')
+        >>> (config_start_dt, config_end_dt) = get_config_dates(pi)
+
+    """
+    now_dt = datetime.utcnow()
+    now_dt.replace(microsecond=0)
+    if pi['config_start_date']:
+        config_start_dt = filt_datetime(pi['config_start_date'])
+    elif pi['config_start_date'] == None:
+        config_start_dt = now_dt
+    if pi['config_end_date']:
+        config_end_dt = filt_datetime(pi['config_end_date'])
+    elif pi['config_end_date'] == None:
+        config_end_dt = now_dt
+    return (config_start_dt, config_end_dt)
+
-def find_configs(platform, yyyy_mm, config_dir=''):
-    """Find which configuration files for specified platform and month
@@ -98 +119 @@
-    configs = glob.glob(os.path.join(config_dir, platform + '_config_*.py'))
-    configs.sort()
-    now_dt = datetime.utcnow()
-    now_dt.replace(microsecond=0)
-    # determine when month starts and ends
-    (prev_month, this_month, next_month) = find_months(yyyy_mm)
@@ -108 +127 @@
-    cns = []
-    for config in configs:
-        # datetime from filename 
-        cn = os.path.splitext(os.path.basename(config))[0]
-        cndt = filt_datetime(os.path.basename(config))
-        pi = get_config(cn+'.platform_info')
-
-
-
-
-
-
-
-
-
+
-        if (config_start_dt <= month_start_dt or config_start_dt <= month_end_dt) and \
-               (config_end_dt >= month_start_dt or config_end_dt >= month_end_dt):
@@ -127 +136 @@
-
-
-''
+defconfigs
-    """Find which configuration files are active
-
@@ -138 +147 @@
-    # list of all config files 
-    configs = glob.glob(os.path.join(config_dir, '*_config_*.py'))
-    now_dt = datetime.utcnow()
-    now_dt.replace(microsecond=0)
-    # 
-    cns = []
-    for config in configs:
-        # datetime from filename 
-        cn = os.path.splitext(os.path.basename(config))[0]
-        cndt = filt_datetime(os.path.basename(config))
-        pi = get_config(cn+'.platform_info')
-        if pi['config_end_date'] == None:
@@ -152 +157 @@
-
-
+def uniqify(seq):
+    seen = {}
+    result = []
+    for item in seq:
+        # in old Python versions:
+        # if seen.has_key(item)
+        # but in new ones:
+        if item in seen: continue
+        seen[item] = 1
+        result.append(item)
+    return result
+                                                       
+
+def get_all_platforms(config_dir=defconfigs):
+    """Get all platform ids
+
+    :Returns:
+       pids : list of str
+           Sorted list of all the platforms
+    """
+    import glob
+    # list of all config files 
+    configs = glob.glob(os.path.join(config_dir, '*_config_*.py'))
+    configs.sort()
+    pids = []
+    for config in configs:
+        # datetime from filename 
+        cn = os.path.splitext(os.path.basename(config))[0]
+        pi = get_config(cn+'.platform_info')
+        if pi['id']:
+            pids.append(pi['id'])
+    pids = uniqify(pids)
+    pids.sort()
+    return pids
+
+def get_all_packages(platform, config_dir=defconfigs):
+    """Get all package ids -- all defined packages in sensor_info{} from all configs for the platform
+
+    :Returns:
+       sids : list of str
+           Sorted list of all the sensor ids for package
+    """
+    import glob
+    # list of all config files 
+    configs = glob.glob(os.path.join(config_dir, platform + '_config_*.py'))
+    configs.sort()
+    #
+    sids = []
+    for config in configs:
+        cn = os.path.splitext(os.path.basename(config))[0]
+        pi = get_config(cn+'.platform_info')
+        sids.extend(list(pi['packages']))
+    sids = uniqify(sids)
+    sids.sort()
+    return sids
+
+def get_all_platform_configs(platform, config_dir=defconfigs):
+    """Get all the config files for a platform
+
+    :Returns:
+       cns : list of config names
+           Sorted list of all the sensor ids for package
+    """
+    import glob
+    # list of all config files 
+    configs = glob.glob(os.path.join(config_dir, platform + '_config_*.py'))
+    configs.sort()
+    #
+    cns = []
+    for config in configs:
+        cn = os.path.splitext(os.path.basename(config))[0]
+        cns.append(cn)
+    return cns
+
+def get_config_packages(cn):
+    """ Get active packages set in platform_info{} from specific config file
+
+    :Returns:
+       sids : list of str
+           Sorted (default) or unsorted list of all the sensor ids for package
+           If empty [], no platform ids were found
+    """
+    pi = get_config(cn+'.platform_info')
+    sids = list(pi['packages'])
+    return sids
+
+def list_months(dts, dte):
+    """ list of datetimes for all months inclusively within given date range 
+    
+    """
+    lom = []
+    if type(dts) == type(dte) == type(datetime.utcnow()) and dts <= dte:
+        years = range(dts.year,dte.year+1)
+        for yyyy in years:
+            if yyyy > dts.year:
+                a = 1
+            else:
+                a = dts.month
+            if yyyy < dte.year:
+                b = 12
+            else:
+                b = dte.month
+            months = range(a, b+1)
+            for mm in months:
+                lom.append(datetime(yyyy, mm, 1).strftime('%Y_%m'))
+    else:
+        print "list_months requires two inputs type datetime.datetime and dts<dte"
+    return lom
+    
+
+def create_spin_list(plats, packs, dates, config_dir=defconfigs):
+    """ create list of params needed to run manual() mutiple ways
+
+    :Returns:
+       spin_list : list of three-tuple each tuple with form (platform, package, yyyy_mm)
+
+    Notes
+    -----
+
+    1. plats -- 'ALL' or ['b1', 'b2']
+    2. packs -- 'ALL' or ['ctd1', 'ctd2']
+    3. dates -- 'ALL' or ['2011_11', '2011_12'] or [dt.datetime(2006,1,1), dt.nowutc()]
+
+    For each platform determin packages for given dates
+    also a good way to get listing platforms and packages for specified dates
+
+    """
+    result = []
+    platforms = []
+    if type(plats) == str:
+        if plats.upper() == 'ALL':
+            platforms = get_all_platforms()
+        else:
+            platforms = [plats] # make one platform iterable
+        
+    print ' Expanded lists for creating spin_list:'
+    print ' ...  platform ids : %s' % platforms
+
+    for platform in platforms:
+        if len(platforms)>1:
+            print '------------------------------------'
+            print ' ... ... platform : %s ' % platform
+        packages = []
+        if type(packs) == str:
+            if packs.upper() == 'ALL':
+                packages = get_all_packages(platform)
+            else:
+                packages = [packs] # make one package iterable
+
+        print ' ... ... packages : %s' % packages
+        for package in packages:
+            # dates is a string 'ALL' or format 'YYYY_MM'
+            months = []
+            if type(dates) == str:
+                if dates.upper() == 'ALL':
+                    cns = get_all_platform_configs(platform)
+                    months = []
+                    for cn in cns:
+                        pi = get_config(cn+'.platform_info')
+                        (dts, dte) = get_config_dates(pi)
+                        if package in pi['packages']:
+                            months.extend(list_months(dts, dte))
+                else:
+                    months = [dates] # make on date iterable
+            # dates is a list
+            if type(dates) == type([]):
+                # if dates has two datetime types
+                if type(dates[0]) == type(dates[1]) == type(datetime.utcnow()):
+                    dt1, dt2 = dates
+                    cns = get_all_platform_configs(platform)
+                    months = []
+                    for cn in cns:
+                        pi = get_config(cn+'.platform_info')
+                        (dts, dte) = get_config_dates(pi)
+
+                        if dts<=dt1 and dt1<=dte: a = dt1
+                        elif dt1<=dts and dt1<=dte: a = dts
+
+                        if dts<dt2 and dt2<=dte: b = dt2
+                        elif dts<dt2 and dte<=dt2: b = dte
+
+                        if dte<dt1 or dt2<dts:
+                            continue
+                        # list only months that are in configs for wide date range
+                        if package in pi['packages']:
+                            months.extend(list_months(a,b))
+                # else if string in list
+                elif type(dates[0]) == str:
+                    months = dates
+            print ' ... ...   months : %s' % months
+            for month in months:
+                # print '... ... %s %s %s' % (platform, package, month)
+                result.append((platform, package, month))
+                
+    return result
+                
-def find_raw(si, yyyy_mm):
-    """Determine which list of raw files to process for month """
@@ -197 +398 @@
-def which_raw(pi, raw_files, dts):
-    """Further limit file names based on configuration file timeframe """
-
-
-
-
-
-
-
-
-
-
-
-
+
-
-    for idx, fn in enumerate(raw_files):
@@ -238 +428 @@
-    :Parameters:
-       proctype : string
-
+ or 'spin'
-
-       platform : string
@@ -251 +441 @@
-    >>> raw2proc(proctype='manual', platform='bogue', package='adcp', yyyy_mm='2007_06')
-    >>> raw2proc('manual', 'bogue', 'adcp', '2007_06')
+
+    Spin
+    ----
+    platform can be list of platforms or 'ALL'
+    package can be list packages or 'ALL'
+    yyyy_mm can be list of months, or datetime range
+    
+    >>> raw2proc('spin', ['b1','b2'], ['ctd1', 'ctd2'], ['2011_11'])
+    >>> raw2proc('spin', ['b1','b2'], ['ctd1', 'ctd2'], 'ALL')
+    >>> raw2proc('spin', ['b1','b2'], ['ctd1', 'ctd2'], [datetime(2011,11,1), datetime(2012,4,1)])
+    >>> raw2proc('spin', ['b1','b2'], 'ALL', 'ALL')
+    
+    Not a good idea but this will reprocess all the data from level0 
+    >>> raw2proc('spin', 'ALL', 'ALL', 'ALL')
-          
-    """
@@ -260 +464 @@
-    elif proctype == 'manual':
-        if platform and package and yyyy_mm:
-ly
+-mode
-            print ' ...  platform id : %s' % platform
-            print ' ... package name : %s' % package
@@ -269 +473 @@
-            print 'raw2proc: Manual operation requires platform, package, and month'
-            print "   >>> raw2proc(proctype='manual', platform='bogue', package='adcp', yyyy_mm='2007_07')"
+    elif proctype == 'spin':
+        if platform and package and yyyy_mm:
+            print 'Processing in spin-mode ...'
+            print ' ...  platform ids : %s' % platform
+            print ' ... package names : %s' % package
+            print ' ...        months : %s' % yyyy_mm
+            print ' ...   starting at : %s' % start_dt.strftime("%Y-%m-%d %H:%M:%S UTC")
+            spin_list = create_spin_list(platform, package, yyyy_mm)
+            spin(spin_list)
+        else:
+            print "raw2proc: Spin operation requires platform(s), package(s), and month(s)"
+            print "   >>> raw2proc(proctype='spin', platform='b1', package='ALL', yyyy_mm='ALL')"
+            print "   >>> raw2proc(proctype='spin', platform='ALL', package='met', yyyy_mm='2011_11')"
+            print "   >>> raw2proc('spin', ['b1','b2'], ['ctd1', 'ctd2'], [datetime(2011,11,1), datetime(2012,4,1)])"
+
-    else:
-        print 'raw2proc: requires either auto or manual operation'
@@ -291 +510 @@
-           
-    """
-    now_dt = datetime.utcnow()
-    now_dt.replace(microsecond=0)
-
-    yyyy_mm = this_month()
-    months = find_months(yyyy_mm)
@@ -307 +523 @@
-            asi = get_config(cn+'.sensor_info')
-            platform = pi['id']
-
-
-
-
-
-
-
-
+
+
-            # for each sensor package
-            for package in asi.keys():
@@ -352 +562 @@
-        print ' ... ... NOTE: No active platforms'
-
+def spin(spin_list):
+    """ wrapper to run manual() for multiple months"""
+    for item in spin_list:
+        platform, package, yyyy_mm = item
+        raw2proc('manual',platform, package, yyyy_mm)
+
-def manual(platform, package, yyyy_mm):
-    """Process data for specified platform, sensor package, and month
@@ -363 +579 @@
-               which raw files
-    """
-    now_dt = datetime.utcnow()
-    now_dt.replace(microsecond=0)
-
-    months = find_months(yyyy_mm)
-    month_start_dt = months[1]
@@ -378 +591 @@
-            print ' ... config file : %s' % cn
-            pi = get_config(cn+'.platform_info')
-
-
-
-
-
-
-
-
+
-            # month start and end dt to pi info
-            asi = get_config(cn+'.sensor_info')
@@ -424 +630 @@
-        print ' ... ... ... NOTE: %s not operational for %s' % (platform, yyyy_mm)
-    
+
-def process(pi, si, raw_files, yyyy_mm):
-    # tailored data processing for different input file formats and control over output
@@ -439 +646 @@
-
-            for index, val in enumerate(data['dt']):
-
+
+
+
+
-                    data['in'][index] = True
-