Changeset 320

Timestamp:

04/27/10 11:55:31

Author:

haines

Message:

catch-up trunk to production code running on cromwell

Files:

• raw2proc/trunk/raw2proc/billymitchell_config_20090623.py (modified) (1 diff)
• raw2proc/trunk/raw2proc/bogue_config_20090915.py (added)
• raw2proc/trunk/raw2proc/crow_config_20050325.py (modified) (1 diff)
• raw2proc/trunk/raw2proc/crow_config_20090122.py (modified) (2 diffs)
• raw2proc/trunk/raw2proc/dukeforest_config_20070514.py (modified) (1 diff)
• raw2proc/trunk/raw2proc/hampton_config_20080930.py (modified) (1 diff)
• raw2proc/trunk/raw2proc/ims_config_20070920.py (modified) (1 diff)
• raw2proc/trunk/raw2proc/meet_config_20010510.py (modified) (3 diffs)
• raw2proc/trunk/raw2proc/meet_config_20040305.py (added)
• raw2proc/trunk/raw2proc/meet_config_20050324.py (modified) (1 diff)
• raw2proc/trunk/raw2proc/meet_config_20090122.py (modified) (1 diff)
• raw2proc/trunk/raw2proc/morgan_config_20080701.py (modified) (1 diff)
• raw2proc/trunk/raw2proc/ncutil.py (modified) (5 diffs)
• raw2proc/trunk/raw2proc/proc_avp_ysi_6600_v1_CDL2.py (modified) (8 diffs)
• raw2proc/trunk/raw2proc/proc_avp_ysi_6600_v2_CDL2.py (modified) (9 diffs)
• raw2proc/trunk/raw2proc/proc_cr1000_flow.py (modified) (3 diffs)
• raw2proc/trunk/raw2proc/proc_cr1000_wq.py (modified) (2 diffs)
• raw2proc/trunk/raw2proc/proc_cr10x_flow_v1.py (modified) (4 diffs)
• raw2proc/trunk/raw2proc/proc_cr10x_flow_v2.py (modified) (4 diffs)
• raw2proc/trunk/raw2proc/proc_cr10x_wq_v1.py (modified) (2 diffs)
• raw2proc/trunk/raw2proc/proc_cr10x_wq_v2.py (modified) (2 diffs)
• raw2proc/trunk/raw2proc/proc_jpier_ascii_met.py (modified) (4 diffs)
• raw2proc/trunk/raw2proc/proc_nortek_wds_dw.py (modified) (2 diffs)
• raw2proc/trunk/raw2proc/proc_nortek_wpa_adcp.py (modified) (3 diffs)
• raw2proc/trunk/raw2proc/procutil.py (modified) (5 diffs)
• raw2proc/trunk/raw2proc/raw2proc.py (modified) (2 diffs)
• raw2proc/trunk/raw2proc/spin_bogue_adcp.py (deleted)
• raw2proc/trunk/raw2proc/spin_bogue_adcpwaves.py (deleted)
• raw2proc/trunk/raw2proc/spin_crow_csv.py (deleted)
• raw2proc/trunk/raw2proc/spin_crow_flow.py (deleted)
• raw2proc/trunk/raw2proc/spin_crow_wq.py (deleted)
• raw2proc/trunk/raw2proc/spin_hampton_avp.py (deleted)
• raw2proc/trunk/raw2proc/spin_hampton_met.py (deleted)
• raw2proc/trunk/raw2proc/spin_jpier_adcp.py (deleted)
• raw2proc/trunk/raw2proc/spin_jpier_met.py (deleted)
• raw2proc/trunk/raw2proc/spin_lsrb_adcp.py (deleted)
• raw2proc/trunk/raw2proc/spin_meet_csv.py (deleted)
• raw2proc/trunk/raw2proc/spin_meet_flow.py (deleted)
• raw2proc/trunk/raw2proc/spin_meet_wq.py (deleted)
• raw2proc/trunk/raw2proc/spin_morgan_avp.py (deleted)
• raw2proc/trunk/raw2proc/spin_morgan_met.py (deleted)
• raw2proc/trunk/raw2proc/spin_stones_avp.py (deleted)
• raw2proc/trunk/raw2proc/spin_stones_met.py (deleted)
• raw2proc/trunk/raw2proc/split_cr1000_by_month.py (added)
• raw2proc/trunk/raw2proc/split_cr1000_this_month.py (added)
• raw2proc/trunk/raw2proc/split_cr10x_by_month.py (added)
• raw2proc/trunk/raw2proc/test_concat_platform_package_data.py (deleted)
• raw2proc/trunk/raw2proc/test_raw2proc.py (deleted)

raw2proc/trunk/raw2proc/billymitchell_config_20090623.py

@@ -24 +24 @@
-               'num_altitudes' : 39,
-               'sensor_elevation' : 0,   # meters
+               'plot_module' : 'billymitchell_sodar_plot', 
+               'plot_names' : ('timeseries', 'wind_vectors', 'wind_barbs'), 
-               },
-    }

raw2proc/trunk/raw2proc/crow_config_20050325.py

@@ -24 +24 @@
-               'process_module' : 'proc_cr10x_flow_v2',
-               'utc_offset' : 4,      # hours offset to utc
+               'press_offset' : 0./12., # pressure gauge offset to staff gauge
-               # 'nbins' : 69,
-               # 'bin_size' : 0.5,      # meters

raw2proc/trunk/raw2proc/crow_config_20090122.py

@@ -24 +24 @@
-               'process_module' : 'proc_cr1000_flow',
-               'utc_offset' : 4,      # hours offset to utc
+               'press_offset' : 0./12., # pressure gauge offset to staff gauge
+               'plot_module' : 'crow_flow_plot', 
+               'plot_names' : ('timeseries',), 
-               # 'nbins' : 69,
-               # 'bin_size' : 0.5,      # meters
@@ -36 +39 @@
-            'process_module' : 'proc_cr1000_wq',
-            'utc_offset' : 4,  # hours offset to utc
+            'plot_module' : 'crow_wq_plot', 
+            'plot_names' : ('timeseries',), 
-                   },
-    }

raw2proc/trunk/raw2proc/dukeforest_config_20070514.py

@@ -8 +8 @@
-    # 
-    'config_start_date' : '2007-05-14 00:00:00',
-None
+'2007-07-11 00:00:00'
-    'packages' : ('pa0',),
-    }

raw2proc/trunk/raw2proc/hampton_config_20080930.py

@@ -33 +33 @@
-              'utc_offset' : 5.,             # hours offset to Eastern Standard
-              'anemometer_height' : 2.,      # meters
-
-
+# 
+# 
-             },
-    }

raw2proc/trunk/raw2proc/ims_config_20070920.py

@@ -8 +8 @@
-    # 
-    'config_start_date' : '2007-09-20 00:00:00',
-None
+'2008-12-11 00:00:00'
-    'packages' : ('pa0',),
-    }

raw2proc/trunk/raw2proc/meet_config_20010510.py

@@ -13 +13 @@
-    # 
-    'config_start_date' : '2001-05-01 00:00:00',
-12-31 00
+03-05 08
-    'packages' : ('flow', 'wq'),
-    }
@@ -19 +19 @@
-    'flow' : { 'id' : 'flow',
-               'description' : 'Stream Flow and Rain Data',
+               # flow and wq data originially on same sample period so not split
+               # during this configuration period so raw flow data in raw wq
-               'raw_dir' : '/seacoos/data/nccoos/level0/meet/wq/',
-               'raw_file_glob' : 'mow_wq_*',
@@ -24 +26 @@
-               'process_module' : 'proc_cr10x_flow_v1',
-               'utc_offset' : 4,      # hours offset to utc
+               'press_offset' : 0./12., # pressure gauge offset to staff gauge
-               # 'nbins' : 69,
-               # 'bin_size' : 0.5,      # meters

raw2proc/trunk/raw2proc/meet_config_20050324.py

@@ -24 +24 @@
-               'process_module' : 'proc_cr10x_flow_v2',
-               'utc_offset' : 4,      # hours offset to utc
+               'press_offset' : 0./12., # pressure gauge offset to staff gauge
-               # 'nbins' : 69,
-               # 'bin_size' : 0.5,      # meters

raw2proc/trunk/raw2proc/meet_config_20090122.py

@@ -24 +24 @@
-               'process_module' : 'proc_cr1000_flow',
-               'utc_offset' : 4,      # hours offset to utc
-
-
-
-
+
+
+
+
+
-               },
-    'wq' : {'id' : 'wq',
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
-    }
-    

raw2proc/trunk/raw2proc/morgan_config_20080701.py

@@ -12 +12 @@
-            'config_end_date' : None, # None or yyyy-mm-dd HH:MM:SS
-            'packages' : ('avp', 'met'),
-      
+       
-
-sensor_info = {

raw2proc/trunk/raw2proc/ncutil.py

@@ -1 +1 @@
-#!/usr/bin/env python
-8-10-23 11:31:24
+9-12-30 15:45:27
-"""
-Create, update and load utilities for netcdf files
@@ -50 +50 @@
-            varName, varData = var
-            # print varName
+            # print varData
-            ncvar = nc.var(varName)
-            # e.g. lat = array(var_data['lat'])
@@ -199 +200 @@
-                                
-
+def nc_file_check(fns):
+    """Check file or list of files to ensure it is a netcdf file
+    If it is not, remove a file or files from the list"""
+    if isinstance(fns, str):
+        try:
+            nc = CDF(fns)
+            nc.close()
+            new_fns = fns
+        except CDFError, msg:
+            print "CDFError:", msg, fns
+            new_fns = None
+        
+    else:
+        new_fns = []
+        for fn in fns:
+            try:
+                nc = CDF(fn)
+                nc.close()
+                new_fns.append(fn)
+            except CDFError, msg:
+                print "CDFError:", msg, fn
+        
+    return tuple(new_fns)
+    
+
-def nc_load(ncFile, varsLoad='all', nameType='variable_name',
-            ga_flag=True, va_flag=True):
@@ -205 +231 @@
-
-    :Parameters:
-
+ or list of strings
-            Path and name of file to load
+            If list, then CDFMF 
-
-    :Other Parameters:
@@ -229 +256 @@
-
-    """
-
-
+
+
+
+
+
+
+
+
-
-        ncdims = nc.dimensions(full=1)

raw2proc/trunk/raw2proc/proc_avp_ysi_6600_v1_CDL2.py

@@ -1 +1 @@
-#!/usr/bin/env python
-01-08 19:45:28
+12-16 16:33:00
-"""
-how to parse data, and assert what data and info goes into
@@ -133 +133 @@
-        'z' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan),
-        #
-
-
+# 
+# 
-        #
-        'stime' : numpy.array(numpy.ones((N,nbins), dtype=long)*numpy.nan),
@@ -182 +182 @@
-            else:
-                profile_dt = scanf_datetime(profile_str, fmt='%m-%d-%Y %H:%M:%S')
-, re.IGNORECASE
+
-            have_location = True
-            # profile location: P180, Instrument Serial No: 0001119E
-            # Profile Location: Hampton Shoal Serial No: 000109DD, ID: Delta
-            sw = re.findall(r'\w+:\s(\w+)*', line)
-
-
-
-
+
+
-                
-            # initialize for new profile at zero for averaging samples within each bin
@@ -207 +205 @@
-            have_head = head.all()
-
+        elif re.search("Error", line):
+            # ignore this line
+            if verbose:
+                print 'skipping bad data line ... ' + str(line)
+            continue
+
-        elif (len(ysi)==13 and have_head):
-            if j>=nbins:
@@ -239 +243 @@
-            data['dt'][i] = profile_dt # profile datetime
-            data['time'][i] = dt2es(profile_dt) # profile time in epoch seconds
-
-
+# 
+# 
-            #
-            data['stime'][i] =  stime # sample time in epoch seconds
@@ -345 +349 @@
-                  'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC
-                  },
-
-
-
-
-
-
-
-
+# 
+# 
+# 
+# 
+# 
+# 
+# 
+# 
-        'wtemp': {'short_name': 'wtemp',
-                        'long_name': 'Water Temperature',
-                        'standard_name': 'water_temperature',                          
- 
+_
-                        },
-        'cond': {'short_name': 'cond',
@@ -391 +395 @@
-        ('lon', 1),
-        ('z', sensor_info['nbins']),
-        ('nchar', 20),
-        )
-    
@@ -407 +410 @@
-        # ('ysi_sn', NC.CHAR, ('time', 'nchar')),
-        # ('ysi_id', NC.CHAR, ('time', 'nchar')),
-IN
+FLOA
-        ('wtemp', NC.FLOAT, ('time', 'z')),
-        ('cond', NC.FLOAT, ('time', 'z')),

raw2proc/trunk/raw2proc/proc_avp_ysi_6600_v2_CDL2.py

@@ -1 +1 @@
-#!/usr/bin/env python
-01-09 12:07:49
+12-16 15:23:36
-"""
-how to parse data, and assert what data and info goes into
@@ -125 +125 @@
-        'wd' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan),
-        'wl' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan),
-
-
+# 
+# 
-        #
-        'stime' : numpy.array(numpy.ones((N,nbins), dtype=long)*numpy.nan),
@@ -179 +179 @@
-            # Profile Location: Stones Bay Serial No: 00016B79, ID: AVP1_SERDP
-            sw = re.findall(r'\w+:\s(\w+)*', line)
-
-
+# 
+# 
-            # initialize for new profile at zero for averaging samples within each bin
-            wtemp = numpy.array(numpy.ones(nbins,), dtype=float)*numpy.nan
@@ -205 +205 @@
-            # get sample datetime from data
-            sample_str = '%02d-%02d-%02d %02d:%02d:%02d' % tuple(ysi[0:6])
-
-
-
-
+
+
-                sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%y %H:%M:%S')
-
+
+
+
+
+
+
+
+
+
+
-            if j<nbins:
-                stime[j] = dt2es(sample_dt) # sample time
@@ -223 +230 @@
-                chl[j] = ysi[12]  # chlorophyll (ug/l)
-                do[j] = ysi[13]   # dissolved oxygen (mg/l)
-
+
-            j = j+1
-
@@ -232 +239 @@
-            data['wd'][i] = -1.*wd
-            data['wl'][i] = platform_info['mean_water_depth'] - (-1*wd)
-
-
+# 
+# 
-
-            data['stime'][i] =  stime # sample time in epoch seconds
@@ -356 +363 @@
-               'units': 'm',
-               },
-
-
-
-
-
-
-
-
+# 
+# 
+# 
+# 
+# 
+# 
+# 
+# 
-        'wtemp': {'short_name': 'wtemp',
-                        'long_name': 'Water Temperature',
-                        'standard_name': 'water_temperature',                          
- 
+_
-                        },
-        'cond': {'short_name': 'cond',
@@ -407 +414 @@
-        ('lon', 1),
-        ('z', sensor_info['nbins']),
-        ('nchar', 20),
-        )
-    
@@ -425 +431 @@
-        # ('ysi_sn', NC.CHAR, ('time', 'nchar')),
-        # ('ysi_id', NC.CHAR, ('time', 'nchar')),
-IN
+FLOA
-        ('wtemp', NC.FLOAT, ('time', 'z')),
-        ('cond', NC.FLOAT, ('time', 'z')),

raw2proc/trunk/raw2proc/proc_cr1000_flow.py

@@ -1 +1 @@
-#!/usr/bin/env python
-06-25 15:12:53
+12-07 14:32:39
-"""
-how to parse data, and assert what data and info goes into
@@ -74 +74 @@
-        'sontek_wl' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
-        'sontek_flow' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
+        'press' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
-        'press_wl' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
-        'press_flow' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
+        'press_csi_ft' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
+        'press_csi_cfs' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
-        }
-
@@ -110 +113 @@
-        data['time'][i] = dt2es(sample_dt) # sample time in epoch seconds
-        
+        # SMH -- 2009-12-05 modification
+        # press_csi water level and flow conversion on the data logger is not correct
+        # this will be reverted to original pressure reading and wl and flow recomputed.
-        if len(csi)==6:
-            # MOW has all six fields but no sontek now
-            data['rain'][i] =  csi[1] # 15 min rain count (inches)
-
-
-wl'][i] = csi[4] #
-flow'][i] = csi[5] # flow
+# 
+# 
+csi_ft'][i] = csi[4] # csi reported
+csi_cfs'][i] = csi[5] # csi reported
-            i=i+1
-        elif len(csi)==4:
-
-
+
-            data['rain'][i] =  csi[1] # 15 min rain count (inches)
-            data['sontek_wl'][i] = csi[2] # sontek water level (ft) 
-            data['sontek_flow'][i] = csi[3] # sontek flow (cfs)
-            data['press_wl'][i] = 0. # pressure water level (ft) 
-            data['press_flow'][i] = 0. # flow flow (cfs)
-            i=i+1
-        else:
-            print ' ... skipping line %d -- %s ' % (i,line)
-            
+
-
-        # if re.search
-    # for line
+
+    # revert  press_csi_ft back  to raw  pressure  reading (eventually
+    # want csi to just report the  raw pressure reading so we can just
+    # do this ourselves.
+    data['press'] = (data['press_csi_ft']+1.5)/27.6778 # raw pressure (psi)
+    # convert psi to height of water column based on hydrostatic eqn
+    data['press_wl'] = data['press']*2.3059+sensor_info['press_offset'] # (feet)
+    
+    # flow based on parameter as computed by data logger
+    # data['press_flow'] = data['press_csi_cfs']
+                                
+    # flow based on calculation from data logger but applied to offset calibration
+    # SMH does not know what equation is based on or how these values are derived
+    data['press_flow'] = ((data['press_wl']*12))*10.81 - 8.81 # cfs
-
-    # check that no data[dt] is set to Nan or anything but datetime

raw2proc/trunk/raw2proc/proc_cr1000_wq.py

@@ -1 +1 @@
-#!/usr/bin/env python
-07-08 18:05:37
+12-15 08:31:52
-"""
-how to parse data, and assert what data and info goes into
@@ -213 +213 @@
-                  'long_name': 'Water Temperature',
-                  'standard_name': 'water_temperature',                          
- 
+_
-                  },
-        'cond': {'short_name': 'cond',

raw2proc/trunk/raw2proc/proc_cr10x_flow_v1.py

@@ -1 +1 @@
-#!/usr/bin/env python
-03-19 14:44:02
+12-07 15:02:51
-"""
-how to parse data, and assert what data and info goes into
@@ -88 +88 @@
-        'time' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan),
-        'rain' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
+        'press' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
-        'press_wl' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
-        'press_flow' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
+        'press_csi_ft' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
+        'press_csi_cfs' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
-        }
-
@@ -142 +145 @@
-            # data['sontek_flow'][i] = csi[6] # sontek flow (cfs)
-
-wl
+csi_ft
-            data['rain'][i] =  csi[11] # 15 sec rain count ??
-flow
+csi_cfs
-            # data['battvolts'][i] = csi[13]   # battery (volts)
-            
@@ -151 +154 @@
-        # if-elif
-    # for line
+
+    # revert  press_csi_ft back  to raw  pressure  reading (eventually
+    # want csi to just report the  raw pressure reading so we can just
+    # do this ourselves.
+    data['press'] = (data['press_csi_ft']+1.5)/27.6778 # raw pressure (psi)
+    # convert psi to height of water column based on hydrostatic eqn
+    data['press_wl'] = data['press']*2.3059+sensor_info['press_offset'] # (feet)
+    
+    # flow based on parameter as computed by data logger
+    # data['press_flow'] = data['press_csi_cfs']
+                                
+    # flow based on calculation from data logger but applied to offset calibration
+    # SMH does not know what equation is based on or how these values are derived
+    data['press_flow'] = ((data['press_wl']*12))*10.81 - 8.81 # cfs
-
-    # check that no data[dt] is set to Nan or anything but datetime

raw2proc/trunk/raw2proc/proc_cr10x_flow_v2.py

@@ -1 +1 @@
-#!/usr/bin/env python
-03-19 14:30:4
+12-07 15:03:0
-"""
-how to parse data, and assert what data and info goes into
@@ -108 +108 @@
-        'sontek_wl' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
-        'sontek_flow' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
+        'press' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
-        'press_wl' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
-        'press_flow' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
+        'press_csi_ft' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
+        'press_csi_cfs' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan),
-        }
-
@@ -154 +157 @@
-                data['sontek_wl'][i] = csi[5] # sontek water level (ft) 
-                data['sontek_flow'][i] = csi[6] # sontek flow (cfs)
-wl
-flow
+csi_ft
+csi_cfs
-            elif len(csi)==7:
-                # 
@@ -168 +171 @@
-        # if re.search
-    # for line
+    
+    # revert  press_csi_ft back  to raw  pressure  reading (eventually
+    # want csi to just report the  raw pressure reading so we can just
+    # do this ourselves.
+    data['press'] = (data['press_csi_ft']+1.5)/27.6778 # raw pressure (psi)
+    # convert psi to height of water column based on hydrostatic eqn
+    data['press_wl'] = data['press']*2.3059+sensor_info['press_offset'] # (feet)
+    
+    # flow based on parameter as computed by data logger
+    # data['press_flow'] = data['press_csi_cfs']
+                                
+    # flow based on calculation from data logger but applied to offset calibration
+    # SMH does not know what equation is based on or how these values are derived
+    data['press_flow'] = ((data['press_wl']*12))*10.81 - 8.81 # cfs
-
-    # check that no data[dt] is set to Nan or anything but datetime

raw2proc/trunk/raw2proc/proc_cr10x_wq_v1.py

@@ -1 +1 @@
-#!/usr/bin/env python
-8-12-16 16:47:38
+9-12-15 08:32:19
-"""
-how to parse data, and assert what data and info goes into
@@ -222 +222 @@
-                  'long_name': 'Water Temperature',
-                  'standard_name': 'water_temperature',                          
- 
+_
-                  },
-        'cond': {'short_name': 'cond',

raw2proc/trunk/raw2proc/proc_cr10x_wq_v2.py

@@ -1 +1 @@
-#!/usr/bin/env python
-8-12-16 10:55:0
+9-12-15 08:32:3
-"""
-how to parse data, and assert what data and info goes into
@@ -251 +251 @@
-                  'long_name': 'Water Temperature',
-                  'standard_name': 'water_temperature',                          
- 
+_
-                  },
-        'cond': {'short_name': 'cond',

raw2proc/trunk/raw2proc/proc_jpier_ascii_met.py

@@ -1 +1 @@
-#!/usr/bin/env python
-8-10-01 12:45
+9-12-15 08:40
-"""
-how to parse data, and assert what data and info goes into
@@ -198 +198 @@
-                                                'long_name': 'Air Temperature',
-                                                'standard_name': 'air_temperature',
- 
+_
-                                                },
-                'humidity' :    {'short_name': 'humidity',
@@ -208 +208 @@
-                                                'long_name': 'Dew Temperature',
-                                                'standard_name': 'dew_temp',                                              
- 
+_
-                                                },
-                'air_pressure' : {'short_name': 'air_pressure',
@@ -231 +231 @@
-                                                'long_name': 'Wind Chill',
-                                                'standard_name': 'wind_chill',
- 
+_
-                                                },
-                'rainfall_rate' :       {'short_name': 'rR',

raw2proc/trunk/raw2proc/proc_nortek_wds_dw.py

@@ -1 +1 @@
-#!/usr/bin/env python
-8-10-01 12:47:05
+9-10-28 17:27:56
-"""
-how to parse data, and assert what data and info goes into
@@ -300 +300 @@
-        # (polar_waves_cur_wds.m, version 8)
-        pi = numpy.pi
-
-
-
-
-
+1
+1
+
+1
+1
-        Dm = numpy.arctan2(sh0,ch0)*180/pi
-        if Dm<0: Dm = Dm+360. 
-        
-
-
+1
+1
-        Dms = numpy.arctan2(sh0s,ch0s)*180/pi
-        if Dms<0: Dms = Dms+360.
-        
-
-
+1
+1
-        Dmw = numpy.arctan2(sh0w,ch0w)*180/pi
-        if Dmw<0: Dmw = Dmw+360.

raw2proc/trunk/raw2proc/proc_nortek_wpa_adcp.py

@@ -1 +1 @@
-#!/usr/bin/env python
-08-10-16 17:02:54
+10-01-22 12:11:55
-"""
-how to parse data, and assert what data and info goes into
@@ -168 +168 @@
-            bin_number = wpa[0]
-            j = wpa[0]-1
+            print j
-            hab[j] = wpa[1]
-            
@@ -368 +369 @@
-                        'long_name': 'Water Temperature at Transducer',
-                        'standard_name': 'water_temperature',                          
- 
+_
-                        },
-        }

raw2proc/trunk/raw2proc/procutil.py

@@ -1 +1 @@
-#!/usr/bin/env python
-09-06-19 15:42:44
+10-04-07 17:41:26
-"""Utilities to help data processing 
-
@@ -289 +289 @@
-    # fn_glob = 'bogue_dspec_plot*'
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-    # dt to be only 1-dimension and data to be 1- or 2-dimensional
-    
@@ -300 +315 @@
-    delta = numpy.diff(dn)
-    sample_interval = numpy.median(delta)
-
+
+
-    # print maxdelta
-    igap = (delta > maxdelta).nonzero()[0]
@@ -309 +325 @@
-        # convert sample interval to dt object
-        sample_interval = timedelta(0.5*sample_interval)
-        # for each gap in data create NaN
-        data_insert = [numpy.nan for gap in igap]
-        # for each gap in time create datetime value
-        dt_insert = [dt[gap]+sample_interval for gap in igap]
-        # insert new sample times at indices of the gaps
-        new_dt = numpy.insert(numpy.array(dt), igap+1, dt_insert)
-
-
+
+
+
+
+
+
-        return (new_dt, new_data)
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-def meters2feet(meters):
-    """Convert meters to feet: <feet> = <meters>*3.28084 """
@@ -438 +510 @@
-    # global replace _FillValue
-    nc_replace_fillvalue(ofn, -99999.0)
+
+def proc2csv(pi, si, yyyy_mm):
+    """Select specific variables and times from current monthly netCDF
+    and post file of csv data.  TEST MODE.
+
+    For each active config file, load specific variables from NCCOOS
+    monthly netCDF, make any necessary changes to data or attributes
+    conform to CSV output, subset data, and
+    create new file in csv directory.
+
+    NOTE: See auto() function for similar action.
+
+    """
+    
+    platform = pi['id']
+    package = si['id']
+    # input file
+    si['proc_filename'] = '%s_%s_%s.nc' % (platform, package, yyyy_mm)
+    ifn = os.path.join(si['proc_dir'], si['proc_filename'])
+    # output file
+    si['csv_filename'] = 'nccoos_%s_%s_latest.csv' % (platform, package)
+    ofn = os.path.join(si['csv_dir'], si['csv_filename'])
+    f = open(ofn, 'w')
+    
+    if os.path.exists(ifn):
+        print ' ... ... csv : %s ' % (ofn,)
+        # get dt from current month file
+        (es, units) = nc_get_time(ifn)
+        dt = [es2dt(e) for e in es]
+        last_dt = dt[-1]
+    else:
+        # no input then report fact csv file 
+        print ' ... ... csv: NO csv data reported '
+        f.write('"No DATA REPORTED", " \\- ", " \\- "\n')
+        f.close()
+        return
+    
+    # determine which index of data is within the specified timeframe (last 2 days)
+    n = len(dt)
+    idx = numpy.array([False for i in range(n)])
+    for i, val in enumerate(dt):
+        if val>last_dt-timedelta(days=1) and val<=last_dt+timedelta(seconds=360):
+            idx[i] = True
+    dt = numpy.array(dt)
+    dt = dt[idx]
+
+    # read in data and unpack tuple
+    d = nc_load(ifn, si['csv_vars'])
+    global_atts, var_atts, dim_inits, var_inits, var_data = d
+
+    # dts = es2dt(dt[-1])
+    # set timezone info to UTC (since data from level1 should be in UTC!!)
+    last_dt = last_dt.replace(tzinfo=tzutc())
+    # return new datetime based on computer local
+    last_dt_local = last_dt.astimezone(tzlocal())
+    
+    diff = abs(last_dt - last_dt_local)
+    if diff.days>0:
+        last_dt_str = last_dt.strftime("%H:%M %Z on %b %d, %Y") + \
+                      ' (' + last_dt_local.strftime("%H:%M %Z, %b %d") + ')'
+    else:
+        last_dt_str = last_dt.strftime("%H:%M %Z") + \
+                      ' (' + last_dt_local.strftime("%H:%M %Z") + ')' \
+                      + last_dt.strftime(" on %b %d, %Y")
+
+    # uses dateutil.tz.tzutc() from dateutil 
+    now_utc_dt = datetime.now(tzutc())
+    now_utc_dt = now_utc_dt.replace(second=0, microsecond=0)
+    # uses dateutil.tz.tzlocal() from dateutil to get timezone settings as known by the operating system
+    now_local_dt = datetime.now(tzlocal())
+    now_local_dt = now_local_dt.replace(second=0, microsecond=0)
+    # if more than a day difference between local time and UTC, specify dates for each
+    # otherwise date for one is sufficient (cuts down on clutter)
+    diff = abs(now_local_dt - now_utc_dt)
+    if diff.days>0:
+        now_str = now_utc_dt.strftime("%H:%M %Z on %b %d, %Y") + \
+                  ' (' + now_local_dt.strftime("%H:%M %Z, %b %d") + ')'
+    else:
+        now_str = now_utc_dt.strftime("%H:%M %Z") + \
+                  ' (' + now_local_dt.strftime("%H:%M %Z") + ')' \
+                  + now_utc_dt.strftime(" on %b %d, %Y")
+
+    # how old is the data
+    stale_diff = abs(now_utc_dt - last_dt)
+    if stale_diff.days>0 or stale_diff.seconds>=8*60*60:
+        stale_str = display_time_diff(stale_diff)
+    else:
+        stale_str = '' # use empty string to keep background white
+            
+    varNames = [vn for vn, vt, vd in var_inits]
+    var_data = list(var_data)
+    for i in range(len(varNames)):
+        vn, vd = var_data[i]
+        vd = vd[idx]
+
+        # (1) var name and units (first td)
+        var_name_str = '%s (%s)' % (var_atts[vn]['long_name'], var_atts[vn]['short_name'])
+        valunits = var_atts[vn]['units']
+        if vn=='rain':
+            val = vd.sum()
+            var_name_str = 'Rain Total (24 hrs)'
+        else:
+            val = vd[-1]
+
+        # if can take the length of val
+        # probably a list, tuple
+        # there will be more than one value of which we want a mean (ignoring NaN')
+        if bool('__len__' in dir(val)):
+            val = numpy.mean(numpy.ma.masked_where(numpy.isnan(val), val))                
+
+        # to metric
+        import udunits
+
+        sn = var_atts[vn]['standard_name']
+        valunits_from = valunits
+        if 'temperature' in sn or sn in ('wind_chill', 'dew_point'):            
+            if valunits_from == 'degrees Celsius':
+                valunits_from = 'degC'
+            valunits_to = 'degC'
+        elif 'velocity' in sn:
+            valunits_to = 'm s-1'
+        elif 'flux' in sn or sn in ('discharge',):
+            if valunits_from == 'cfs':
+                valunits_from = 'ft^3/sec'
+            valunits_to = 'm^3/s'
+        elif 'rain' in sn:
+            valunits_to = 'mm'
+        elif 'level' in sn or 'height' in sn or 'depth' in sn:
+            valunits_to = 'm'
+        else:
+            # can't find a conversion we want so convert to itself
+            valunits_to = valunits_from
+            
+        cnv = udunits.udunits(valunits_from, valunits_to)
+
+        if cnv[0]==0:
+            val_to = val*cnv[1] + cnv[2]
+            if val_to > 99:
+                metric_str = '%.4g (%s)' % (val_to, valunits_to)
+            else:
+                metric_str = '%.2g (%s)' % (val_to, valunits_to)
+        # handle errors
+        # [-1, 'Unable to parse from', 'NTU', -3, 'Conversion not possible']
+        # [-2, 'Unable to parse to', 'NTU', -3, 'Conversion not possible']
+        # [-3, 'Conversion not possible']
+        elif cnv[0]==-1 or cnv[0]==-2:
+            if val > 99:
+                metric_str = '%.4g (%s)' % (val, valunits)
+            else:
+                metric_str = '%.2g (%s)' % (val, valunits)
+        else:
+            metric_str = '\-'
+                
+        # to english units
+        if 'temperature' in sn or sn in ('wind_chill', 'dew_point'):            
+            if valunits_from == 'degrees Celsius':
+                valunits_from = 'degC'
+            valunits_to = 'degF'
+        elif 'velocity' in sn:
+            valunits_to = 'knots'
+        elif 'flux' in sn or sn in ('discharge',):
+            if valunits_from == 'cfs':
+                valunits_from = 'ft^3/sec'
+            valunits_to = 'ft^3/s'
+        elif 'rain' in sn:
+            valunits_to = 'in'
+        elif 'level' in sn or 'height' in sn or 'depth' in sn:
+            valunits_to = 'ft'
+        else:
+            valunits_to = valunits_from
+        # 
+        cnv = udunits.udunits(valunits_from, valunits_to)
+        if cnv[0]==0:
+            val_to = val*cnv[1] + cnv[2]            
+            if val > 99:
+                english_str ='%.4g (%s)' % (val_to, valunits_to)
+            else:
+                english_str = '%.2g (%s)' % (val_to, valunits_to)
+        # handle errors
+        # [-1, 'Unable to parse from', 'NTU', -3, 'Conversion not possible']
+        # [-2, 'Unable to parse to', 'NTU', -3, 'Conversion not possible']
+        # [-3, 'Conversion not possible']
+        elif cnv[0]==-1 or cnv[0]==-2:
+            if val > 99:
+                english_str = '%.4g (%s)' % (val, valunits)
+            else:
+                english_str = '%.2g (%s)' % (val, valunits)
+        else:
+            english_str = '\-'
+
+        if metric_str == english_str:
+            english_str = '\-'
+       
+        if vn=='time':
+            f.write('"**%s:** %s", ""\n' % ('Sample Time', last_dt_str))
+        else:
+            f.write('"%s", "%s", "%s"\n' % (var_name_str, metric_str, english_str))
+
+    f.close()
+
+
+
+

raw2proc/trunk/raw2proc/raw2proc.py

@@ -1 +1 @@
-#!/usr/bin/env python
-06-23 16:41:19
+12-05 21:32:18
-"""Process raw data to monthly netCDF data files
-
@@ -325 +325 @@
-                    # print ' ... ... latest : %s ' % si['latest_dir']
-                    proc2latest(pi, si, yyyy_mm)
+
+                if 'csv_dir' in si.keys():
+                    proc2csv(pi, si, yyyy_mm)
-    #
-    else: