Changeset 488

Timestamp:

04/24/12 12:33:22

Author:

haines

Message:

removed test_ and scr_ files not necessary for SVN

Files:

• raw2proc/trunk/raw2proc/b1_config_20111112.py (modified) (4 diffs)
• raw2proc/trunk/raw2proc/b2_config_20111005.py (modified) (4 diffs)
• raw2proc/trunk/raw2proc/proc_codar_totals.py (modified) (1 diff)
• raw2proc/trunk/raw2proc/proc_cr1000_ctd.py (added)
• raw2proc/trunk/raw2proc/proc_cr1000_met.py (modified) (4 diffs)
• raw2proc/trunk/raw2proc/proc_cr1000_wind.py (modified) (6 diffs)
• raw2proc/trunk/raw2proc/procutil.py (modified) (2 diffs)
• raw2proc/trunk/raw2proc/raw2proc.py (modified) (10 diffs)
• raw2proc/trunk/raw2proc/scr_read_avp_ysi_6600_v2_fixed_profiler.py (deleted)
• raw2proc/trunk/raw2proc/scr_read_avp_ysi_6600_v2_moving_point.py (deleted)
• raw2proc/trunk/raw2proc/test_codar.py (deleted)
• raw2proc/trunk/raw2proc/test_proc_codar_totals.py (deleted)

raw2proc/trunk/raw2proc/b1_config_20111112.py

@@ -7 +7 @@
-            'altitude': 0.,   # (approx.) station altitude
-            'altitude_units' : 'm',
-above_sea_level
+sea_surface
-            #
-            'mean_water_depth': -32.0,
@@ -15 +15 @@
-            'config_start_date' : '2011-11-13 00:00:00',
-            'config_end_date' : None, # None or yyyy-mm-dd HH:MM:SS
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-            }
-
@@ -28 +43 @@
-              'barometer_height'   : 1.5,   # meters
-              'temperature_height' : 1.5,   # meters 
+              'height_units' : 'm',
+              'height_reference' : 'sea_surface',
-              # 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0',
-
+
+
+
-             },
-    'wind' : { 'id' : 'wind',
@@ -41 +60 @@
-              'anemometer2_height' : 3.35,  # meters (11 ft above sea surface)
-              'height_units' : 'm',
-above_sea_level
+sea_surface
-              # 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0',
-              # 'latest_vars' : ('time','lat','lon','z','u', 'v', 'wspd', 'wdir'),
+              # Recommended
+              'source': 'RM Young Marine Wind Monitor 5106',
+             },
+    'ctd1' : { 'id' : 'ctd1',
+              'description' : 'Near-surface CTD Data each sample period',
+              'raw_dir' : '/seacoos/data/nccoos/level0/b1/ctd1/',
+              'raw_file_glob' : '*.dat',
+              'proc_dir' : '/seacoos/data/nccoos/level1/b1/ctd1/',
+              'process_module' : 'proc_cr1000_ctd',
+              'utc_offset' : 0,  # hours offset to utc of sampling time
+              'nominal_depth' : -2.0,  # meters 
+              'depth_units' : 'm',
+              'depth_reference' : 'sea_surface',
+              # 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0',
+              # 'latest_vars' : ('time','lat','lon','z','depth', 'wtemp', 'cond', 'salin', 'density'),
+              # Recommended
+              'source': 'Seabird (SBE) 37 IMP',
+             },
+    'ctd2' : { 'id' : 'ctd2',
+               'description' : 'Mid-level CTD Data each sample period',
+              'raw_dir' : '/seacoos/data/nccoos/level0/b1/ctd2/',
+              'raw_file_glob' : '*.dat',
+              'proc_dir' : '/seacoos/data/nccoos/level1/b1/ctd2/',
+              'process_module' : 'proc_cr1000_ctd',
+              'utc_offset' : 0,  # hours offset to utc of sampling time
+              'nominal_depth' : -15.0,  # meters 
+              'depth_units' : 'm',
+              'depth_reference' : 'sea_surface',
+              # 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0',
+              # 'latest_vars' : ('time','lat','lon','z','depth', 'wtemp', 'cond', 'salin', 'density'),
+              # Recommended
+              'source': 'Seabird (SBE) 37 IMP',
-             },
-    }

raw2proc/trunk/raw2proc/b2_config_20111005.py

@@ -7 +7 @@
-            'altitude': 0.,   # (approx.) station altitude
-            'altitude_units' : 'm',
-above_sea_level
+sea_surface
-            #
-            'mean_water_depth': -29.2, # meters (96 feet)
@@ -16 +16 @@
-            'config_end_date' : None, # None or yyyy-mm-dd HH:MM:SS
-            'packages' : ('met', 'wind',),
+            # Required by CF
+            'institution' : 'Unversity of North Carolina at Chapel Hill (UNC-CH)',
+            'institution_url' : 'http://nccoos.org',
+            'institution_dods_url' : 'http://nccoos.org',
+            'contact' : 'Sara Haines (haines@email.unc.edu)',
+            'conventions' : 'CF-1.0; SEACOOS-CDL-v2.0',
+            # Required by Scout
+            'format_category_code' : 'fixed-point',
+            'institution_code' : 'nccoos',
+            # Recommended
+            'project' : 'North Carolina Coastal Ocean Observing System (NCCOOS)',
+            'project_url' : 'http://nccoos.org',            
+            'metadata_url' : 'http://nccoos.org',
+            'references' : 'http://nccoos.org',
+            'source': 'Buoy CR1000 Datalogger',
-            }
-
@@ -29 +44 @@
-              'temperature_height' : 1.5,   # meters 
-              'height_units' : 'm',
-above_sea_level
+sea_surface
-              # 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0',
-              # 'latest_vars' : ('time','lat','lon','z','u', 'v', 'wspd', 'wdir'),
+              # Recommended
+              'source': 'Heise Baro, Rotronics Temp/RH, RM Young Precip, Eppley PSP/PIR',
-             },
-    'wind' : { 'id' : 'wind',
@@ -43 +60 @@
-              'anemometer2_height' : 3.35,  # meters (11 ft above sea surface)
-              'height_units' : 'm',
-above_sea_level
+sea_surface
-              # 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0',
-              # 'latest_vars' : ('time','lat','lon','z','u', 'v', 'wspd', 'wdir'),
-             },
+              # Recommended
+              'source': 'RM Young Marine Wind Monitor 5106',
+    'ctd1' : { 'id' : 'ctd1',
+              'description' : 'Near-surface CTD Data each sample period',
+              'raw_dir' : '/seacoos/data/nccoos/level0/b2/ctd1/',
+              'raw_file_glob' : '*.dat',
+              'proc_dir' : '/seacoos/data/nccoos/level1/b2/ctd1/',
+              'process_module' : 'proc_cr1000_ctd',
+              'utc_offset' : 0,  # hours offset to utc of sampling time
+              'nominal_depth' : -2.0,  # meters 
+              'depth_units' : 'm',
+              'depth_reference' : 'sea_surface',
+              # 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0',
+              # 'latest_vars' : ('time','lat','lon','z','depth', 'wtemp', 'cond', 'salin', 'density'),
+              # Recommended
+              'source': 'Seabird (SBE) 37 IMP',
+             },
+    'ctd2' : { 'id' : 'ctd2',
+               'description' : 'Mid-level CTD Data each sample period',
+              'raw_dir' : '/seacoos/data/nccoos/level0/b2/ctd2/',
+              'raw_file_glob' : '*.dat',
+              'proc_dir' : '/seacoos/data/nccoos/level1/b2/ctd2/',
+              'process_module' : 'proc_cr1000_ctd',
+              'utc_offset' : 0,  # hours offset to utc of sampling time
+              'nominal_depth' : -15.0,  # meters 
+              'depth_units' : 'm',
+              'depth_reference' : 'sea_surface',
+              # 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0',
+              # 'latest_vars' : ('time','lat','lon','z','depth', 'wtemp', 'cond', 'salin', 'density'),
+              # Recommended
+              'source': 'Seabird (SBE) 37 IMP',
+             },
-    }

raw2proc/trunk/raw2proc/proc_codar_totals.py

@@ -1 +1 @@
-#!/usr/bin/env python
-0-09-20 12:32:19
+2-04-17 13:56:34
-"""
-how to parse data, and assert what data and info goes into

raw2proc/trunk/raw2proc/proc_cr1000_met.py

@@ -1 +1 @@
-#!/usr/bin/env python
-1-12-16 14:10:43
+2-04-23 14:12:38
-"""
-how to parse data, and assert what data and info goes into
@@ -145 +145 @@
-    #
-    # 
+    # subset data only to month being processed (see raw2proc.process())
+    i = data['in']
+
-    title_str = sensor_info['description']+' at '+ platform_info['location']
- 
+
-        'title' : title_str,
-'University of North Carolina at Chapel Hill (UNC-CH)'
-'http://nccoos.org'
-'http://nccoos.org'
-'http://nccoos.org'
-'http://nccoos.org'
-'Sara Haines (haines@email.unc.edu)'
+platform_info['institution']
+platform_info['institution_url']
+platform_info['institution_dods_url']
+platform_info['metadata_url']
+platform_info['references']
+platform_info['contact']
-        # 
-'buoy station'
+platform_info['source']+' '+sensor_info['source']
-        'history' : 'raw2proc using ' + sensor_info['process_module'],
-        'comment' : 'File created using pycdf'+pycdfVersion()+' and numpy '+pycdfArrayPkg(),
-        # conventions
-'CF-1.0; SEACOOS-CDL-v2.0'
+platform_info['conventions']
-        # SEACOOS CDL codes
-'fixed-point'
-
+platform_info['format_category_code']
+_code
-        'platform_code' : platform_info['id'],
-        'package_code' : sensor_info['id'],
-        # institution specific
-'North Carolina Coastal Ocean Observing System (NCCOOS)'
-'http://nccoos.org'
+platform_info['project']
+platform_info['project_url']
-        # timeframe of data contained in file yyyy-mm-dd HH:MM:SS
-
-
+
+
+
+
-        'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
-        #
@@ -330 +335 @@
-def updater(platform_info, sensor_info, data):
-    #
+    # subset data only to month being processed (see raw2proc.process())
+    i = data['in']
+
-    global_atts = { 
-        # update times of data contained in file (yyyy-mm-dd HH:MM:SS)
-        # last date in monthly file
-
+i][
-        'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
-        #
@@ -351 +359 @@
-    #    }
-    
-    # subset data only to month being processed (see raw2proc.process())
-    i = data['in']
-
-    # data 
-    var_data = (

raw2proc/trunk/raw2proc/proc_cr1000_wind.py

@@ -1 +1 @@
-#!/usr/bin/env python
-1-12-16 14:36:47
+2-04-23 14:13:36
-"""
-how to parse data, and assert what data and info goes into
@@ -146 +146 @@
-    #
-    # 
+    # subset data only to month being processed (see raw2proc.process())
+    i = data['in']
+
-    title_str = sensor_info['description']+' at '+ platform_info['location']
- 
+
-        'title' : title_str,
-'University of North Carolina at Chapel Hill (UNC-CH)'
-'http://nccoos.org'
-'http://nccoos.org'
-'http://nccoos.org'
-'http://nccoos.org'
-'Sara Haines (haines@email.unc.edu)'
+platform_info['institution']
+platform_info['institution_url']
+platform_info['institution_dods_url']
+platform_info['metadata_url']
+platform_info['references']
+platform_info['contact']
-        # 
-'buoy station'
+platform_info['source']+' '+sensor_info['source']
-        'history' : 'raw2proc using ' + sensor_info['process_module'],
-        'comment' : 'File created using pycdf'+pycdfVersion()+' and numpy '+pycdfArrayPkg(),
-        # conventions
-'CF-1.0; SEACOOS-CDL-v2.0'
+platform_info['conventions']
-        # SEACOOS CDL codes
-'fixed-point'
-
+platform_info['format_category_code']
+_code
-        'platform_code' : platform_info['id'],
-        'package_code' : sensor_info['id'],
-        # institution specific
-'North Carolina Coastal Ocean Observing System (NCCOOS)'
-'http://nccoos.org'
+platform_info['project']
+platform_info['project_url']
-        # timeframe of data contained in file yyyy-mm-dd HH:MM:SS
-
-
+
+
+
+
-        'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
-        #
@@ -181 +186 @@
-        '_FillValue' : -99999.,
-        }
+
-
-    var_atts = {
@@ -313 +319 @@
-        )
-
-    # subset data only to month being processed (see raw2proc.process())
-    i = data['in']
-    
-    # var data 
-    var_data = (
@@ -338 +341 @@
-def updater(platform_info, sensor_info, data):
-    #
+    # subset data only to month being processed (see raw2proc.process())
+    i = data['in']
+
-    global_atts = { 
-        # update times of data contained in file (yyyy-mm-dd HH:MM:SS)
-        # last date in monthly file
-
+i][
-        'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"),
-        #
@@ -359 +365 @@
-    #    }
-    
-    # subset data only to month being processed (see raw2proc.process())
-    i = data['in']
-
-    # data 
-    var_data = (

raw2proc/trunk/raw2proc/procutil.py

@@ -1 +1 @@
-#!/usr/bin/env python
-1-12-16 15:18:49
+2-04-17 12:09:47
-"""Utilities to help data processing 
-
@@ -128 +128 @@
-        # e.g. month 1...12, something parsed wrong
-        dt = None
-
-
-
-
-
-
-
-
+# 
+#
+# 
+# 
+# 
+# 
+# 
+# 
-
-    return dt

raw2proc/trunk/raw2proc/raw2proc.py

@@ -1 +1 @@
-#!/usr/bin/env python
-1-12-16 17:05:08
+2-04-17 13:22:05
-"""Process raw data to monthly netCDF data files
-
@@ -182 +182 @@
-    for fn in all_raw_files:
-        (fndt, granularity) = filt_datetime(os.path.basename(fn), gran=True)
-
-        # "ind" var from filt_datetime() - what level of granularity was used
-        if granularity == 4:
-            # change dt_start to before monthly filename filt_datetime() date
@@ -191 +189 @@
-            dt_start = si['proc_start_dt']-timedelta(days=31)
-            # print dt_start
-
-        if fndt:
-            if dt_start <= fndt <= dt_end or m:
@@ -213 +210 @@
-        config_end_dt = now_dt
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
-
-    new_list = [raw_files[i] for i in range(len(raw_files)) \
@@ -296 +291 @@
-           
-    """
+    now_dt = datetime.utcnow()
+    now_dt.replace(microsecond=0)
+
-    yyyy_mm = this_month()
-    months = find_months(yyyy_mm)
@@ -309 +307 @@
-            asi = get_config(cn+'.sensor_info')
-            platform = pi['id']
+            if pi['config_start_date']:
+                pi['config_start_dt'] = filt_datetime(pi['config_start_date'])
+            elif pi['config_start_date'] == None:
+                pi['config_start_dt'] = now_dt
+            if pi['config_end_date']:
+                pi['config_end_dt'] = filt_datetime(pi['config_end_date'])
+            elif pi['config_end_date'] == None:
+                pi['config_end_dt'] = now_dt
-            # for each sensor package
-            for package in asi.keys():
@@ -357 +363 @@
-               which raw files
-    """
-
+
+
+
-    months = find_months(yyyy_mm)
-    month_start_dt = months[1]
@@ -370 +378 @@
-            print ' ... config file : %s' % cn
-            pi = get_config(cn+'.platform_info')
+            if pi['config_start_date']:
+                pi['config_start_dt'] = filt_datetime(pi['config_start_date'])
+            elif pi['config_start_date'] == None:
+                pi['config_start_dt'] = now_dt
+            if pi['config_end_date']:
+                pi['config_end_dt'] = filt_datetime(pi['config_end_date'])
+            elif pi['config_end_date'] == None:
+                pi['config_end_dt'] = now_dt
-            # month start and end dt to pi info
-            asi = get_config(cn+'.sensor_info')
@@ -381 +397 @@
-                ofn = os.path.join(si['proc_dir'], si['proc_filename'])
-                (raw_files, raw_dts) = find_raw(si, yyyy_mm)
-
-
+# 
+# 
-                raw_files = which_raw(pi, raw_files, raw_dts)
-
-
+# 
+# 
-                # remove any previous netcdf file (platform_package_yyyy_mm.nc)
-                if index==0  and os.path.exists(ofn):
@@ -423 +439 @@
-
-            for index, val in enumerate(data['dt']):
-si['proc_start_dt'] and val<=si['proc
+pi['config_start_dt'] and val>=si['proc_start_dt'] and val<=si['proc_end_dt'] and val<=pi['config
-                    data['in'][index] = True
-