Index: raw2proc/trunk/raw2proc/b1_config_20111112.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/b1_config_20111112.py (revision 478) @@ -1,0 +1,47 @@ +platform_info = { + 'id' : 'b1', + 'location' : 'Hatteras Bay, 20 nm East of Oregon Inlet, NC', + 'lat' : 35.7885, # degrees true (-) south, (+) north + 'lon' : -75.1053, # degrees true (-) west, (+) east + 'mvar' : -11.3, # degrees (-) west, (+) east + 'altitude': 0., # (approx.) station altitude + 'altitude_units' : 'm', + 'altitude_reference' : 'above_sea_level', + # + 'mean_water_depth': -32.0, + 'mean_water_depth_time_period': 'Not determined', + 'institution' : 'nccoos', + # + 'config_start_date' : '2011-11-13 00:00:00', + 'config_end_date' : None, # None or yyyy-mm-dd HH:MM:SS + 'packages' : ('met', 'wind', ), + } + +sensor_info = { + 'met' : { 'id' : 'met', + 'description' : 'Meterological Data averaged for one minute each sample period', + 'raw_dir' : '/seacoos/data/nccoos/level0/b1/met/', + 'raw_file_glob' : '*.dat', + 'proc_dir' : '/seacoos/data/nccoos/level1/b1/met/', + 'process_module' : 'proc_cr1000_met', + 'utc_offset' : 0, # hours offset to utc of sampling time + 'barometer_height' : 1.5, # meters + 'temperature_height' : 1.5, # meters + # 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + # 'latest_vars' : ('time','lat','lon','z','u', 'v', 'wspd', 'wdir'), + }, + 'wind' : { 'id' : 'wind', + 'description' : 'Wind Data averaged for one minute each sample period', + 'raw_dir' : '/seacoos/data/nccoos/level0/b1/wind/', + 'raw_file_glob' : '*.dat', + 'proc_dir' : '/seacoos/data/nccoos/level1/b1/wind/', + 'process_module' : 'proc_cr1000_wind', + 'utc_offset' : 0, # hours offset to utc of sampling time + 'anemometer1_height' : 3.66, # meters (12 ft) + 'anemometer2_height' : 3.35, # meters (11 ft above sea surface) + 'height_units' : 'm', + 'height_reference' : 'above_sea_level', + # 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + # 'latest_vars' : ('time','lat','lon','z','u', 'v', 'wspd', 'wdir'), + }, + } Index: raw2proc/trunk/raw2proc/b2_config_20111005.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/b2_config_20111005.py (revision 478) @@ -1,0 +1,49 @@ +platform_info = { + 'id' : 'b2', + 'location' : 'Raleigh Bay, 17 nm South of Ocracoke, NC', + 'lat' : 34.7824, # degrees true (-) south, (+) north + 'lon' : -75.9410, # degrees true (-) west, (+) east + 'mvar' : -10.4, # degrees (-) west, (+) east + 'altitude': 0., # (approx.) station altitude + 'altitude_units' : 'm', + 'altitude_reference' : 'above_sea_level', + # + 'mean_water_depth': -29.2, # meters (96 feet) + 'mean_water_depth_time_period': 'Not determined', + 'institution' : 'nccoos', + # + 'config_start_date' : '2011-10-06 00:00:00', + 'config_end_date' : None, # None or yyyy-mm-dd HH:MM:SS + 'packages' : ('met', 'wind',), + } + +sensor_info = { + 'met' : { 'id' : 'met', + 'description' : 'Meterological Data from Buoy', + 'raw_dir' : '/seacoos/data/nccoos/level0/b2/met/', + 'raw_file_glob' : '*.dat', + 'proc_dir' : '/seacoos/data/nccoos/level1/b2/met/', + 'process_module' : 'proc_cr1000_met', + 'utc_offset' : 0, # hours offset to utc of sampling time + 'barometer_height' : 1.5, # meters + 'temperature_height' : 1.5, # meters + 'height_units' : 'm', + 'height_reference' : 'above_sea_level', + # 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + # 'latest_vars' : ('time','lat','lon','z','u', 'v', 'wspd', 'wdir'), + }, + 'wind' : { 'id' : 'wind', + 'description' : 'Wind Data averaged for one minute each sample period', + 'raw_dir' : '/seacoos/data/nccoos/level0/b2/wind/', + 'raw_file_glob' : '*.dat', + 'proc_dir' : '/seacoos/data/nccoos/level1/b2/wind/', + 'process_module' : 'proc_cr1000_wind', + 'utc_offset' : 0, # hours offset to utc of sampling time + 'anemometer1_height' : 3.66, # meters (12 ft) + 'anemometer2_height' : 3.35, # meters (11 ft above sea surface) + 'height_units' : 'm', + 'height_reference' : 'above_sea_level', + # 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + # 'latest_vars' : ('time','lat','lon','z','u', 'v', 'wspd', 'wdir'), + }, + } Index: raw2proc/trunk/raw2proc/billymitchell_config_20110914.py =================================================================== --- raw2proc/trunk/raw2proc/billymitchell_config_20110914.py (revision 467) +++ raw2proc/trunk/raw2proc/billymitchell_config_20110914.py (revision 478) @@ -7,5 +7,5 @@ 'institution' : 'nccoos', 'config_start_date' : '2011-09-14 00:00:00', - 'config_end_date' : None, # None or yyyy-mm-dd HH:MM:SS + 'config_end_date' : '2011-10-29 00:00:00', # None or yyyy-mm-dd HH:MM:SS 'packages' : ('sfas',), } Index: raw2proc/trunk/raw2proc/billymitchell_config_20111029.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/billymitchell_config_20111029.py (revision 478) @@ -1,0 +1,29 @@ +platform_info = { + 'id' : 'billymitchell', + 'location' : 'Billy Mitchell Airfield, Frisco, NC', + 'lat' : 35.231691, # degrees true (-) south, (+) north + 'lon' : -75.622614, # degrees true (-) west, (+) east + 'mvar' : -10.783333, # degrees (-) west, (+) east + 'institution' : 'nccoos', + 'config_start_date' : '2011-10-29 00:30:00', + 'config_end_date' : None, # None or yyyy-mm-dd HH:MM:SS + 'packages' : ('sfas',), + } +sensor_info = { + 'sfas' : { 'id' : 'sodar', + 'description' : 'Wind profile data', + 'raw_dir' : '/seacoos/data/nccoos/level0/' + 'billymitchell/sodar1/mnd', + 'raw_file_glob' : '*.mnd', + 'proc_dir' : '/seacoos/data/nccoos/level1/' + 'billymitchell/sodar1', + 'process_module' : 'proc_scintec_maindata_sfas', + 'utc_offset' : 0, # hours offset to utc + 'min_altitude' : 15, # meters + 'altitude_interval' : 5, # meters + 'num_altitudes' : 37, + 'sensor_elevation' : 0, # meters (runway elev is at 5.2 m) + 'plot_module' : 'billymitchell_sodar_plot', + 'plot_names' : ('timeseries', 'wind_vectors', 'wind_barbs'), + }, + } Index: raw2proc/trunk/raw2proc/proc_cr1000_met.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/proc_cr1000_met.py (revision 478) @@ -1,0 +1,373 @@ +#!/usr/bin/env python +# Last modified: Time-stamp: <2011-12-16 14:10:43 haines> +""" +how to parse data, and assert what data and info goes into +creating and updating monthly netcdf files + +parse data met data collected on Campbell Scientific DataLogger (loggernet) (csi) + +parser : sample date and time, + +creator : lat, lon, z, time, +updator : time, + + +Examples +-------- + +>> (parse, create, update) = load_processors('proc_csi_adcp_v2') +or +>> si = get_config(cn+'.sensor_info') +>> (parse, create, update) = load_processors(si['adcp']['proc_module']) + +>> lines = load_data(filename) +>> data = parse(platform_info, sensor_info, lines) +>> create(platform_info, sensor_info, data) or +>> update(platform_info, sensor_info, data) + +""" + + +from raw2proc import * +from procutil import * +from ncutil import * + +now_dt = datetime.utcnow() +now_dt.replace(microsecond=0) + +def parser(platform_info, sensor_info, lines): + """ + Example met data + + "TOA5","CR1000_B1","CR1000","37541","CR1000.Std.21","CPU:NCWIND_12_Buoy_All.CR1","58723","AMet_6Min" + "TIMESTAMP","RECORD","Baro_mbar_Avg","RHumidity_Avg","RHumidity_Std","AirTempC_Avg","AirTempC_Std","Rain","Psp_Avg","Psp_Std","Pir_Wm2_Avg","Pir_Wm2_Std" + "TS","RN","","","","","","","","","","" + "","","Avg","Avg","Std","Avg","Std","Smp","Avg","Std","Avg","Std" + "2011-11-01 00:00:59",4590,14.3792,75.59,0.579,15.67,0.05,-22.35,1197.037,45.58967,371.5126,0.9030571 + "2011-11-01 00:06:59",4591,14.37995,74.96,0.912,16.61,0.048,-21,-1071.813,129.5147,381.2539,0.2076943 + "2011-11-01 00:12:59",4592,14.3792,72.71,2.677,17.29,0.032,-15.58,-2056.658,0,381.1828,0.1402813 + "2011-11-01 00:18:59",4593,14.3791,72.63,0.928,17.67,0.041,-19.64,-1895.86,9.866026,381.0333,0.2442325 + + """ + + import numpy + from datetime import datetime + from time import strptime + + # get sample datetime from filename + fn = sensor_info['fn'] + sample_dt_start = filt_datetime(fn) + + # how many samples (don't count header 4 lines) + nsamp = len(lines[4:]) + + N = nsamp + data = { + 'dt' : numpy.array(numpy.ones((N,), dtype=object)*numpy.nan), + 'time' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan), + 'air_press' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'rh' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'rh_std' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'air_temp' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'air_temp_std' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'rain' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'psp' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'psp_std' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'pir' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'pir_std' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + } + + # sample count + i = 0 + + for line in lines[4:]: + csi = [] + # split line + sw = re.split(',', line) + if len(sw)<=0: + print ' ... skipping line %d ' % (i,) + continue + + # replace any "NAN" text with a number + for index, s in enumerate(sw): + m = re.search(NAN_RE_STR, s) + if m: + sw[index] = '-99999' + + # parse date-time, and all other float and integers + for s in sw[1:]: + m = re.search(REAL_RE_STR, s) + if m: + csi.append(float(m.groups()[0])) + + if sensor_info['utc_offset']: + sample_dt = scanf_datetime(sw[0], fmt='"%Y-%m-%d %H:%M:%S"') + \ + timedelta(hours=sensor_info['utc_offset']) + else: + sample_dt = scanf_datetime(sw[0], fmt='"%Y-%m-%d %H:%M:%S"') + + data['dt'][i] = sample_dt # sample datetime + data['time'][i] = dt2es(sample_dt) # sample time in epoch seconds + + if len(csi)==11: + # + # data['samplenum'][i] = csi[0] # sample number assigned by datalogger in table + data['air_press'][i] = csi[1] # Heise Barometer (mbar) + data['rh'][i] = csi[2] # relative humidity avg (60 samples for 1 min) + data['rh_std'][i] = csi[3] # relative humidity std + data['air_temp'][i] = csi[4] # air temperature avg (deg C) + data['air_temp_std'][i] = csi[5] # air temperature std (deg C) + data['rain'][i] = csi[6] # precip gauge cummulative + data['psp'][i] = csi[7] # PSP avg + data['psp_std'][i] = csi[8] # PSP std + data['pir'][i] = csi[9] # PIR avg (W m-2) + data['pir_std'][i] = csi[10] # PIR std (W m-2) + i=i+1 + else: + print ' ... skipping line %d -- %s ' % (i,line) + continue + + # if re.search + # for line + + + # check that no data[dt] is set to Nan or anything but datetime + # keep only data that has a resolved datetime + keep = numpy.array([type(datetime(1970,1,1)) == type(dt) for dt in data['dt'][:]]) + if keep.any(): + for param in data.keys(): + data[param] = data[param][keep] + + return data + + +def creator(platform_info, sensor_info, data): + # + # + title_str = sensor_info['description']+' at '+ platform_info['location'] + global_atts = { + 'title' : title_str, + 'institution' : 'University of North Carolina at Chapel Hill (UNC-CH)', + 'institution_url' : 'http://nccoos.org', + 'institution_dods_url' : 'http://nccoos.org', + 'metadata_url' : 'http://nccoos.org', + 'references' : 'http://nccoos.org', + 'contact' : 'Sara Haines (haines@email.unc.edu)', + # + 'source' : 'buoy station', + 'history' : 'raw2proc using ' + sensor_info['process_module'], + 'comment' : 'File created using pycdf'+pycdfVersion()+' and numpy '+pycdfArrayPkg(), + # conventions + 'Conventions' : 'CF-1.0; SEACOOS-CDL-v2.0', + # SEACOOS CDL codes + 'format_category_code' : 'fixed-point', + 'institution_code' : platform_info['institution'], + 'platform_code' : platform_info['id'], + 'package_code' : sensor_info['id'], + # institution specific + 'project' : 'North Carolina Coastal Ocean Observing System (NCCOOS)', + 'project_url' : 'http://nccoos.org', + # timeframe of data contained in file yyyy-mm-dd HH:MM:SS + 'start_date' : data['dt'][0].strftime("%Y-%m-%d %H:%M:%S"), + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + # + 'creation_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + 'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + 'process_level' : 'level1', + # + # must type match to data (e.g. fillvalue is real if data is real) + '_FillValue' : -99999., + } + + var_atts = { + # coordinate variables + 'time' : {'short_name': 'time', + 'long_name': 'Time', + 'standard_name': 'time', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + 'axis': 'T', + }, + 'lat' : {'short_name': 'lat', + 'long_name': 'Latitude', + 'standard_name': 'latitude', + 'reference':'geographic coordinates', + 'units': 'degrees_north', + 'valid_range':(-90.,90.), + 'axis': 'Y', + }, + 'lon' : {'short_name': 'lon', + 'long_name': 'Longitude', + 'standard_name': 'longitude', + 'reference':'geographic coordinates', + 'units': 'degrees_east', + 'valid_range':(-180.,180.), + 'axis': 'Y', + }, + 'z' : {'short_name': 'z', + 'long_name': 'Altitude', + 'standard_name': 'altitude', + 'reference':'zero at mean sea level', + 'positive' : 'up', + 'units': 'm', + 'axis': 'Z', + }, + # data variables + 'air_press': {'short_name': 'air_press', + 'long_name': 'Air Pressure', + 'standard_name': 'air_pressure', + 'units': 'mbar', + 'z': sensor_info['barometer_height'], + 'z_units' : 'meter', + }, + 'air_temp': {'short_name': 'air_temp', + 'long_name': 'Air Temperature', + 'standard_name': 'air_temperature', + 'units': 'degC', + 'z': sensor_info['temperature_height'], + 'z_units' : 'meter', + }, + 'air_temp_std': {'short_name': 'air_temp_std', + 'long_name': 'Standard Deviation of Air Temperature', + 'standard_name': 'air_temperature', + 'units': 'degC', + }, + 'rh': {'short_name': 'rh', + 'long_name': 'Relative Humidity', + 'standard_name': 'relative_humidity', + 'units': '%', + 'z': sensor_info['temperature_height'], + 'z_units' : 'meter', + }, + 'rh_std': {'short_name': 'rh_std', + 'long_name': 'Standard Deviation of Relative Humidity', + 'standard_name': 'relative_humidity', + 'units': '%', + }, + 'rain': {'short_name': 'rain', + 'long_name': '6-Minute Rain', + 'standard_name': 'rain', + 'units': 'inches', + }, + 'psp': {'short_name': 'psp', + 'long_name': 'Short-wave Radiation', + 'standard_name': 'downwelling_shortwave_irradiance', + 'units': 'W m-2', + }, + 'psp_std': {'short_name': 'psp_std', + 'long_name': 'Standard Deviation of Short-wave Radiation', + 'standard_name': 'shortwave_radiation', + 'units': 'W m-2', + }, + 'pir': {'short_name': 'pir', + 'long_name': 'Long-wave Radiation', + 'standard_name': 'longwave_radiation', + 'units': 'W m-2', + }, + 'pir_std': {'short_name': 'pir_std', + 'long_name': 'Standard Deviation of Long-wave Radiation', + 'standard_name': 'longwave_radiation', + 'units': 'W m-2', + }, + } + + # dimension names use tuple so order of initialization is maintained + dim_inits = ( + ('ntime', NC.UNLIMITED), + ('nlat', 1), + ('nlon', 1), + ('nz', 1), + ) + + # using tuple of tuples so order of initialization is maintained + # using dict for attributes order of init not important + # use dimension names not values + # (varName, varType, (dimName1, [dimName2], ...)) + var_inits = ( + # coordinate variables + ('time', NC.INT, ('ntime',)), + ('lat', NC.FLOAT, ('nlat',)), + ('lon', NC.FLOAT, ('nlon',)), + ('z', NC.FLOAT, ('nz',)), + # data variables + ('air_press', NC.FLOAT, ('ntime',)), + ('rh', NC.FLOAT, ('ntime',)), + ('rh_std', NC.FLOAT, ('ntime',)), + ('air_temp', NC.FLOAT, ('ntime',)), + ('air_temp_std', NC.FLOAT, ('ntime',)), + ('rain', NC.FLOAT, ('ntime',)), + ('psp', NC.FLOAT, ('ntime',)), + ('psp_std', NC.FLOAT, ('ntime',)), + ('pir', NC.FLOAT, ('ntime',)), + ('pir_std', NC.FLOAT, ('ntime',)), + ) + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # var data + var_data = ( + ('lat', platform_info['lat']), + ('lon', platform_info['lon']), + ('z', platform_info['altitude']), + # + ('time', data['time'][i]), + # + ('air_press', data['air_press'][i]), + ('rh', data['rh'][i]), + ('rh_std', data['rh_std'][i]), + ('air_temp', data['air_temp'][i]), + ('air_temp_std', data['air_temp_std'][i]), + ('rain', data['rain'][i]), + ('psp', data['psp'][i]), + ('psp_std', data['psp_std'][i]), + ('pir', data['pir'][i]), + ('pir_std', data['pir_std'][i]), + ) + + return (global_atts, var_atts, dim_inits, var_inits, var_data) + +def updater(platform_info, sensor_info, data): + # + global_atts = { + # update times of data contained in file (yyyy-mm-dd HH:MM:SS) + # last date in monthly file + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + # + 'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + } + + # data variables + # update any variable attributes like range, min, max + var_atts = {} + # var_atts = { + # 'wtemp': {'max': max(data.u), + # 'min': min(data.v), + # }, + # 'cond': {'max': max(data.u), + # 'min': min(data.v), + # }, + # } + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # data + var_data = ( + ('time', data['time'][i]), + # + ('air_press', data['air_press'][i]), + ('rh', data['rh'][i]), + ('rh_std', data['rh_std'][i]), + ('air_temp', data['air_temp'][i]), + ('air_temp_std', data['air_temp_std'][i]), + ('rain', data['rain'][i]), + ('psp', data['psp'][i]), + ('psp_std', data['psp_std'][i]), + ('pir', data['pir'][i]), + ('pir_std', data['pir_std'][i]), + ) + + return (global_atts, var_atts, var_data) +# Index: raw2proc/trunk/raw2proc/proc_cr1000_wind.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/proc_cr1000_wind.py (revision 478) @@ -1,0 +1,379 @@ +#!/usr/bin/env python +# Last modified: Time-stamp: <2011-12-16 14:36:47 haines> +""" +how to parse data, and assert what data and info goes into +creating and updating monthly netcdf files + +parse data met data collected on Campbell Scientific DataLogger (loggernet) (csi) + +parser : sample date and time, + +creator : lat, lon, z, time, +updator : time, + + +Examples +-------- + +>> (parse, create, update) = load_processors('proc_csi_adcp_v2') +or +>> si = get_config(cn+'.sensor_info') +>> (parse, create, update) = load_processors(si['adcp']['proc_module']) + +>> lines = load_data(filename) +>> data = parse(platform_info, sensor_info, lines) +>> create(platform_info, sensor_info, data) or +>> update(platform_info, sensor_info, data) + +""" + + +from raw2proc import * +from procutil import * +from ncutil import * + +now_dt = datetime.utcnow() +now_dt.replace(microsecond=0) + +def parser(platform_info, sensor_info, lines): + """ + Example wind data + + Stats (avg, std, and max) for wind sampled every second for one minute DURING a 6 minute time period. Stats are NOT over 6 minutes, as + the time stamp would have you believe. + + "TOA5","CR1000_B1","CR1000","37541","CR1000.Std.21","CPU:NCWIND_12_Buoy_All.CR1","58723","AWind_6Min" + "TIMESTAMP","RECORD","W1_SpeedAvg","W1_DirAvg","W1_SpeedMax","W1_SpeedStd","W2_SpeedAvg","W2_DirAvg","W2_SpeedMax","W2_SpeedStd" + "TS","RN","","Deg","","","","Deg","","" + "","","WVc","WVc","Max","Std","WVc","WVc","Max","Std" + "2011-12-01 00:01:59",6507,8.32,319.1,10.09,0.781,8.15,310.9,10.09,0.832 + "2011-12-01 00:07:59",6508,9.43,323.3,11.27,1.094,9.11,315.8,10.68,1.015 + "2011-12-01 00:13:59",6509,9.94,308.6,12.35,1.077,9.74,301.3,11.96,1.027 + "2011-12-01 00:19:59",6510,8.86,304.5,10.98,1.003,8.8,296.4,11.27,1.066 + "2011-12-01 00:25:59",6511,9.02,310.8,10.98,1.023,8.95,302.4,10.78,0.964 + "2011-12-01 00:31:59",6512,9.58,304.9,11.76,1.156,9.39,296.7,11.76,1.167 + + """ + + import numpy + from datetime import datetime + from time import strptime + + # get sample datetime from filename + fn = sensor_info['fn'] + sample_dt_start = filt_datetime(fn) + + # how many samples (don't count header 4 lines) + nsamp = len(lines[4:]) + + N = nsamp + data = { + 'dt' : numpy.array(numpy.ones((N,), dtype=object)*numpy.nan), + 'time' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan), + 'wspd1' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'wspd1_std' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'wgust1' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'wdir1' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'wspd2' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'wspd2_std' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'wgust2' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'wdir2' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + } + + # sample count + i = 0 + + for line in lines[4:]: + csi = [] + # split line + sw = re.split(',', line) + if len(sw)<=0: + print ' ... skipping line %d ' % (i,) + continue + + # replace any "NAN" text with a number + for index, s in enumerate(sw): + m = re.search(NAN_RE_STR, s) + if m: + sw[index] = '-99999' + + # parse date-time, and all other float and integers + for s in sw[1:]: + m = re.search(REAL_RE_STR, s) + if m: + csi.append(float(m.groups()[0])) + + if sensor_info['utc_offset']: + sample_dt = scanf_datetime(sw[0], fmt='"%Y-%m-%d %H:%M:%S"') + \ + timedelta(hours=sensor_info['utc_offset']) + else: + sample_dt = scanf_datetime(sw[0], fmt='"%Y-%m-%d %H:%M:%S"') + + data['dt'][i] = sample_dt # sample datetime + data['time'][i] = dt2es(sample_dt) # sample time in epoch seconds + + if len(csi)==9: + # + # data['samplenum'][i] = csi[0] # sample number assigned by datalogger in table + data['wspd1'][i] = csi[1] # + data['wdir1'][i] = csi[2] # + data['wgust1'][i] = csi[3] # relative humidity std + data['wspd1_std'][i] = csi[4] # air temperature avg (deg C) + data['wspd2'][i] = csi[5] # air temperature std (deg C) + data['wdir2'][i] = csi[6] # precip gauge cummulative + data['wgust2'][i] = csi[7] # PSP avg + data['wspd2_std'][i] = csi[8] # PSP std + i=i+1 + else: + print ' ... skipping line %d -- %s ' % (i,line) + continue + + # if re.search + # for line + + + # check that no data[dt] is set to Nan or anything but datetime + # keep only data that has a resolved datetime + keep = numpy.array([type(datetime(1970,1,1)) == type(dt) for dt in data['dt'][:]]) + if keep.any(): + for param in data.keys(): + data[param] = data[param][keep] + + return data + + +def creator(platform_info, sensor_info, data): + # + # + title_str = sensor_info['description']+' at '+ platform_info['location'] + global_atts = { + 'title' : title_str, + 'institution' : 'University of North Carolina at Chapel Hill (UNC-CH)', + 'institution_url' : 'http://nccoos.org', + 'institution_dods_url' : 'http://nccoos.org', + 'metadata_url' : 'http://nccoos.org', + 'references' : 'http://nccoos.org', + 'contact' : 'Sara Haines (haines@email.unc.edu)', + # + 'source' : 'buoy station', + 'history' : 'raw2proc using ' + sensor_info['process_module'], + 'comment' : 'File created using pycdf'+pycdfVersion()+' and numpy '+pycdfArrayPkg(), + # conventions + 'Conventions' : 'CF-1.0; SEACOOS-CDL-v2.0', + # SEACOOS CDL codes + 'format_category_code' : 'fixed-point', + 'institution_code' : platform_info['institution'], + 'platform_code' : platform_info['id'], + 'package_code' : sensor_info['id'], + # institution specific + 'project' : 'North Carolina Coastal Ocean Observing System (NCCOOS)', + 'project_url' : 'http://nccoos.org', + # timeframe of data contained in file yyyy-mm-dd HH:MM:SS + 'start_date' : data['dt'][0].strftime("%Y-%m-%d %H:%M:%S"), + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + # + 'creation_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + 'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + 'process_level' : 'level1', + # + # must type match to data (e.g. fillvalue is real if data is real) + '_FillValue' : -99999., + } + + var_atts = { + # coordinate variables + 'time' : {'short_name': 'time', + 'long_name': 'Time', + 'standard_name': 'time', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + 'axis': 'T', + }, + 'lat' : {'short_name': 'lat', + 'long_name': 'Latitude', + 'standard_name': 'latitude', + 'reference':'geographic coordinates', + 'units': 'degrees_north', + 'valid_range':(-90.,90.), + 'axis': 'Y', + }, + 'lon' : {'short_name': 'lon', + 'long_name': 'Longitude', + 'standard_name': 'longitude', + 'reference':'geographic coordinates', + 'units': 'degrees_east', + 'valid_range':(-180.,180.), + 'axis': 'Y', + }, + 'z' : {'short_name': 'z', + 'long_name': 'Altitude', + 'standard_name': 'altitude', + 'reference':'zero at mean sea level', + 'positive' : 'up', + 'units': 'm', + 'axis': 'Z', + }, + # data variables + 'wspd1' : {'short_name': 'wspd', + 'long_name': 'Wind Speed', + 'standard_name': 'wind_speed', + 'units': 'm s-1', + 'can_be_normalized': 'no', + 'z' : sensor_info['anemometer1_height'], + 'z_units' : 'meter', + }, + 'wdir1' : {'short_name': 'wdir', + 'long_name': 'Wind Direction from', + 'standard_name': 'wind_from_direction', + 'reference': 'clockwise from Magnetic North', + 'valid_range': (0., 360), + 'units': 'degrees', + 'z' : sensor_info['anemometer1_height'], + 'z_units' : 'meter', + }, + 'wgust1' : {'short_name': 'wgust', + 'long_name': 'Wind Gust', + 'standard_name': 'wind_gust', + 'units': 'm s-1', + 'can_be_normalized': 'no', + 'z' : sensor_info['anemometer1_height'], + 'z_units' : 'meter', + }, + 'wspd1_std' : {'short_name': 'wspd std', + 'long_name': 'Standard Deviation of Wind Speed ', + 'standard_name': 'wind_speed standard_deviation', + 'units': 'm s-1', + 'can_be_normalized': 'no', + 'z' : sensor_info['anemometer1_height'], + 'z_units' : 'meter', + }, + # Second anemometer + 'wspd2' : {'short_name': 'wspd', + 'long_name': 'Wind Speed', + 'standard_name': 'wind_speed', + 'units': 'm s-1', + 'can_be_normalized': 'no', + 'z' : sensor_info['anemometer2_height'], + 'z_units' : 'meter', + }, + 'wdir2' : {'short_name': 'wdir', + 'long_name': 'Wind Direction from', + 'standard_name': 'wind_from_direction', + 'reference': 'clockwise from Magnetic North', + 'valid_range': (0., 360), + 'units': 'degrees', + 'z' : sensor_info['anemometer2_height'], + 'z_units' : 'meter', + }, + 'wgust2' : {'short_name': 'wgust', + 'long_name': 'Wind Gust', + 'standard_name': 'wind_gust', + 'units': 'm s-1', + 'can_be_normalized': 'no', + 'z' : sensor_info['anemometer2_height'], + 'z_units' : 'meter', + }, + 'wspd2_std' : {'short_name': 'wspd std', + 'long_name': 'Standard Deviation of Wind Speed ', + 'standard_name': 'wind_speed standard_deviation', + 'units': 'm s-1', + 'can_be_normalized': 'no', + 'z' : sensor_info['anemometer2_height'], + 'z_units' : 'meter', + }, + } + + # dimension names use tuple so order of initialization is maintained + dim_inits = ( + ('ntime', NC.UNLIMITED), + ('nlat', 1), + ('nlon', 1), + ('nz', 1), + ) + + # using tuple of tuples so order of initialization is maintained + # using dict for attributes order of init not important + # use dimension names not values + # (varName, varType, (dimName1, [dimName2], ...)) + var_inits = ( + # coordinate variables + ('time', NC.INT, ('ntime',)), + ('lat', NC.FLOAT, ('nlat',)), + ('lon', NC.FLOAT, ('nlon',)), + ('z', NC.FLOAT, ('nz',)), + # data variables + ('wspd1', NC.FLOAT, ('ntime',)), + ('wdir1', NC.FLOAT, ('ntime',)), + ('wgust1', NC.FLOAT, ('ntime',)), + ('wspd1_std', NC.FLOAT, ('ntime',)), + ('wspd2', NC.FLOAT, ('ntime',)), + ('wdir2', NC.FLOAT, ('ntime',)), + ('wgust2', NC.FLOAT, ('ntime',)), + ('wspd2_std', NC.FLOAT, ('ntime',)), + ) + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # var data + var_data = ( + ('lat', platform_info['lat']), + ('lon', platform_info['lon']), + ('z', platform_info['altitude']), + # + ('time', data['time'][i]), + # + ('wspd1', data['wspd1'][i]), + ('wdir1', data['wdir1'][i]), + ('wgust1', data['wgust1'][i]), + ('wspd1_std', data['wspd1_std'][i]), + ('wspd2', data['wspd2'][i]), + ('wdir2', data['wdir2'][i]), + ('wgust2', data['wgust2'][i]), + ('wspd2_std', data['wspd2_std'][i]), + ) + + return (global_atts, var_atts, dim_inits, var_inits, var_data) + +def updater(platform_info, sensor_info, data): + # + global_atts = { + # update times of data contained in file (yyyy-mm-dd HH:MM:SS) + # last date in monthly file + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + # + 'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + } + + # data variables + # update any variable attributes like range, min, max + var_atts = {} + # var_atts = { + # 'wtemp': {'max': max(data.u), + # 'min': min(data.v), + # }, + # 'cond': {'max': max(data.u), + # 'min': min(data.v), + # }, + # } + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # data + var_data = ( + ('time', data['time'][i]), + # + ('wspd1', data['wspd1'][i]), + ('wdir1', data['wdir1'][i]), + ('wgust1', data['wgust1'][i]), + ('wspd1_std', data['wspd1_std'][i]), + ('wspd2', data['wspd2'][i]), + ('wdir2', data['wdir2'][i]), + ('wgust2', data['wgust2'][i]), + ('wspd2_std', data['wspd2_std'][i]), + ) + + return (global_atts, var_atts, var_data) +# Index: raw2proc/trunk/raw2proc/proc_jpier_ascii_met.py =================================================================== --- raw2proc/trunk/raw2proc/proc_jpier_ascii_met.py (revision 320) +++ raw2proc/trunk/raw2proc/proc_jpier_ascii_met.py (revision 478) @@ -1,4 +1,4 @@ #!/usr/bin/env python -# Last modified: Time-stamp: <2009-12-15 08:40:54 haines> +# Last modified: Time-stamp: <2011-11-22 13:54:28 haines> """ how to parse data, and assert what data and info goes into @@ -133,5 +133,5 @@ 'metadata_url' : 'http://nccoos.org', 'references' : 'http://nccoos.org', - 'contact' : 'Jesse Cleary (jcleary@email.unc.edu)', + 'contact' : 'Sara Haines (haines@email.unc.edu)', # 'source' : 'TWS Met station observation', Index: raw2proc/trunk/raw2proc/procutil.py =================================================================== --- raw2proc/trunk/raw2proc/procutil.py (revision 448) +++ raw2proc/trunk/raw2proc/procutil.py (revision 478) @@ -1,4 +1,4 @@ #!/usr/bin/env python -# Last modified: Time-stamp: <2010-12-15 09:32:53 haines> +# Last modified: Time-stamp: <2011-12-16 15:18:49 haines> """Utilities to help data processing Index: raw2proc/trunk/raw2proc/raw2proc.py =================================================================== --- raw2proc/trunk/raw2proc/raw2proc.py (revision 454) +++ raw2proc/trunk/raw2proc/raw2proc.py (revision 478) @@ -1,4 +1,4 @@ #!/usr/bin/env python -# Last modified: Time-stamp: <2011-02-17 10:57:48 haines> +# Last modified: Time-stamp: <2011-12-16 17:05:08 haines> """Process raw data to monthly netCDF data files @@ -185,5 +185,8 @@ # "ind" var from filt_datetime() - what level of granularity was used if granularity == 4: - # change dt_start to before monthly filename filt_datetime() date + # change dt_start to before monthly filename filt_datetime() date + # for filenames with just YYYY_MM or YYYYMM add or substract 30 days to + # see if it falls within config range. It won't hurt to add names to files + # parsed. dt_start = si['proc_start_dt']-timedelta(days=31) # print dt_start @@ -209,5 +212,15 @@ elif pi['config_end_date'] == None: config_end_dt = now_dt - + + # # check if raw_files are monthly (granularity == 4) + # for idx, fn in enumerate(raw_files): + # (fndt, granularity) = filt_datetime(os.path.basename(fn), gran=True) + # # "ind" var from filt_datetime() - what level of granularity was used + # if granularity == 4: + # if fndt < config_start_dt: + # dts[idx] = fndt + timedelta(days=31) + # if fndt > config_end_dt: + # dts[idx] = fndt - timedelta(days=31) + new_list = [raw_files[i] for i in range(len(raw_files)) \ if config_start_dt <= dts[i] <= config_end_dt] @@ -368,6 +381,9 @@ ofn = os.path.join(si['proc_dir'], si['proc_filename']) (raw_files, raw_dts) = find_raw(si, yyyy_mm) + print raw_files + print raw_dts raw_files = which_raw(pi, raw_files, raw_dts) - # print raw_files + print raw_files + print raw_dts # remove any previous netcdf file (platform_package_yyyy_mm.nc) if index==0 and os.path.exists(ofn):