Index: raw2proc/trunk/raw2proc/bogue_config_20100712.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/bogue_config_20100712.py (revision 451) @@ -1,0 +1,53 @@ +platform_info = { + 'id' : 'bogue', + 'location' : 'Bogue Inlet Pier, Bogue, NC', + 'lat' : 34.661568, # degrees true (-) south, (+) north + 'lon' : -77.034131, # degrees true (-) west, (+) east + 'mvar' : -9.7, # degrees (-) west, (+) east + 'mean_water_depth' : -8.8, # meters (-) down, (+) up + 'mean_water_depth_time_period' : 'Sept 2009 - Aug 2010', + 'institution' : 'nccoos', + # + 'config_start_date' : '2010-07-12 00:00:00', + 'config_end_date' : None, # None or yyyy-mm-dd HH:MM:SS + 'packages' : ('adcp', 'adcpwaves'), + } +sensor_info = { + 'adcp' : { 'id' : 'adcp', + 'description' : 'Current profile data', + 'raw_dir' : '/seacoos/data/nccoos/level0/bogue/adcp_ascii', + 'raw_file_glob' : '*.wpa', + 'proc_dir' : '/seacoos/data/nccoos/level1/bogue/adcp', + 'process_module' : 'proc_nortek_wpa_adcp', + 'utc_offset' : 4., # hours offset to utc + 'nbins' : 20, + 'bin_size' : 0.5, # meters + 'transducer_ht' : 0.75, # meters above the bottom + 'blanking_ht' : 0.9, # meters above transducer + 'plot_module' : 'bogue_adcp_plot', + 'plot_names' : ('timeseries',), + 'csv_dir' : '/seacoos/data/nccoos/latest_csv', + 'csv_vars' : ('time','u','v','w','wd','wl','water_temp'), + 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + 'latest_vars' : ('time','lat','lon','z','u','v','wd','wl'), + }, + 'adcpwaves' : {'id' : 'adcpwaves', + 'description' : 'Directional wave data', + 'raw_dir' : '/seacoos/data/nccoos/level0/bogue/adcp_ascii', + 'raw_file_glob' : '*.wds', + 'proc_dir' : '/seacoos/data/nccoos/level1/bogue/adcpwaves', + 'process_module' : 'proc_nortek_wds_dw', + 'utc_offset' : 4., # hours offset to utc + 'ndir' : 90., + 'nfreq' : 97., + 'plot_module' : 'bogue_waves_plot', + 'plot_names' : ('allwaves', 'swellwaves', 'windwaves'), + 'csv_dir' : '/seacoos/data/nccoos/latest_csv', + 'csv_vars' : ('time','Hs','Tp','Dp','Dm', + 'Hs_swell','Tp_swell','Dp_swell','Dm_swell', + 'Hs_wind','Tp_wind','Dp_wind','Dm_wind'), + 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + 'latest_vars' : ('time','lat','lon','z','Tp','Hs','Dp'), + }, + } + Index: raw2proc/trunk/raw2proc/proc_avp_ysi_6600_v2_CDL1.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/proc_avp_ysi_6600_v2_CDL1.py (revision 451) @@ -1,0 +1,411 @@ +#!/usr/bin/env python +# Last modified: Time-stamp: <2008-10-09 17:31:44 haines> +""" +how to parse data, and assert what data and info goes into +creating and updating monthly netcdf files + +parse data from YSI 6600 V2-2 on an automated veritical profiler (avp) + +parser : sample date and time, water_depth for each profile + water temperature, conductivity, pressure (depth), salinity, pH, dissolved oxygen, turbidity, and chlorophyll + raw data averaged to 10 cm bins + +creator : lat, lon, z, time, water_depth, water_temp, cond, salin, ph, turb, chl, do +updator : time, water_depth, water_temp, cond, salin, ph, turb, chl, do + + +Examples +-------- + +>> (parse, create, update) = load_processors('proc_avp_ysi_6600_v2') +or +>> si = get_config(cn+'.sensor_info') +>> (parse, create, update) = load_processors(si['adcp']['proc_module']) + +>> lines = load_data(filename) +>> data = parse(platform_info, sensor_info, lines) +>> create(platform_info, sensor_info, data) or +>> update(platform_info, sensor_info, data) + +""" + + +from raw2proc import * +from procutil import * +from ncutil import * + +now_dt = datetime.utcnow() +now_dt.replace(microsecond=0) + +def parser(platform_info, sensor_info, lines): + """ + parse Automated Vertical Profile Station (AVP) Water Quality Data + + month, day, year, hour, min, sec, temp (deg. C), conductivity + (mS/cm), salinity (ppt or PSU), depth (meters), pH, turbidity (NTU), + chlorophyll (micrograms per liter), DO (micrograms per liter) + + Notes + ----- + 1. Column Format + + temp, cond, salin, depth, pH, turb, chl, DO + (C), (mS/cm), (ppt), (m), pH, (NTU), (ug/l), (ug/l) + + Profile Time: 00:30:00 + Profile Date: 08/18/2008 + Profile Depth: 255.0 cm + Profile Location: Stones Bay Serial No: 00016B79, ID: AVP1_SERDP + 08/18/08 00:30:06 26.94 41.87 26.81 0.134 8.00 3.4 4.5 6.60 + 08/18/08 00:30:07 26.94 41.87 26.81 0.143 8.00 3.4 4.8 6.59 + 08/18/08 00:30:08 26.94 41.87 26.81 0.160 8.00 3.4 4.8 6.62 + 08/18/08 00:30:09 26.94 41.87 26.81 0.183 8.00 3.4 4.8 6.66 + + """ + import numpy + from datetime import datetime + from time import strptime + + # get sample datetime from filename + fn = sensor_info['fn'] + sample_dt_start = filt_datetime(fn)[0] + + # how many profiles in one file, count number of "Profile Time:" in lines + nprof = 0 + for line in lines: + m=re.search("Profile Time:", line) + if m: + nprof=nprof+1 + + # remove first occurrence of blank line if within first 10-40 lines + # and put it on the end to signal end of profile after last profile + for i in range(len(lines[0:40])): + if re.search("^ \r\n", lines[i]): + # print str(i) + " " + lines[i] + " " + lines[i+1] + blank_line = lines.pop(i) + lines.append(blank_line) + + bin_size = sensor_info['bin_size'] # Bin Size (meters) + nominal_depth = platform_info['water_depth'] # Mean sea level at station (meters) or nominal water depth + z = numpy.arange(0, -1*nominal_depth, -1*bin_size, dtype=float) + + N = nprof + nbins = len(z) + + if nbins != sensor_info['nbins']: + print 'Number of bins computed from water_depth and bin_size ('+ \ + str(nbins)+') does not match config number ('+ \ + str(sensor_info['nbins'])+')' + + data = { + 'dt' : numpy.array(numpy.ones((N,), dtype=object)*numpy.nan), + 'time' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan), + 'z' : numpy.array(numpy.ones((nbins,), dtype=float)*numpy.nan), + # + 'wd' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan), + 'wtemp' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + 'cond' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + 'salin' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + 'turb' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + 'ph' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + 'chl' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + 'do' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + } + + # current profile count + i = 0 + + for line in lines: + ysi = [] + # split line and parse float and integers + sw = re.split('[\s/\:]*', line) + for s in sw: + m = re.search(REAL_RE_STR, s) + if m: + ysi.append(float(m.groups()[0])) + + if re.search("Profile Time:", line): + HH=ysi[0] + MM=ysi[1] + SS=ysi[2] + elif re.search("Profile Date:", line): + mm=ysi[0] + dd=ysi[1] + yyyy=ysi[2] + elif re.search("Profile Depth:", line): + wd = ysi[0]/100. # cm to meters + sample_str = '%02d-%02d-%4d %02d:%02d:%02d' % (mm,dd,yyyy,HH,MM,SS) + if sensor_info['utc_offset']: + sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%Y %H:%M:%S') + \ + timedelta(hours=sensor_info['utc_offset']) + else: + sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%Y %H:%M:%S') + + # initialize for new profile at zero for averaging samples within each bin + wtemp = numpy.zeros(nbins) + cond = numpy.zeros(nbins) + salin = numpy.zeros(nbins) + turb = numpy.zeros(nbins) + ph = numpy.zeros(nbins) + chl = numpy.zeros(nbins) + do = numpy.zeros(nbins) + Ns = numpy.zeros(nbins) # count samples per bin for averaging + elif len(ysi)==14: + # get sample datetime from data + # sample_str = '%02d-%02d-%2d %02d:%02d:%02d' % tuple(ysi[0:6]) + # if sensor_info['utc_offset']: + # sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%Y %H:%M:%S') + \ + # timedelta(hours=sensor_info['utc_offset']) + # else: + # sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%y %H:%M:%S') + + depth = -1*ysi[9] # depth (m, positive up) + ibin = ((z)<=depth)*(depth<(z+bin_size)) + + Ns[ibin] = Ns[ibin]+1 + wtemp[ibin] = wtemp[ibin]+ysi[6] # water temperature (C) + cond[ibin] = cond[ibin]+ysi[7] # conductivity (mS/cm) + salin[ibin] = salin[ibin]+ysi[8] # salinity (ppt or PSU??) + # + ph[ibin] = ph[ibin]+ysi[10] # ph + turb[ibin] = turb[ibin]+ysi[11] # turbidity (NTU) + chl[ibin] = chl[ibin]+ysi[12] # chlorophyll (ug/l) + do[ibin] = do[ibin]+ysi[13] # dissolved oxygen (mg/l) + + elif (len(ysi)==0): # each profile separated by empty line + # average summations by sample count per bin + # where count is zero make it NaN so average is not divide by zero + Ns[Ns==0]=numpy.nan*Ns[Ns==0] + + data['dt'][i] = sample_dt # sample datetime + data['time'][i] = dt2es(sample_dt) # sample time in epoch seconds + data['wd'][i] = wd + data['z'] = z + # divide by counts + data['wtemp'][i] = wtemp/Ns + data['cond'][i] = cond/Ns + data['salin'][i] = salin/Ns + data['turb'][i] = turb/Ns + data['ph'][i] = ph/Ns + data['chl'][i] = chl/Ns + data['do'][i] = do/Ns + + i=i+1 + + # if-elif + # for line + + return data + + +def creator(platform_info, sensor_info, data): + # + # + title_str = sensor_info['description']+' at '+ platform_info['location'] + global_atts = { + 'title' : title_str, + 'institution' : 'Unversity of North Carolina at Chapel Hill (UNC-CH)', + 'institution_url' : 'http://nccoos.unc.edu', + 'institution_dods_url' : 'http://nccoos.unc.edu', + 'metadata_url' : 'http://nccoos.unc.edu', + 'references' : 'http://nccoos.unc.edu', + 'contact' : 'Sara Haines (haines@email.unc.edu)', + # + 'source' : 'fixed-automated-profiler observation', + 'history' : 'raw2proc using ' + sensor_info['process_module'], + 'comment' : 'File created using pycdf'+pycdfVersion()+' and numpy '+pycdfArrayPkg(), + # conventions + 'Conventions' : 'CF-1.0; SEACOOS-CDL-v2.0', + # SEACOOS CDL codes + 'format_category_code' : 'fixed-profiler', + 'institution_code' : platform_info['institution'], + 'platform_code' : platform_info['id'], + 'package_code' : sensor_info['id'], + # institution specific + 'project' : 'North Carolina Coastal Ocean Observing System (NCCOOS)', + 'project_url' : 'http://nccoos.unc.edu', + # timeframe of data contained in file yyyy-mm-dd HH:MM:SS + # first date in monthly file + 'start_date' : data['dt'][0].strftime("%Y-%m-%d %H:%M:%S"), + # last date in monthly file + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + # + 'creation_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + 'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + 'process_level' : 'level1', + # + # must type match to data (e.g. fillvalue is real if data is real) + '_FillValue' : -99999., + } + + var_atts = { + # coordinate variables + 'time' : {'short_name': 'time', + 'long_name': 'Time', + 'standard_name': 'time', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + 'axis': 'T', + }, + 'lat' : {'short_name': 'lat', + 'long_name': 'Latitude', + 'standard_name': 'latitude', + 'reference':'geographic coordinates', + 'units': 'degrees_north', + 'valid_range':(-90.,90.), + 'axis': 'Y', + }, + 'lon' : {'short_name': 'lon', + 'long_name': 'Longitude', + 'standard_name': 'longitude', + 'reference':'geographic coordinates', + 'units': 'degrees_east', + 'valid_range':(-180.,180.), + 'axis': 'Y', + }, + 'z' : {'short_name': 'z', + 'long_name': 'Height', + 'standard_name': 'height', + 'reference':'zero at sea-surface', + 'positive' : 'up', + 'units': 'm', + 'axis': 'Z', + }, + # data variables + 'wtemp': {'short_name': 'wtemp', + 'long_name': 'Water Temperature', + 'standard_name': 'water_temperature', + 'units': 'degrees Celsius', + }, + 'cond': {'short_name': 'cond', + 'long_name': 'Conductivity', + 'standard_name': 'conductivity', + 'units': 'mS cm-1', + }, + 'salin': {'short_name': 'salin', + 'long_name': 'Salinity', + 'standard_name': 'salinity', + 'units': 'PSU', + }, + 'depth': {'short_name': 'depth', + 'long_name': 'Depth', + 'standard_name': 'depth', + 'units': 'm', + 'reference':'zero at sea-surface', + 'positive' : 'up', + }, + 'turb': {'short_name': 'turb', + 'long_name': 'Turbidity', + 'standard_name': 'turbidity', + 'units': 'NTU', + }, + 'ph': {'short_name': 'ph', + 'long_name': 'pH', + 'standard_name': 'ph', + 'units': '', + }, + 'chl': {'short_name': 'chl', + 'long_name': 'Chlorophyll', + 'standard_name': 'chlorophyll', + 'units': 'ug l-1', + }, + 'do': {'short_name': 'do', + 'long_name': 'Dissolved Oxygen', + 'standard_name': 'dissolved_oxygen', + 'units': 'mg l-1', + }, + } + + # dimension names use tuple so order of initialization is maintained + dim_inits = ( + ('ntime', NC.UNLIMITED), + ('nlat', 1), + ('nlon', 1), + ('nz', 1), + ) + + # using tuple of tuples so order of initialization is maintained + # using dict for attributes order of init not important + # use dimension names not values + # (varName, varType, (dimName1, [dimName2], ...)) + var_inits = ( + # coordinate variables + ('time', NC.INT, ('ntime',)), + ('lat', NC.FLOAT, ('nlat',)), + ('lon', NC.FLOAT, ('nlon',)), + ('z', NC.FLOAT, ('nz',)), + # data variables + ('wtemp', NC.FLOAT, ('ntime',)), + ('cond', NC.FLOAT, ('ntime',)), + ('salin', NC.FLOAT, ('ntime',)), + ('depth', NC.FLOAT, ('ntime',)), + ('turb', NC.FLOAT, ('ntime',)), + ('ph', NC.FLOAT, ('ntime',)), + ('chl', NC.FLOAT, ('ntime',)), + ('do', NC.FLOAT, ('ntime',)), + ) + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # var data + var_data = ( + ('lat', platform_info['lat']), + ('lon', platform_info['lon']), + # + ('time', data['time'][i]), + ('z', 0), + # + ('wtemp', data['wtemp'][i]), + ('cond', data['cond'][i]), + ('salin', data['salin'][i]), + ('depth', data['depth'][i]), + ('turb', data['turb'][i]), + ('ph', data['ph'][i]), + ('chl', data['chl'][i]), + ('do', data['do'][i]), + ) + + return (global_atts, var_atts, dim_inits, var_inits, var_data) + +def updater(platform_info, sensor_info, data): + # + global_atts = { + # update times of data contained in file (yyyy-mm-dd HH:MM:SS) + # last date in monthly file + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + # + 'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + } + + # data variables + # update any variable attributes like range, min, max + var_atts = {} + # var_atts = { + # 'wtemp': {'max': max(data.u), + # 'min': min(data.v), + # }, + # 'cond': {'max': max(data.u), + # 'min': min(data.v), + # }, + # } + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # data + var_data = ( + ('time', data['time'][i]), + ('wtemp', data['wtemp'][i]), + ('cond', data['cond'][i]), + ('salin', data['salin'][i]), + ('depth', data['depth'][i]), + ('turb', data['turb'][i]), + ('ph', data['ph'][i]), + ('chl', data['chl'][i]), + ('do', data['do'][i]), + ) + + return (global_atts, var_atts, var_data) +# Index: raw2proc/trunk/raw2proc/proc_avp_ysi_6600_v2_moving_point.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/proc_avp_ysi_6600_v2_moving_point.py (revision 451) @@ -1,0 +1,444 @@ +#!/usr/bin/env python +# Last modified: Time-stamp: <2011-05-05 14:43:47 haines> +""" +how to parse data, and assert what data and info goes into +creating and updating monthly netcdf files + +parse data from YSI 6600 V2-2 on an automated veritical profiler (avp) + +parser : date and time, water_depth for each profile + + sample time, sample depth, as cast measures water + temperature, conductivity, salinity, pH, dissolved oxygen, + turbidity, and chlorophyll + + +creator : lat, lon, z, time, water_depth, water_temp, cond, + salin, ph, turb, chl, do + +updator : z, time, water_depth, water_temp, cond, salin, ph, + turb, chl, do + +using moving point CDL + + +Examples +-------- + +>> (parse, create, update) = load_processors('proc_avp_ysi_6600_v2') +or +>> si = get_config(cn+'.sensor_info') +>> (parse, create, update) = load_processors(si['adcp']['proc_module']) + +>> lines = load_data(filename) +>> data = parse(platform_info, sensor_info, lines) +>> create(platform_info, sensor_info, data) or +>> update(platform_info, sensor_info, data) + +""" + + +from raw2proc import * +from procutil import * +from ncutil import * + +now_dt = datetime.utcnow() +now_dt.replace(microsecond=0) + +def parser(platform_info, sensor_info, lines): + """ + parse Automated Vertical Profile Station (AVP) Water Quality Data + + month, day, year, hour, min, sec, temp (deg. C), conductivity + (mS/cm), salinity (ppt or PSU), depth (meters), pH, turbidity (NTU), + chlorophyll (micrograms per liter), DO (micrograms per liter) + + Notes + ----- + 1. Column Format + + temp, cond, salin, depth, pH, turb, chl, DO + (C), (mS/cm), (ppt), (m), pH, (NTU), (ug/l), (ug/l) + + Profile Time: 00:30:00 + Profile Date: 08/18/2008 + Profile Depth: 255.0 cm + Profile Location: Stones Bay Serial No: 00016B79, ID: AVP1_SERDP + 08/18/08 00:30:06 26.94 41.87 26.81 0.134 8.00 3.4 4.5 6.60 + 08/18/08 00:30:07 26.94 41.87 26.81 0.143 8.00 3.4 4.8 6.59 + 08/18/08 00:30:08 26.94 41.87 26.81 0.160 8.00 3.4 4.8 6.62 + 08/18/08 00:30:09 26.94 41.87 26.81 0.183 8.00 3.4 4.8 6.66 + + + """ + import numpy + from datetime import datetime + from time import strptime + + # get sample datetime from filename + fn = sensor_info['fn'] + sample_dt_start = filt_datetime(fn) + + # how many samples + nsamp = 0 + for line in lines: + # if line has weird ascii chars -- skip it and iterate to next line + if re.search(r"[\x1a]", line): + # print 'skipping bad data line ... ' + str(line) + continue + m=re.search("^\d{2}\/\d{2}\/\d{2}", line) + if m: + nsamp=nsamp+1 + + N = nsamp + + data = { + 'dt' : numpy.array(numpy.ones((N,), dtype=object)*numpy.nan), + 'time' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan), + 'z' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan), + 'wd' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan), + 'wl' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan), + 'batt' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'wtemp' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'cond' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'salin' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'turb' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'ph' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'chl' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'do' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + } + + # setting all dates to this old data so eliminated for this month + for i in range(N): + data['dt'][i] = datetime(1970,1,1) + + # sample count + i = 0 + + for line in lines: + # if line has weird ascii chars -- skip it and iterate to next line + if re.search(r"[\x1a]", line): + # print 'skipping bad data line ... ' + str(line) + continue + + ysi = [] + # split line and parse float and integers + sw = re.split('[\s/\:]*', line) + for s in sw: + m = re.search(REAL_RE_STR, s) + if m: + ysi.append(float(m.groups()[0])) + + if re.search("Profile Depth:", line) and i=6 and len(ysi)<14): + print 'skipping bad data line ... ' + str(line) + + # if-elif + # for line + + return data + + +def creator(platform_info, sensor_info, data): + # + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + dt = data['dt'][i] + # + title_str = sensor_info['description']+' at '+ platform_info['location'] + global_atts = { + 'title' : title_str, + 'institution' : 'Unversity of North Carolina at Chapel Hill (UNC-CH)', + 'institution_url' : 'http://nccoos.unc.edu', + 'institution_dods_url' : 'http://nccoos.unc.edu', + 'metadata_url' : 'http://nccoos.unc.edu', + 'references' : 'http://nccoos.unc.edu', + 'contact' : 'Sara Haines (haines@email.unc.edu)', + # + 'source' : 'fixed-automated-profiler observation', + 'history' : 'raw2proc using ' + sensor_info['process_module'], + 'comment' : 'File created using pycdf'+pycdfVersion()+' and numpy '+pycdfArrayPkg(), + # conventions + 'Conventions' : 'CF-1.0; SEACOOS-CDL-v2.0', + # SEACOOS CDL codes + 'format_category_code' : 'fixed-profiler-ragged', + 'institution_code' : platform_info['institution'], + 'platform_code' : platform_info['id'], + 'package_code' : sensor_info['id'], + # institution specific + 'project' : 'North Carolina Coastal Ocean Observing System (NCCOOS)', + 'project_url' : 'http://nccoos.unc.edu', + # timeframe of data contained in file yyyy-mm-dd HH:MM:SS + # first date in monthly file + 'start_date' : dt[0].strftime("%Y-%m-%d %H:%M:%S"), + # last date in monthly file + 'end_date' : dt[-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + # + 'creation_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + 'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + 'process_level' : 'level1', + # + # must type match to data (e.g. fillvalue is real if data is real) + '_FillValue' : numpy.nan, + } + + var_atts = { + # coordinate variables + 'time' : {'short_name': 'time', + 'long_name': 'Time of Profile', + 'standard_name': 'time', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + 'axis': 'T', + }, + 'lat' : {'short_name': 'lat', + 'long_name': 'Latitude', + 'standard_name': 'latitude', + 'reference':'geographic coordinates', + 'units': 'degrees_north', + 'valid_range':(-90.,90.), + 'axis': 'Y', + }, + 'lon' : {'short_name': 'lon', + 'long_name': 'Longitude', + 'standard_name': 'longitude', + 'reference':'geographic coordinates', + 'units': 'degrees_east', + 'valid_range':(-180.,180.), + 'axis': 'Y', + }, + 'z' : {'short_name': 'z', + 'long_name': 'z', + 'standard_name': 'z', + 'reference':'zero is surface', + 'positive' : 'up', + 'units': 'm', + 'axis': 'Z', + }, + # data variables + 'batt': {'short_name': 'batt', + 'long_name': 'Battery', + 'standard_name': 'battery_voltage', + 'units': 'volts', + }, + 'wd': {'short_name': 'wd', + 'long_name': 'Water Depth', + 'standard_name': 'water_depth', + 'reference' : 'zero at sea-surface', + 'positive' : 'up', + 'units': 'm', + }, + 'wl': {'short_name': 'wl', + 'long_name': 'Water Level', + 'standard_name': 'water_level', + 'reference':'MSL', + 'reference_to_MSL' : 0., + 'reference_MSL_datum' : platform_info['mean_water_depth'], + 'reference_MSL_datum_time_period' : platform_info['mean_water_depth_time_period'], + 'positive' : 'up', + 'z' : 0., + 'units': 'm', + }, + 'wtemp': {'short_name': 'wtemp', + 'long_name': 'Water Temperature', + 'standard_name': 'water_temperature', + 'units': 'degrees_Celsius', + }, + 'cond': {'short_name': 'cond', + 'long_name': 'Conductivity', + 'standard_name': 'conductivity', + 'units': 'mS cm-1', + }, + 'salin': {'short_name': 'salin', + 'long_name': 'Salinity', + 'standard_name': 'salinity', + 'units': 'PSU', + }, + 'turb': {'short_name': 'turb', + 'long_name': 'Turbidity', + 'standard_name': 'turbidity', + 'units': 'NTU', + }, + 'ph': {'short_name': 'ph', + 'long_name': 'pH', + 'standard_name': 'ph', + 'units': '', + }, + 'chl': {'short_name': 'chl', + 'long_name': 'Chlorophyll', + 'standard_name': 'chlorophyll', + 'units': 'ug l-1', + }, + 'do': {'short_name': 'do', + 'long_name': 'Dissolved Oxygen', + 'standard_name': 'dissolved_oxygen', + 'units': 'mg l-1', + }, + } + + # dimension names use tuple so order of initialization is maintained + dim_inits = ( + ('ntime', NC.UNLIMITED), + ('nlat', 1), + ('nlon', 1), + ) + + # using tuple of tuples so order of initialization is maintained + # using dict for attributes order of init not important + # use dimension names not values + # (varName, varType, (dimName1, [dimName2], ...)) + var_inits = ( + # coordinate variables + ('time', NC.INT, ('ntime',)), + ('lat', NC.FLOAT, ('nlat',)), + ('lon', NC.FLOAT, ('nlon',)), + ('z', NC.FLOAT, ('ntime',)), + # data variables + ('batt', NC.FLOAT, ('ntime',)), + ('wd', NC.FLOAT, ('ntime',)), + ('wl', NC.FLOAT, ('ntime',)), + # + ('wtemp', NC.FLOAT, ('ntime',)), + ('cond', NC.FLOAT, ('ntime',)), + ('salin', NC.FLOAT, ('ntime',)), + ('turb', NC.FLOAT, ('ntime',)), + ('ph', NC.FLOAT, ('ntime',)), + ('chl', NC.FLOAT, ('ntime',)), + ('do', NC.FLOAT, ('ntime',)), + ) + + # var data + var_data = ( + ('lat', platform_info['lat']), + ('lon', platform_info['lon']), + ('time', data['time'][i]), + ('z', data['z'][i]), + # + ('batt', data['batt'][i]), + ('wd', data['wd'][i]), + ('wl', data['wl'][i]), + # + ('wtemp', data['wtemp'][i]), + ('cond', data['cond'][i]), + ('salin', data['salin'][i]), + ('turb', data['turb'][i]), + ('ph', data['ph'][i]), + ('chl', data['chl'][i]), + ('do', data['do'][i]), + ) + + return (global_atts, var_atts, dim_inits, var_inits, var_data) + +def updater(platform_info, sensor_info, data): + # + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + dt = data['dt'][i] + # + global_atts = { + # update times of data contained in file (yyyy-mm-dd HH:MM:SS) + # last date in monthly file + 'end_date' : dt[-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + # + 'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), + } + + # data variables + # update any variable attributes like range, min, max + var_atts = {} + # var_atts = { + # 'wtemp': {'max': max(data.u), + # 'min': min(data.v), + # }, + # 'cond': {'max': max(data.u), + # 'min': min(data.v), + # }, + # } + + # data + var_data = ( + ('time', data['time'][i]), + ('z', data['z'][i]), + # + ('batt', data['batt'][i]), + ('wd', data['wd'][i]), + ('wl', data['wl'][i]), + # + ('wtemp', data['wtemp'][i]), + ('cond', data['cond'][i]), + ('salin', data['salin'][i]), + ('turb', data['turb'][i]), + ('ph', data['ph'][i]), + ('chl', data['chl'][i]), + ('do', data['do'][i]), + ) + + return (global_atts, var_atts, var_data) +# Index: raw2proc/trunk/raw2proc/qcutil.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/qcutil.py (revision 451) @@ -1,0 +1,69 @@ +#!/usr/bin/env python +# Last modified: Time-stamp: <2011-05-11 15:59:31 haines> +""" +Quality Control Tests and Utilities +""" + +def range_test(val, lower_limit, upper_limit): + """ QC Range Test + + Test whether value is inclusively within established upper and lower limit + + :Parameters: + val : scalar, list, or numpy.array + value(s) to be tested + lower_limit : scalar + Any value less than lower limit fails + upper_limit : scalar + Any value more than upper limit fails + + :Returns: + flag : boolean, list, numpy.array + True = Pass + False = Fail + + """ + flag = (val > lower_limit) & (val < upper_limit) + return (flag) + +def time_continuity_test(val, factor, v1, v2, t1, t2): + """ QC Time-Continuity Test + + Checks the amount of change in each measurment's value over the + given time period. If same or less than empirical form or constant + maximum of allowable change for a given time difference, then the + test passes. + + """ + flag = abs((v2-v1)/(t2-t1)) <= factor + return (flag) + +# **** pack and unpack go in ncutil.py +def nc_pack_qcflags(qcflag, rtype='int8'): + """ Pack boolean numpy.array and cast for netcdf + padding where necessary + """ + # check if boolean + + # convert bool to int (0,1) + q1=qcflag.astype(int) + # pack 1s and 0s into a uint8 array + # (other dtype in this step not optional with numpy.packbits) + q2=numpy.packbits(q1,axis=1) + # cast uint8 to int8 (NC_BYTE) + qcflag=numpy.cast['int8'](q2) + + return (qcflag) + +def nc_unpack_qcflags(qcflag): + """ + """ + + # check if uint8 (?) + + # cast int8 (NC_BYTE) to uint8 + q1=numpy.cast['uint8'](qcflag) + # unpack uint8 to 1s and 0s + q2=numpy.unpackbits(q1, axis=1) + # + qcflags=q2.astype(bool) Index: raw2proc/trunk/raw2proc/scr_read_avp_ysi_6600_v2_fixed_profiler.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/scr_read_avp_ysi_6600_v2_fixed_profiler.py (revision 451) @@ -1,0 +1,221 @@ +#!/usr/bin/env python +# Last modified: Time-stamp: <2008-09-09 12:56:46 haines> +""" +parse ascii text file of YSI 6600 V2 water quality data (.dat) + +load data file +parse data into variables for appending to netCDF data + +""" + +REAL_RE_STR = '\\s*(-?\\d(\\.\\d+|)[Ee][+\\-]\\d\\d?|-?(\\d+\\.\\d*|\\d*\\.\\d+)|-?\\d+)\\s*' + +import sys +import os +import re + +def parse_avp_YSI_6600V2(fn, lines): + """ + parse Automated Vertical Profile Station (AVP) Water Quality Data + + month, day, year, hour, min, sec, temp (deg. C), conductivity + (mS/cm), salinity (ppt or PSU), depth (meters), pH, turbidity (NTU), + chlorophyll (micrograms per liter), DO (micrograms per liter) + + Notes + ----- + 1. Column Format + + temp, cond, salin, depth, pH, turb, chl, DO + (C), (mS/cm), (ppt), (m), pH, (NTU), (ug/l), (ug/l) + + Profile Time: 00:30:00 + Profile Date: 08/18/2008 + Profile Depth: 255.0 cm + Profile Location: Stones Bay Serial No: 00016B79, ID: AVP1_SERDP + 08/18/08 00:30:06 26.94 41.87 26.81 0.134 8.00 3.4 4.5 6.60 + 08/18/08 00:30:07 26.94 41.87 26.81 0.143 8.00 3.4 4.8 6.59 + 08/18/08 00:30:08 26.94 41.87 26.81 0.160 8.00 3.4 4.8 6.62 + 08/18/08 00:30:09 26.94 41.87 26.81 0.183 8.00 3.4 4.8 6.66 + + 2. While each parameter is measured uniquely with time and depth such that, temp(t) and z(t) + match up with time, we want to grid depth every 1 cm and make each param as temp(t,z). + + Tony Whipple at IMS says 'The AVPs sample at one second intervals. + Between the waves and the instrument descending from a spool of + line with variable radius it works out to about 3-5 cm between + observations on average. When I process the data to make the + images, I bin the data every 10 cm and take the average of however + many observations fell within that bin.' + + Do we interpolate or average samples in bin? + + """ + import numpy + from datetime import datetime + from time import strptime + + # get sample datetime from filename + # fn = sensor_info['fn'] + sample_dt_start = filt_datetime(fn)[0] + + # how many profiles in one file, count number of "Profile Time:" in lines + nprof = 0 + for line in lines: + m=re.search("Profile Time:", line) + if m: + nprof=nprof+1 + + # remove first occurrence of blank line if within first 10-40 lines + # and put it on the end to signal end of profile after last profile + for i in range(len(lines[0:40])): + if re.search("^ \r\n", lines[i]): + # print str(i) + " " + lines[i] + " " + lines[i+1] + blank_line = lines.pop(i) + lines.append(blank_line) + + bin_size = 0.1 # 10cm or 0.1m + z = numpy.arange(0,4.0,bin_size,dtype=float) + + N = nprof + nbins = len(z) + data = { + 'dt' : numpy.array(numpy.ones((N,), dtype=object)*numpy.nan), + 'time' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan), + 'water_depth' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan), + 'wtemp' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + 'cond' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + 'salin' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + 'turb' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + 'ph' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + 'chl' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + 'do' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + } + + # current profile count + i = 0 + + for line in lines: + ysi = [] + # split line and parse float and integers + sw = re.split('[\s/\:]*', line) + for s in sw: + m = re.search(REAL_RE_STR, s) + if m: + ysi.append(float(m.groups()[0])) + + if re.search("Profile Time:", line): + HH=ysi[0] + MM=ysi[1] + SS=ysi[2] + + elif re.search("Profile Date:", line): + dd=ysi[0] + mm=ysi[1] + yyyy=ysi[2] + + elif re.search("Profile Depth:", line): + water_depth = ysi[0]/100. # cm to meters + sample_str = '%02d-%02d-%d %02d:%02d:%02d' % (mm,dd,yyyy,HH,MM,SS) + # if sensor_info['utc_offset']: + # sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%Y %H:%M:%S') + \ + # timedelta(hours=sensor_info['utc_offset']) + # else: + sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%Y %H:%M:%S') + + # initialize for new profile at zero for averaging samples within each bin + wtemp = numpy.zeros(nbins) + cond = numpy.zeros(nbins) + salin = numpy.zeros(nbins) + turb = numpy.zeros(nbins) + ph = numpy.zeros(nbins) + chl = numpy.zeros(nbins) + do = numpy.zeros(nbins) + + Ns = numpy.zeros(nbins) # count samples per bin for averaging + + elif len(ysi)==14: + # get sample datetime from data + # sample_str = '%02d-%02d-%2d %02d:%02d:%02d' % tuple(ysi[0:6]) + # if sensor_info['utc_offset']: + # sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%Y %H:%M:%S') + \ + # timedelta(hours=sensor_info['utc_offset']) + # else: + # sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%y %H:%M:%S') + + depth = ysi[9] # depth (m) + ibin = ((z)<=depth)*(depth<(z+bin_size)) + + Ns[ibin] = Ns[ibin]+1 + wtemp[ibin] = wtemp[ibin]+ysi[6] # water temperature (C) + cond[ibin] = cond[ibin]+ysi[7] # conductivity (mS/cm) + salin[ibin] = salin[ibin]+ysi[8] # salinity (ppt or PSU??) + # + ph[ibin] = ph[ibin]+ysi[10] # ph + turb[ibin] = turb[ibin]+ysi[11] # turbidity (NTU) + chl[ibin] = chl[ibin]+ysi[12] # chlorophyll (ug/l) + do[ibin] = do[ibin]+ysi[13] # dissolved oxygen (ug/l) + + elif (len(ysi)==0): + # average summations by sample count per bin + # where count is zero make it NaN so average is not divide by zero + Ns[Ns==0]=numpy.nan*Ns[Ns==0] + + data['dt'][i] = sample_dt # sample datetime + data['time'][i] = dt2es(sample_dt) # sample time in epoch seconds + data['water_depth'][i] = water_depth + # divide by counts + data['wtemp'][i] = wtemp/Ns + data['cond'][i] = cond/Ns + data['salin'][i] = salin/Ns + data['turb'][i] = turb/Ns + data['ph'][i] = ph/Ns + data['chl'][i] = chl/Ns + data['do'][i] = do/Ns + + i=i+1 + + # if-elif + # for line + + return data + + +def load_data(inFile): + lines=None + if os.path.exists(inFile): + f = open(inFile, 'r') + lines = f.readlines() + f.close() + if len(lines)<=0: + print 'Empty file: '+ inFile + else: + print 'File does not exist: '+ inFile + return lines + +# from jpier_config_20080411 import * +from raw2proc import * + +def test1(fn): + lines = load_data(fn) + return parse_avp_YSI_6600V2(fn, lines) + +def test2(logFile): + pi = get_config('stones_config_YYYYMMDD.platform_info') + asi = get_config('stones_config_YYYYMMDD.sensor_info') + si = asi['met'] + lines = load_data(logFile) + si['fn'] = logFile + (parse, create, update) = import_processors(si['process_module']) + return parse(pi, si, logFile) + +if __name__ == '__main__': + fn = '/seacoos/data/nccoos/level0/stones/avp/2008_08/AVP1_20080811.dat' + # dataFile = 'D:/haines/nc-coos/raw2proc/stones/met/2008_08/AVP1_20080811.wnd' + + # logFile = sys.argv[1] + try: + data = test1(fn) + except: + pass + Index: raw2proc/trunk/raw2proc/scr_read_avp_ysi_6600_v2_moving_point.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/scr_read_avp_ysi_6600_v2_moving_point.py (revision 451) @@ -1,0 +1,165 @@ +#!/usr/bin/env python +# Last modified: Time-stamp: <2011-02-16 16:32:33 haines> +""" +parse ascii text file of YSI 6600 V2 water quality data (.dat) + +load data file +parse data into variables for appending to netCDF data + +water depth is read from the file but stored in separate netCDF +because of different sample interval + +""" + +REAL_RE_STR = '\\s*(-?\\d(\\.\\d+|)[Ee][+\\-]\\d\\d?|-?(\\d+\\.\\d*|\\d*\\.\\d+)|-?\\d+)\\s*' + +import sys +import os +import re + +def parse_avp_YSI_6600V2(fn, lines): + """ + parse Automated Vertical Profile Station (AVP) Water Quality Data + + month, day, year, hour, min, sec, temp (deg. C), conductivity + (mS/cm), salinity (ppt or PSU), depth (meters), pH, turbidity (NTU), + chlorophyll (micrograms per liter), DO (micrograms per liter) + + Notes + ----- + 1. Column Format + + temp, cond, salin, depth, pH, turb, chl, DO + (C), (mS/cm), (ppt), (m), pH, (NTU), (ug/l), (ug/l) + + Profile Time: 00:30:00 + Profile Date: 08/18/2008 + Profile Depth: 255.0 cm + Profile Location: Stones Bay Serial No: 00016B79, ID: AVP1_SERDP + 08/18/08 00:30:06 26.94 41.87 26.81 0.134 8.00 3.4 4.5 6.60 + 08/18/08 00:30:07 26.94 41.87 26.81 0.143 8.00 3.4 4.8 6.59 + 08/18/08 00:30:08 26.94 41.87 26.81 0.160 8.00 3.4 4.8 6.62 + 08/18/08 00:30:09 26.94 41.87 26.81 0.183 8.00 3.4 4.8 6.66 + + 2. read each sample and create timeseries for each parameter along + with the recorded depth of each sample, z(t) + + """ + import numpy + from datetime import datetime + from time import strptime + + # get sample datetime from filename + # fn = sensor_info['fn'] + sample_dt_start = filt_datetime(fn)[0] + + # how many samples + nsamp = 0 + for line in lines: + m=re.search("^\d{2}\/\d{2}\/\d{2}", line) + if m: + nsamp=nsamp+1 + + N = nsamp + data = { + 'dt' : numpy.array(numpy.ones((N,), dtype=object)*numpy.nan), + 'time' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan), + 'wtemp' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'cond' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'salin' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'depth' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'turb' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'ph' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'chl' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + 'do' : numpy.array(numpy.ones((N,), dtype=float)*numpy.nan), + } + + # sample count + i = 0 + + for line in lines: + ysi = [] + # split line and parse float and integers + sw = re.split('[\s/\:]*', line) + for s in sw: + m = re.search(REAL_RE_STR, s) + if m: + ysi.append(float(m.groups()[0])) + + if len(ysi)==14: + # get sample datetime from data + sample_str = '%02d-%02d-%2d %02d:%02d:%02d' % tuple(ysi[0:6]) + # if sensor_info['utc_offset']: + # sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%Y %H:%M:%S') + \ + # timedelta(hours=sensor_info['utc_offset']) + # else: + sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%y %H:%M:%S') + + wtemp[i] = ysi[6] # water temperature (C) + cond[i] = ysi[7] # conductivity (mS/cm) + salin[i] = ysi[8] # salinity (ppt or PSU??) + depth[i] = ysi[9] # depth (m) + # + ph[i] = ysi[10] # ph + turb[i] = ysi[11] # turbidity (NTU) + chl[i] = ysi[12] # chlorophyll (ug/l) + do[i] = ysi[13] # dissolved oxygen (ug/l) + + data['dt'][i] = sample_dt # sample datetime + data['time'][i] = dt2es(sample_dt) # sample time in epoch seconds + # + data['wtemp'][i] = wtemp + data['cond'][i] = cond + data['salin'][i] = salin + data['depth'][i] = depth + data['turb'][i] = turb + data['ph'][i] = ph + data['chl'][i] = chl + data['do'][i] = do + + i=i+1 + + # if-elif + # for line + + return data + + +def load_data(inFile): + lines=None + if os.path.exists(inFile): + f = open(inFile, 'r') + lines = f.readlines() + f.close() + if len(lines)<=0: + print 'Empty file: '+ inFile + else: + print 'File does not exist: '+ inFile + return lines + +# from jpier_config_20080411 import * +from raw2proc import * + +def test1(fn): + lines = load_data(fn) + return parse_avp_YSI_6600V2(fn, lines) + +def test2(logFile): + pi = get_config('stones_config_YYYYMMDD.platform_info') + asi = get_config('stones_config_YYYYMMDD.sensor_info') + si = asi['met'] + lines = load_data(logFile) + si['fn'] = logFile + (parse, create, update) = import_processors(si['process_module']) + return parse(pi, si, logFile) + +if __name__ == '__main__': + fn = '/seacoos/data/nccoos/level0/stones/avp/2008_08/AVP1_20080811.dat' + # dataFile = 'D:/haines/nc-coos/raw2proc/stones/met/2008_08/AVP1_20080811.wnd' + + # logFile = sys.argv[1] + try: + data = test1(fn) + except: + pass + Index: raw2proc/trunk/raw2proc/spin_morgan_avp.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/spin_morgan_avp.py (revision 451) @@ -1,0 +1,41 @@ +#!/usr/bin/env python +# Last modified: Time-stamp: <2011-03-01 09:26:47 haines> +"""Spin processing of raw data to monthly netCDF data files""" + +from raw2proc import * + +# raw2proc('manual', 'morgan', 'avp', '2011_02') +# raw2proc('manual', 'morgan', 'avp', '2011_01') + +# raw2proc('manual', 'morgan', 'avp', '2010_12') +# raw2proc('manual', 'morgan', 'avp', '2010_11') +# raw2proc('manual', 'morgan', 'avp', '2010_10') +# raw2proc('manual', 'morgan', 'avp', '2010_09') +# raw2proc('manual', 'morgan', 'avp', '2010_08') +# raw2proc('manual', 'morgan', 'avp', '2010_07') +# raw2proc('manual', 'morgan', 'avp', '2010_06') +# raw2proc('manual', 'morgan', 'avp', '2010_05') +# raw2proc('manual', 'morgan', 'avp', '2010_04') +# raw2proc('manual', 'morgan', 'avp', '2010_03') +# raw2proc('manual', 'morgan', 'avp', '2010_02') +# raw2proc('manual', 'morgan', 'avp', '2010_01') + +# raw2proc('manual', 'morgan', 'avp', '2009_12') +# raw2proc('manual', 'morgan', 'avp', '2009_11') +# raw2proc('manual', 'morgan', 'avp', '2009_10') +# raw2proc('manual', 'morgan', 'avp', '2009_09') +# raw2proc('manual', 'morgan', 'avp', '2009_08') +# raw2proc('manual', 'morgan', 'avp', '2009_07') +# raw2proc('manual', 'morgan', 'avp', '2009_06') +# raw2proc('manual', 'morgan', 'avp', '2009_05') +# raw2proc('manual', 'morgan', 'avp', '2009_04') +# raw2proc('manual', 'morgan', 'avp', '2009_03') +raw2proc('manual', 'morgan', 'avp', '2009_02') +raw2proc('manual', 'morgan', 'avp', '2009_01') + +raw2proc('manual', 'morgan', 'avp', '2008_12') +raw2proc('manual', 'morgan', 'avp', '2008_11') +raw2proc('manual', 'morgan', 'avp', '2008_10') +raw2proc('manual', 'morgan', 'avp', '2008_09') +raw2proc('manual', 'morgan', 'avp', '2008_08') +raw2proc('manual', 'morgan', 'avp', '2008_07') Index: raw2proc/trunk/raw2proc/test_codar.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/test_codar.py (revision 451) @@ -1,0 +1,106 @@ +#!/usr/bin/env python +# Last modified: Time-stamp: <2010-12-09 16:17:25 haines> + +import sys +import os +import re +from procutil import * +from raw2proc import * + +def load_data(inFile): + lines=None + if os.path.exists(inFile): + f = open(inFile, 'r') + lines = f.readlines() + f.close() + if len(lines)<=0: + print 'Empty file: '+ inFile + else: + print 'File does not exist: '+ inFile + return lines + +import numpy +from datetime import datetime +from time import strptime +from StringIO import StringIO +from matplotlib.mlab import griddata + +fn = '/seacoos/data/nc-coos/level0/ouba/hfr_totals/2010_07/TOTL_OUBA_2010_07_14_0000.tuv' +lines = load_data(fn) + +# get sample datetime from filename +# fn = sensor_info['fn'] +# sample_dt_start = filt_datetime(fn) + +# read header that match '%(k): (v)\n' pairs on each line +m = re.findall(r'^(%.*):\s*(.*)$', ''.join(lines), re.MULTILINE) +for k,v in m: + if k == '%TimeStamp': + sample_dt = scanf_datetime(v, fmt='%Y %m %d %H %M %S') + elif k == '%TableType': + ftype = v + elif k == '%TableColumns': + ncol = int(v) + elif k == '%TableRows': + nrow = int(v) + +# read data from string of lines but make it behave like a file object with StringIO +s = StringIO(''.join(lines)) +s.seek(0) # ensures start posn of file +d = numpy.loadtxt(s, comments='%') +# lat, lon, u, v = numpy.loadtxt(s, usecols=(0,1,2,3), comments='%', unpack=True) +if 'TOT4' in ftype: + lon = d[:,0] + lat = d[:,1] + wu = d[:,2] + wv = d[:,3] + gridflag = d[:,4] + wu_std_qual = d[:,5] + wv_std_qual = d[:,6] + cov_qual = d[:,7] + x_dist = d[:,8] + y_dist = d[:,9] + rang = d[:,10] + bearing = d[:,11] + vel_mag = d[:,12] + vel_dir = d[:,13] + s1 = d[:,14] + s2 = d[:,15] + s3 = d[:,16] + s4 = d[:,17] + s5 = d[:,18] + s6 = d[:,19] + +# define the lat/lon grid based on 6km resolution +# For Outer Banks north and east of Cape Hatteras, bounding box is defined by: +minlat, maxlat = (34.5, 38) +minlon, maxlon = (-76, -73.) +midlat = minlat + 0.5*(maxlat-minlat) +# ~111 km = 1 deg latitude +nlat = numpy.round((maxlat-minlat) *111/6) +nlon = numpy.round((maxlon-minlon) * math.cos(midlat*math.pi/180)*111/6) +yi = numpy.linspace(minlat, maxlat, nlat) +xi = numpy.linspace(minlon, maxlon, nlon) +ui = griddata(lon, lat, wu, xi, yi) +vi = griddata(lon, lat, wv, xi, yi) +# --------------------------------------------------------------- +data = { + 'dt' : numpy.array(numpy.ones((1,), dtype=object)*numpy.nan), + 'time' : numpy.array(numpy.ones((1,), dtype=long)*numpy.nan), + 'lon' : numpy.array(numpy.ones((1,nlon), dtype=float)*numpy.nan), + 'lat' : numpy.array(numpy.ones((1,nlat), dtype=float)*numpy.nan), + 'u' : numpy.array(numpy.ones((1,nlon,nlat), dtype=float)*numpy.nan), + 'v' : numpy.array(numpy.ones((1,nlon,nlat), dtype=float)*numpy.nan), + } +# --------------------------------------------------------------- +i = 0 +data['dt'][i] = sample_dt # +data['time'][i] = dt2es(sample_dt) # +data['lon'][i] = xi # new longitude grid +data['lat'][i] = yi # new latitude grid +data['u'][i] = ui # u-component of water velocity (cm/s) +data['v'][i] = vi # v-component of water velocity + + + + Index: raw2proc/trunk/raw2proc/test_proc_codar_totals.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/test_proc_codar_totals.py (revision 451) @@ -1,0 +1,153 @@ +#!/usr/bin/env python +# Last modified: Time-stamp: <2010-12-09 16:16:12 haines> +""" +parse datestr from cr1000 files to create monthly files + +input file +/seacoos/data/nccoos/level0/crow/crow_csi_loggernet_yyyymmdd-yyyymmdd.dat + +Output form +/seacoos/data/nccoos/level0/crow/yyyy_mm/wq/csi_wq_yyyy_mm.dat +/seacoos/data/nccoos/level0/crow/yyyy_mm/flow/csi_flow_yyyy_mm.dat + +load data file +parse lines for time YYYY, jjj, HHMM +what year and month? + +create YYYY_MM directory and output file if does not exist. +write line to YYYY_MM/csi_loggernet_yyyy_mm.dat output file + +""" + +REAL_RE_STR = '\\s*(-?\\d(\\.\\d+|)[Ee][+\\-]\\d\\d?|-?(\\d+\\.\\d*|\\d*\\.\\d+)|-?\\d+)\\s*' + +import sys +import os +import re +from procutil import * + +def parser_codar(fn, lines): + """ + parse and assign data to variables from CODAR Totals LLUV format + + Notes + ----- + 1. Requires grid definition + + """ + + import numpy + from datetime import datetime + from time import strptime + from StringIO import StringIO + from matplotlib.mlab import griddata + + # get sample datetime from filename + # fn = sensor_info['fn'] + sample_dt_start = filt_datetime(fn) + + # read header that match '%(k): (v)\n' pairs on each line + m = re.findall(r'^(%.*):\s*(.*)$', ''.join(lines), re.MULTILINE) + for k,v in m: + if k == '%TimeStamp': + sample_dt = scanf_datetime(v, fmt='%Y %m %d %H %M %S') + + elif k == '%TableType': + ftype = v + elif k == '%TableColumns': + ncol = int(v) + elif k == '%TableRows': + nrow = int(v) + + # read data from string of lines but make it behave like a file object with StringIO + s = StringIO(''.join(lines)) + s.seek(0) # ensures start posn of file + d = numpy.loadtxt(s, comments='%') + # lat, lon, u, v = numpy.loadtxt(s, usecols=(0,1,2,3), comments='%', unpack=True) + if 'TOT4' in ftype: + lon = d[:,0] + lat = d[:,1] + wu = d[:,2] + wv = d[:,3] + gridflag = d[:,4] + wu_std_qual = d[:,5] + wv_std_qual = d[:,6] + cov_qual = d[:,7] + x_dist = d[:,8] + y_dist = d[:,9] + rang = d[:,10] + bearing = d[:,11] + vel_mag = d[:,12] + vel_dir = d[:,13] + s1 = d[:,14] + s2 = d[:,15] + s3 = d[:,16] + s4 = d[:,17] + s5 = d[:,18] + s6 = d[:,19] + + # define the lat/lon grid based on 6km resolution + # For Outer Banks north and east of Cape Hatteras, bounding box is defined by: + minlat, maxlat = (34.5, 38) + minlon, maxlon = (-76, -73.) + midlat = minlat + 0.5*(maxlat-minlat) + # ~111 km = 1 deg latitude + nlat = numpy.round((maxlat-minlat) *111/6) + nlon = numpy.round((maxlon-minlon) * math.cos(midlat*math.pi/180)*111/6) + yi = numpy.linspace(minlat, maxlat, nlat) + xi = numpy.linspace(minlon, maxlon, nlon) + uim = griddata(lon, lat, wu, xi, yi) + vim = griddata(lon, lat, wv, xi, yi) + # returned masked array as an ndarray with masked values filled with fill_value + ui = uim.filled(fill_value=numpy.nan) + vi = vim.filled(fill_value=numpy.nan) + # --------------------------------------------------------------- + data = { + 'dt' : numpy.array(numpy.ones((1,), dtype=object)*numpy.nan), + 'time' : numpy.array(numpy.ones((1,), dtype=long)*numpy.nan), + 'lon' : numpy.array(numpy.ones((1,nlon), dtype=float)*numpy.nan), + 'lat' : numpy.array(numpy.ones((1,nlat), dtype=float)*numpy.nan), + 'u' : numpy.array(numpy.ones((1,nlon,nlat), dtype=float)*numpy.nan), + 'v' : numpy.array(numpy.ones((1,nlon,nlat), dtype=float)*numpy.nan), + } + # --------------------------------------------------------------- + i = 0 + data['dt'][i] = sample_dt # + data['time'][i] = dt2es(sample_dt) # + data['lon'][i] = xi # new longitude grid + data['lat'][i] = yi # new latitude grid + data['u'][i] = ui.T # u-component of water velocity (cm/s) + data['v'][i] = vi.T # v-component of water velocity + + return data + + +def load_data(inFile): + lines=None + if os.path.exists(inFile): + f = open(inFile, 'r') + lines = f.readlines() + f.close() + if len(lines)<=0: + print 'Empty file: '+ inFile + else: + print 'File does not exist: '+ inFile + return lines + +from raw2proc import * + +def test1(fn): + lines = load_data(fn) + return parser_codar(fn, lines) + + +if __name__ == '__main__': + fn = '/seacoos/data/nc-coos/level0/ouba/hfr_totals/2010_07/TOTL_OUBA_2010_07_14_0000.tuv' + + # + # fn = sys.argv[1] + try: + test1(fn) + except: + pass +