Index: raw2proc/trunk/raw2proc/morgan_config_20080701.py =================================================================== --- raw2proc/trunk/raw2proc/morgan_config_20080701.py (revision 211) +++ raw2proc/trunk/raw2proc/morgan_config_20080701.py (revision 218) @@ -2,8 +2,9 @@ 'id' : 'morgan', 'location' : 'Morgan Bay, New River, NC', - 'lat' : 34.7037, # degrees true (-) south, (+) north - 'lon' : -77.4022, # degrees true (-) west, (+) east - 'mvar' : -9.42, # degrees (-) west, (+) east - 'water_depth': 4.0, # nominal depth in meters (should be MSL) + 'lat' : 34.7037, # degrees true (-) south, (+) north + 'lon' : -77.4022, # degrees true (-) west, (+) east + 'mvar' : -9.42, # degrees (-) west, (+) east + 'mean_water_depth': -4.0, # nominal depth in meters (should be MSL) + 'mean_water_depth_time_period': 'Not determined', 'institution' : 'nccoos', # @@ -19,10 +20,10 @@ 'raw_file_glob' : '*.dat', 'proc_dir' : '/seacoos/data/nccoos/level1/morgan/avp/', - 'process_module' : 'proc_avp_ysi_6600_v2', + 'process_module' : 'proc_avp_ysi_6600_v2_CDL2', 'utc_offset' : 5., # hours offset to Eastern Standard 'bin_size' : 0.1, # meters - 'nbins' : 40, # for now, water_depth (MSL) divided by bin_size - 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', - 'latest_vars' : ('time','lat','lon','z','wtemp','salin'), + 'nbins' : 150, # max number of samples in profile + # 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + # 'latest_vars' : ('time','lat','lon','z','wtemp','salin'), }, 'met' : { 'id' : 'met', Index: raw2proc/trunk/raw2proc/proc_avp_ascii_met.py =================================================================== --- raw2proc/trunk/raw2proc/proc_avp_ascii_met.py (revision 211) +++ raw2proc/trunk/raw2proc/proc_avp_ascii_met.py (revision 218) @@ -1,4 +1,4 @@ #!/usr/bin/env python -# Last modified: Time-stamp: <2008-10-01 12:46:26 haines> +# Last modified: Time-stamp: <2008-11-13 11:55:34 haines> """ how to parse data, and assert what data and info goes into @@ -244,8 +244,8 @@ # dimension names use tuple so order of initialization is maintained dim_inits = ( - ('ntime', NC.UNLIMITED), - ('nlat', 1), - ('nlon', 1), - ('nz', 1) + ('time', NC.UNLIMITED), + ('lat', 1), + ('lon', 1), + ('z', 1) ) @@ -256,15 +256,15 @@ var_inits = ( # coordinate variables - ('time', NC.INT, ('ntime',)), - ('lat', NC.FLOAT, ('nlat',)), - ('lon', NC.FLOAT, ('nlon',)), - ('z', NC.FLOAT, ('nz',)), + ('time', NC.INT, ('time',)), + ('lat', NC.FLOAT, ('lat',)), + ('lon', NC.FLOAT, ('lon',)), + ('z', NC.FLOAT, ('z',)), # data variables - ('wspd', NC.FLOAT, ('ntime',)), - ('wdir', NC.FLOAT, ('ntime',)), - ('cdir', NC.FLOAT, ('ntime',)), - ('u', NC.FLOAT, ('ntime',)), - ('v', NC.FLOAT, ('ntime',)), - ('nwnd', NC.FLOAT, ('ntime',)), + ('wspd', NC.FLOAT, ('time',)), + ('wdir', NC.FLOAT, ('time',)), + ('cdir', NC.FLOAT, ('time',)), + ('u', NC.FLOAT, ('time',)), + ('v', NC.FLOAT, ('time',)), + ('nwnd', NC.FLOAT, ('time',)), ) Index: raw2proc/trunk/raw2proc/proc_avp_ysi_6600_v2.py =================================================================== --- raw2proc/trunk/raw2proc/proc_avp_ysi_6600_v2.py (revision 211) +++ (revision ) @@ -1,420 +1,0 @@ -#!/usr/bin/env python -# Last modified: Time-stamp: <2008-10-01 12:46:41 haines> -""" -how to parse data, and assert what data and info goes into -creating and updating monthly netcdf files - -parse data from YSI 6600 V2-2 on an automated veritical profiler (avp) - -parser : sample date and time, water_depth for each profile - water temperature, conductivity, pressure (depth), salinity, pH, dissolved oxygen, turbidity, and chlorophyll - raw data averaged to 10 cm bins - -creator : lat, lon, z, time, water_depth, water_temp, cond, salin, ph, turb, chl, do -updator : time, water_depth, water_temp, cond, salin, ph, turb, chl, do - - -Examples --------- - ->> (parse, create, update) = load_processors('proc_avp_ysi_6600_v2') -or ->> si = get_config(cn+'.sensor_info') ->> (parse, create, update) = load_processors(si['adcp']['proc_module']) - ->> lines = load_data(filename) ->> data = parse(platform_info, sensor_info, lines) ->> create(platform_info, sensor_info, data) or ->> update(platform_info, sensor_info, data) - -""" - - -from raw2proc import * -from procutil import * -from ncutil import * - -now_dt = datetime.utcnow() -now_dt.replace(microsecond=0) - -def parser(platform_info, sensor_info, lines): - """ - parse Automated Vertical Profile Station (AVP) Water Quality Data - - month, day, year, hour, min, sec, temp (deg. C), conductivity - (mS/cm), salinity (ppt or PSU), depth (meters), pH, turbidity (NTU), - chlorophyll (micrograms per liter), DO (micrograms per liter) - - Notes - ----- - 1. Column Format - - temp, cond, salin, depth, pH, turb, chl, DO - (C), (mS/cm), (ppt), (m), pH, (NTU), (ug/l), (ug/l) - - Profile Time: 00:30:00 - Profile Date: 08/18/2008 - Profile Depth: 255.0 cm - Profile Location: Stones Bay Serial No: 00016B79, ID: AVP1_SERDP - 08/18/08 00:30:06 26.94 41.87 26.81 0.134 8.00 3.4 4.5 6.60 - 08/18/08 00:30:07 26.94 41.87 26.81 0.143 8.00 3.4 4.8 6.59 - 08/18/08 00:30:08 26.94 41.87 26.81 0.160 8.00 3.4 4.8 6.62 - 08/18/08 00:30:09 26.94 41.87 26.81 0.183 8.00 3.4 4.8 6.66 - - 2. While each parameter is measured uniquely with time and depth such that, temp(t) and z(t) - match up with time, we want to grid depth every 1 cm and make each param as temp(t,z). - - Tony Whipple at IMS says 'The AVPs sample at one second intervals. - Between the waves and the instrument descending from a spool of - line with variable radius it works out to about 3-5 cm between - observations on average. When I process the data to make the - images, I bin the data every 10 cm and take the average of however - many observations fell within that bin.' - - Do we interpolate or average samples in bin? - - """ - import numpy - from datetime import datetime - from time import strptime - - # get sample datetime from filename - fn = sensor_info['fn'] - sample_dt_start = filt_datetime(fn)[0] - - # how many profiles in one file, count number of "Profile Time:" in lines - nprof = 0 - for line in lines: - m=re.search("Profile Time:", line) - if m: - nprof=nprof+1 - - # remove first occurrence of blank line if within first 10-40 lines - # and put it on the end to signal end of profile after last profile - for i in range(len(lines[0:40])): - if re.search("^ \r\n", lines[i]): - # print str(i) + " " + lines[i] + " " + lines[i+1] - blank_line = lines.pop(i) - lines.append(blank_line) - - bin_size = sensor_info['bin_size'] # Bin Size (meters) - nominal_depth = platform_info['water_depth'] # Mean sea level at station (meters) or nominal water depth - z = numpy.arange(0, -1*nominal_depth, -1*bin_size, dtype=float) - - N = nprof - nbins = len(z) - - if nbins != sensor_info['nbins']: - print 'Number of bins computed from water_depth and bin_size ('+ \ - str(nbins)+') does not match config number ('+ \ - str(sensor_info['nbins'])+')' - - data = { - 'dt' : numpy.array(numpy.ones((N,), dtype=object)*numpy.nan), - 'time' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan), - 'z' : numpy.array(numpy.ones((nbins,), dtype=float)*numpy.nan), - # - 'wd' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan), - 'wtemp' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), - 'cond' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), - 'salin' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), - 'turb' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), - 'ph' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), - 'chl' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), - 'do' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), - } - - # current profile count - i = 0 - - for line in lines: - ysi = [] - # split line and parse float and integers - sw = re.split('[\s/\:]*', line) - for s in sw: - m = re.search(REAL_RE_STR, s) - if m: - ysi.append(float(m.groups()[0])) - - if re.search("Profile Time:", line): - HH=ysi[0] - MM=ysi[1] - SS=ysi[2] - elif re.search("Profile Date:", line): - mm=ysi[0] - dd=ysi[1] - yyyy=ysi[2] - elif re.search("Profile Depth:", line): - wd = ysi[0]/100. # cm to meters - sample_str = '%02d-%02d-%4d %02d:%02d:%02d' % (mm,dd,yyyy,HH,MM,SS) - if sensor_info['utc_offset']: - sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%Y %H:%M:%S') + \ - timedelta(hours=sensor_info['utc_offset']) - else: - sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%Y %H:%M:%S') - - # initialize for new profile at zero for averaging samples within each bin - wtemp = numpy.zeros(nbins) - cond = numpy.zeros(nbins) - salin = numpy.zeros(nbins) - turb = numpy.zeros(nbins) - ph = numpy.zeros(nbins) - chl = numpy.zeros(nbins) - do = numpy.zeros(nbins) - Ns = numpy.zeros(nbins) # count samples per bin for averaging - elif len(ysi)==14: - # get sample datetime from data - # sample_str = '%02d-%02d-%2d %02d:%02d:%02d' % tuple(ysi[0:6]) - # if sensor_info['utc_offset']: - # sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%Y %H:%M:%S') + \ - # timedelta(hours=sensor_info['utc_offset']) - # else: - # sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%y %H:%M:%S') - - depth = -1*ysi[9] # depth (m, positive up) - ibin = ((z)<=depth)*(depth<(z+bin_size)) - - Ns[ibin] = Ns[ibin]+1 - wtemp[ibin] = wtemp[ibin]+ysi[6] # water temperature (C) - cond[ibin] = cond[ibin]+ysi[7] # conductivity (mS/cm) - salin[ibin] = salin[ibin]+ysi[8] # salinity (ppt or PSU??) - # - ph[ibin] = ph[ibin]+ysi[10] # ph - turb[ibin] = turb[ibin]+ysi[11] # turbidity (NTU) - chl[ibin] = chl[ibin]+ysi[12] # chlorophyll (ug/l) - do[ibin] = do[ibin]+ysi[13] # dissolved oxygen (mg/l) - - elif (len(ysi)==0): # each profile separated by empty line - # average summations by sample count per bin - # where count is zero make it NaN so average is not divide by zero - Ns[Ns==0]=numpy.nan*Ns[Ns==0] - - data['dt'][i] = sample_dt # sample datetime - data['time'][i] = dt2es(sample_dt) # sample time in epoch seconds - data['wd'][i] = wd - data['z'] = z - # divide by counts - data['wtemp'][i] = wtemp/Ns - data['cond'][i] = cond/Ns - data['salin'][i] = salin/Ns - data['turb'][i] = turb/Ns - data['ph'][i] = ph/Ns - data['chl'][i] = chl/Ns - data['do'][i] = do/Ns - - i=i+1 - - # if-elif - # for line - - return data - - -def creator(platform_info, sensor_info, data): - # - # - title_str = sensor_info['description']+' at '+ platform_info['location'] - global_atts = { - 'title' : title_str, - 'institution' : 'University of North Carolina at Chapel Hill (UNC-CH)', - 'institution_url' : 'http://nccoos.unc.edu', - 'institution_dods_url' : 'http://nccoos.unc.edu', - 'metadata_url' : 'http://nccoos.unc.edu', - 'references' : 'http://nccoos.unc.edu', - 'contact' : 'Sara Haines (haines@email.unc.edu)', - # - 'source' : 'fixed-automated-profiler observation', - 'history' : 'raw2proc using ' + sensor_info['process_module'], - 'comment' : 'File created using pycdf'+pycdfVersion()+' and numpy '+pycdfArrayPkg(), - # conventions - 'Conventions' : 'CF-1.0; SEACOOS-CDL-v2.0', - # SEACOOS CDL codes - 'format_category_code' : 'fixed-profiler', - 'institution_code' : platform_info['institution'], - 'platform_code' : platform_info['id'], - 'package_code' : sensor_info['id'], - # institution specific - 'project' : 'North Carolina Coastal Ocean Observing System (NCCOOS)', - 'project_url' : 'http://nccoos.unc.edu', - # timeframe of data contained in file yyyy-mm-dd HH:MM:SS - # first date in monthly file - 'start_date' : data['dt'][0].strftime("%Y-%m-%d %H:%M:%S"), - # last date in monthly file - 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), - 'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), - # - 'creation_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), - 'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), - 'process_level' : 'level1', - # - # must type match to data (e.g. fillvalue is real if data is real) - '_FillValue' : -99999., - } - - var_atts = { - # coordinate variables - 'time' : {'short_name': 'time', - 'long_name': 'Time', - 'standard_name': 'time', - 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC - 'axis': 'T', - }, - 'lat' : {'short_name': 'lat', - 'long_name': 'Latitude', - 'standard_name': 'latitude', - 'reference':'geographic coordinates', - 'units': 'degrees_north', - 'valid_range':(-90.,90.), - 'axis': 'Y', - }, - 'lon' : {'short_name': 'lon', - 'long_name': 'Longitude', - 'standard_name': 'longitude', - 'reference':'geographic coordinates', - 'units': 'degrees_east', - 'valid_range':(-180.,180.), - 'axis': 'Y', - }, - 'z' : {'short_name': 'z', - 'long_name': 'Height', - 'standard_name': 'height', - 'reference':'zero at sea-surface', - 'positive' : 'up', - 'units': 'm', - 'axis': 'Z', - }, - # data variables - 'wd': {'short_name': 'wd', - 'long_name': 'Water Depth', - 'standard_name': 'water_depth', - 'units': 'm', - }, - 'wtemp': {'short_name': 'wtemp', - 'long_name': 'Water Temperature', - 'standard_name': 'water_temperature', - 'units': 'degrees Celsius', - }, - 'cond': {'short_name': 'cond', - 'long_name': 'Conductivity', - 'standard_name': 'conductivity', - 'units': 'mS cm-1', - }, - 'salin': {'short_name': 'salin', - 'long_name': 'Salinity', - 'standard_name': 'salinity', - 'units': 'PSU', - }, - 'turb': {'short_name': 'turb', - 'long_name': 'Turbidity', - 'standard_name': 'turbidity', - 'units': 'NTU', - }, - 'ph': {'short_name': 'ph', - 'long_name': 'pH', - 'standard_name': 'ph', - 'units': '', - }, - 'chl': {'short_name': 'chl', - 'long_name': 'Chlorophyll', - 'standard_name': 'chlorophyll', - 'units': 'ug l-1', - }, - 'do': {'short_name': 'do', - 'long_name': 'Dissolved Oxygen', - 'standard_name': 'dissolved_oxygen', - 'units': 'mg l-1', - }, - } - - # dimension names use tuple so order of initialization is maintained - dim_inits = ( - ('ntime', NC.UNLIMITED), - ('nlat', 1), - ('nlon', 1), - ('nz', sensor_info['nbins']) - ) - - # using tuple of tuples so order of initialization is maintained - # using dict for attributes order of init not important - # use dimension names not values - # (varName, varType, (dimName1, [dimName2], ...)) - var_inits = ( - # coordinate variables - ('time', NC.INT, ('ntime',)), - ('lat', NC.FLOAT, ('nlat',)), - ('lon', NC.FLOAT, ('nlon',)), - ('z', NC.FLOAT, ('nz',)), - # data variables - ('wd', NC.FLOAT, ('ntime',)), - ('wtemp', NC.FLOAT, ('ntime', 'nz')), - ('cond', NC.FLOAT, ('ntime', 'nz')), - ('salin', NC.FLOAT, ('ntime', 'nz')), - ('turb', NC.FLOAT, ('ntime', 'nz')), - ('ph', NC.FLOAT, ('ntime', 'nz')), - ('chl', NC.FLOAT, ('ntime', 'nz')), - ('do', NC.FLOAT, ('ntime', 'nz')), - ) - - # subset data only to month being processed (see raw2proc.process()) - i = data['in'] - - # var data - var_data = ( - ('lat', platform_info['lat']), - ('lon', platform_info['lon']), - ('z', data['z']), - # - ('time', data['time'][i]), - ('wd', data['wd'][i]), - ('wtemp', data['wtemp'][i]), - ('cond', data['cond'][i]), - ('salin', data['salin'][i]), - ('turb', data['turb'][i]), - ('ph', data['ph'][i]), - ('chl', data['chl'][i]), - ('do', data['do'][i]), - ) - - return (global_atts, var_atts, dim_inits, var_inits, var_data) - -def updater(platform_info, sensor_info, data): - # - global_atts = { - # update times of data contained in file (yyyy-mm-dd HH:MM:SS) - # last date in monthly file - 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), - 'release_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), - # - 'modification_date' : now_dt.strftime("%Y-%m-%d %H:%M:%S"), - } - - # data variables - # update any variable attributes like range, min, max - var_atts = {} - # var_atts = { - # 'wtemp': {'max': max(data.u), - # 'min': min(data.v), - # }, - # 'cond': {'max': max(data.u), - # 'min': min(data.v), - # }, - # } - - # subset data only to month being processed (see raw2proc.process()) - i = data['in'] - - # data - var_data = ( - ('time', data['time'][i]), - ('wd', data['wd'][i]), - ('wtemp', data['wtemp'][i]), - ('cond', data['cond'][i]), - ('salin', data['salin'][i]), - ('turb', data['turb'][i]), - ('ph', data['ph'][i]), - ('chl', data['chl'][i]), - ('do', data['do'][i]), - ) - - return (global_atts, var_atts, var_data) -# Index: raw2proc/trunk/raw2proc/proc_avp_ysi_6600_v2_CDL2.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/proc_avp_ysi_6600_v2_CDL2.py (revision 218) @@ -1,0 +1,501 @@ +#!/usr/bin/env python +# Last modified: Time-stamp: <2009-01-07 17:54:32 haines> +""" +how to parse data, and assert what data and info goes into +creating and updating monthly netcdf files + +parse data from YSI 6600 V2-2 on an automated veritical profiler (avp) + +parser : date and time, water_depth for each profile + + sample time, sample depth, as cast measures water + temperature, conductivity, salinity, pH, dissolved oxygen, + turbidity, and chlorophyll + + +creator : lat, lon, z, stime, (time, water_depth), water_temp, cond, + salin, ph, turb, chl, do + +updator : z, stime, (time, water_depth), water_temp, cond, salin, ph, + turb, chl, do + +using fixed profiler CDL but modified to have raw data for each cast +along each column + + +Examples +-------- + +>> (parse, create, update) = load_processors('proc_avp_ysi_6600_v2') +or +>> si = get_config(cn+'.sensor_info') +>> (parse, create, update) = load_processors(si['adcp']['proc_module']) + +>> lines = load_data(filename) +>> data = parse(platform_info, sensor_info, lines) +>> create(platform_info, sensor_info, data) or +>> update(platform_info, sensor_info, data) + +""" + + +from raw2proc import * +from procutil import * +from ncutil import * + +now_dt = datetime.utcnow() +now_dt.replace(microsecond=0) + +def parser(platform_info, sensor_info, lines): + """ + parse Automated Vertical Profile Station (AVP) Water Quality Data + + month, day, year, hour, min, sec, temp (deg. C), conductivity + (mS/cm), salinity (ppt or PSU), depth (meters), pH, turbidity (NTU), + chlorophyll (micrograms per liter), DO (micrograms per liter) + + Notes + ----- + 1. Column Format + + temp, cond, salin, depth, pH, turb, chl, DO + (C), (mS/cm), (ppt), (m), pH, (NTU), (ug/l), (ug/l) + + Profile Time: 00:30:00 + Profile Date: 08/18/2008 + Profile Depth: 255.0 cm + Profile Location: Stones Bay Serial No: 00016B79, ID: AVP1_SERDP + 08/18/08 00:30:06 26.94 41.87 26.81 0.134 8.00 3.4 4.5 6.60 + 08/18/08 00:30:07 26.94 41.87 26.81 0.143 8.00 3.4 4.8 6.59 + 08/18/08 00:30:08 26.94 41.87 26.81 0.160 8.00 3.4 4.8 6.62 + 08/18/08 00:30:09 26.94 41.87 26.81 0.183 8.00 3.4 4.8 6.66 + + 2. While each parameter is measured uniquely with time and depth such that, temp(t) and z(t) + match up with time, we want to grid depth every 1 cm and make each param as temp(t,z). + + Tony Whipple at IMS says 'The AVPs sample at one second intervals. + Between the waves and the instrument descending from a spool of + line with variable radius it works out to about 3-5 cm between + observations on average. When I process the data to make the + images, I bin the data every 10 cm and take the average of however + many observations fell within that bin.' + + Do we interpolate or average samples in bin? + + """ + import numpy + from datetime import datetime + from time import strptime + + # get sample datetime from filename + fn = sensor_info['fn'] + sample_dt_start = filt_datetime(fn)[0] + + # how many profiles in one file, count number of "Profile Time:" in lines + nprof = 0 + for line in lines: + m=re.search("Profile Time:", line) + if m: + nprof=nprof+1 + + # remove first occurrence of blank line if within first 40 lines + for i in range(len(lines[0:40])): + if re.search("^ \r\n", lines[i]): + # print str(i) + " " + lines[i] + " " + lines[i+1] + blank_line = lines.pop(i) + # lines.append(blank_line) + + # ensure signal end of profile after last profile by appending a blank line to data file + lines.append(' \r\n') + + N = nprof + nbins = sensor_info['nbins'] + + data = { + 'dt' : numpy.array(numpy.ones((N,), dtype=object)*numpy.nan), + 'time' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan), + 'z' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + # + 'wd' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan), + 'wl' : numpy.array(numpy.ones((N,), dtype=long)*numpy.nan), + 'ysi_sn' : numpy.array(['' for i in range(N)] , dtype='|S20'), + 'ysi_id' : numpy.array(['' for i in range(N)] , dtype='|S20'), + # + 'stime' : numpy.array(numpy.ones((N,nbins), dtype=long)*numpy.nan), + 'wtemp' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + 'cond' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + 'salin' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + 'turb' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + 'ph' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + 'chl' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + 'do' : numpy.array(numpy.ones((N,nbins), dtype=float)*numpy.nan), + } + + # current profile count + i = 0 + have_date = have_time = have_wd = have_location = have_head = False + + for line in lines: + # if line has weird ascii chars -- skip it and iterate to next line + if re.search(r"[\x1a]", line): + print 'skipping bad data line ... ' + str(line) + continue + + ysi = [] + # split line and parse float and integers + sw = re.split('[\s/\:]*', line) + for s in sw: + m = re.search(REAL_RE_STR, s) + if m: + ysi.append(float(m.groups()[0])) + + if re.search("Profile Time:", line): + have_time = True + HH=ysi[0] + MM=ysi[1] + SS=ysi[2] + elif re.search("Profile Date:", line): + have_date = True + mm=ysi[0] + dd=ysi[1] + yyyy=ysi[2] + elif re.search("Profile Depth:", line): + have_wd = True + wd = ysi[0]/100. # cm to meters + profile_str = '%02d-%02d-%4d %02d:%02d:%02d' % (mm,dd,yyyy,HH,MM,SS) + if sensor_info['utc_offset']: + profile_dt = scanf_datetime(profile_str, fmt='%m-%d-%Y %H:%M:%S') + \ + timedelta(hours=sensor_info['utc_offset']) + else: + profile_dt = scanf_datetime(profile_str, fmt='%m-%d-%Y %H:%M:%S') + elif re.search("Profile Location:", line): + have_location = True + # Profile Location: Stones Bay Serial No: 00016B79, ID: AVP1_SERDP + sw = re.findall(r'\w+:\s(\w+)*', line) + ysi_sn = sw[1] + ysi_id = sw[2] + # initialize for new profile at zero for averaging samples within each bin + wtemp = numpy.array(numpy.ones(nbins,), dtype=float)*numpy.nan + depth =numpy.array(numpy.ones(nbins,), dtype=float)*numpy.nan + cond = numpy.array(numpy.ones(nbins,), dtype=float)*numpy.nan + salin = numpy.array(numpy.ones(nbins,), dtype=float)*numpy.nan + turb = numpy.array(numpy.ones(nbins,), dtype=float)*numpy.nan + ph = numpy.array(numpy.ones(nbins,), dtype=float)*numpy.nan + chl = numpy.array(numpy.ones(nbins,), dtype=float)*numpy.nan + do = numpy.array(numpy.ones(nbins,), dtype=float)*numpy.nan + stime = numpy.array(numpy.ones(nbins,), dtype=long)*numpy.nan + # keep track of number of samples in one profile so not to exceed nbins + j = 0 + # have all the headers stuff + head = numpy.array([have_date, have_time, have_wd, have_location]) + have_head = head.all() + + elif (len(ysi)==14 and have_head): + if j>=nbins: + print 'Sample number (' + str(j) + \ + ') in profile exceeds maximum value ('+ \ + str(nbins) + ') in config' + + # get sample datetime from data + sample_str = '%02d-%02d-%02d %02d:%02d:%02d' % tuple(ysi[0:6]) + if sensor_info['utc_offset']: + sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%y %H:%M:%S') + \ + timedelta(hours=sensor_info['utc_offset']) + else: + sample_dt = scanf_datetime(sample_str, fmt='%m-%d-%y %H:%M:%S') + + if j +# Last modified: Time-stamp: <2008-12-17 16:49:11 haines> """Utilities to help data processing @@ -289,9 +289,42 @@ # fn_glob = 'bogue_dspec_plot*' +def addnan(dt, data): + # dt to be only 1-dimension and data to be 1- or 2-dimensional + + from matplotlib.dates import date2num, num2date + + # print dt.shape + # print data.shape + + dn = date2num(dt) + delta = numpy.diff(dn) + sample_interval = numpy.median(delta) + maxdelta = 1.5*sample_interval + # print maxdelta + igap = (delta > maxdelta).nonzero()[0] + ngap = len(igap) + if not ngap: + return (dt, data) + else: + # convert sample interval to dt object + sample_interval = timedelta(0.5*sample_interval) + # for each gap in data create NaN + data_insert = [numpy.nan for gap in igap] + # for each gap in time create datetime value + dt_insert = [dt[gap]+sample_interval for gap in igap] + # insert new sample times at indices of the gaps + new_dt = numpy.insert(numpy.array(dt), igap+1, dt_insert) + # insert NaN data at the indices that match the above times + new_data = numpy.insert(numpy.array(data), igap+1, data_insert, axis=0) + return (new_dt, new_data) + # unit conversions def meters2feet(meters): """Convert meters to feet: = *3.28084 """ return meters*3.28084 - + +def feet2meters(feet): + """Convert feet to meters: = *0.3048 """ + return feet*0.3048 def millibar2inches_Hg(millibar): @@ -299,14 +332,15 @@ return millibar*0.0295301 - def celsius2fahrenheit(celsius): """Convert deg Celsius to deg Fahrenheit: = ((1.8*)+32) """ return (1.8*celsius)+32 - def millimeters2inches(millimeters): """ Convert millimeter to inches: = *0.0393700787) """ return millimeters*0.0393700787 +def inches2millimeters(inches): + """ Convert = *25.4 """ + return inches*25.4 def meters_sec2knots(meters_sec): @@ -314,15 +348,12 @@ return meters_sec*1.94384449 - def wind_vector2u(wind_speed, wind_from_direction): """ Convert wind vector to U (east) component: = *sine(*pi/180) """ return wind_speed*math.sin(wind_from_direction*math.pi/180) - def wind_vector2v(wind_speed, wind_from_direction): """ Convert wind vector to V (north) component: = *cosine(*pi/180) """ return wind_speed*math.cos(wind_from_direction*math.pi/180) - def proc2latest(pi, si, yyyy_mm): """Select specific variables and times from current monthly netCDF Index: raw2proc/trunk/raw2proc/raw2proc.py =================================================================== --- raw2proc/trunk/raw2proc/raw2proc.py (revision 213) +++ raw2proc/trunk/raw2proc/raw2proc.py (revision 218) @@ -1,4 +1,4 @@ #!/usr/bin/env python -# Last modified: Time-stamp: <2008-10-23 11:05:43 haines> +# Last modified: Time-stamp: <2009-01-07 17:12:01 haines> """Process raw data to monthly netCDF data files @@ -402,5 +402,5 @@ for index, val in enumerate(data['dt']): - if val>si['proc_start_dt'] and val<=si['proc_end_dt']: + if val>=si['proc_start_dt'] and val<=si['proc_end_dt']: data['in'][index] = True Index: raw2proc/trunk/raw2proc/stones_config_20080701.py =================================================================== --- raw2proc/trunk/raw2proc/stones_config_20080701.py (revision 211) +++ raw2proc/trunk/raw2proc/stones_config_20080701.py (revision 218) @@ -2,8 +2,9 @@ 'id' : 'stones', 'location' : 'Stones Bay, New River, NC', - 'lat' : 34.5962, # degrees true (-) south, (+) north - 'lon' : -77.4120, # degrees true (-) west, (+) east - 'mvar' : -9.38, # degrees (-) west, (+) east - 'water_depth': 4.0, # nominal depth in meters (should be MSL) + 'lat' : 34.5962, # degrees true (-) south, (+) north + 'lon' : -77.4120, # degrees true (-) west, (+) east + 'mvar' : -9.38, # degrees (-) west, (+) east + 'mean_water_depth': -4.0, + 'mean_water_depth_time_period': 'Not determined', 'institution' : 'nccoos', # @@ -19,10 +20,10 @@ 'raw_file_glob' : '*.dat', 'proc_dir' : '/seacoos/data/nccoos/level1/stones/avp', - 'process_module' : 'proc_avp_ysi_6600_v2', + 'process_module' : 'proc_avp_ysi_6600_v2_CDL2', 'utc_offset' : 5., # hours offset to Eastern Standard 'bin_size' : 0.1, # meters - 'nbins' : 40, # for now, water_depth (MSL) divided by bin_size - 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', - 'latest_vars' : ('time','lat','lon','z','wtemp','salin'), + 'nbins' : 150, # max number of samples in profile + # 'latest_dir' : '/seacoos/data/nccoos/latest_v2.0', + # 'latest_vars' : ('time','lat','lon','z','wtemp','salin'), }, 'met' : { 'id' : 'met',