Index: sodar/branches/raw2proc-dev/proc_remtech_rawdata_pa0.py =================================================================== --- sodar/branches/raw2proc-dev/proc_remtech_rawdata_pa0.py (revision 189) +++ sodar/branches/raw2proc-dev/proc_remtech_rawdata_pa0.py (revision 190) @@ -13,43 +13,11 @@ import math import numpy as n -from datetime import timedelta -from procutil import scanf_datetime, dt2es +import pycdf +import datetime +import procutil INVALID = '-9999' -BLOCKNUMBER = 'BL#' -MONTH = 'MONTH' -DAY = 'DAY' -YEAR = 'YEAR' -HOUR = 'HOUR' -MINUTE = 'MIN' -VALIDATIONS1 = 'VAL1' -VALIDATIONS2 = 'VAL2' -VALIDATIONS3 = 'VAL3' -VALIDATIONS4 = 'VAL4' -PROBABILITY1 = 'SPU1' -PROBABILITY2 = 'SPU2' -PROBABILITY3 = 'SPU3' -PROBABILITY4 = 'SPU4' -AMBIENTNOISE1 = 'NOIS1' -AMBIENTNOISE2 = 'NOIS2' -AMBIENTNOISE3 = 'NOIS3' -AMBIENTNOISE4 = 'NOIS4' -CLUTTER = 'FEMAX' -SOFTWARE = 'SOFTW' -FREQUENCIES11 = 'FE11' -FREQUENCIES12 = 'FE12' -FREQUENCIES21 = 'FE21' -FREQUENCIES22 = 'FE22' -SIGNALNOISE1 = 'SNR1' -SIGNALNOISE2 = 'SNR2' -SIGNALNOISE3 = 'SNR3' -SIGNALNOISE4 = 'SNR4' -REFERENCE = 'CHECK' -JAM = 'JAM' -ALTITUDE = 'ALT' -ECHO = 'CT' -RADIAL = 'SPEED' -THETA = 'DIR' -VERTICAL = 'W' + +nowDt = datetime.datetime.utcnow().replace(microseconds=0) def parser(platform_info, sensor_info, lines): @@ -113,56 +81,56 @@ sampleIndex = rawDataObject.index(sample) - data['block'][sampleIndex] = int(sample[BLOCKNUM]) + data['block'][sampleIndex] = int(sample['BL#']) - dt = {'month' : int(sample[MONTH]), - 'day' : int(sample[DAY]), - 'year' : int(sample[YEAR]), - 'hour' : int(sample[HOUR]), - 'min' : int(sample[MINUTE]), + dt = {'month' : int(sample['MONTH']), + 'day' : int(sample['DAY']), + 'year' : int(sample['YEAR']), + 'hour' : int(sample['HOUR']), + 'min' : int(sample['MIN']), } dt = '%(month)02d-%(day)02d-%(year)04d %(hour)02d:%02d(min)' % dt - dt = scanf_datetime(dt, fmt='%m-%d-%Y %H:%M') + dt = procutil.scanf_datetime(dt, fmt='%m-%d-%Y %H:%M') if sensor_info['utc_offset']: - dt = dt + timedelta(hours=sensor_info['utc_offset']) + dt = dt + datetime.timedelta(hours=sensor_info['utc_offset']) data['dt'][sampleIndex] = dt - data['es'][sampleIndex] = dt2es(dt) - - data['val1'][sampleIndex] = int(sample[VALIDATIONS1]) - data['val2'][sampleIndex] = int(sample[VALIDATIONS2]) - data['val3'][sampleIndex] = int(sample[VALIDATIONS3]) - data['val4'][sampleIndex] = int(sample[VALIDATIONS4]) - - data['spu1'][sampleIndex] = int(sample[PROBABILITY1]) - data['spu2'][sampleIndex] = int(sample[PROBABILITY2]) - data['spu3'][sampleIndex] = int(sample[PROBABILITY]) - data['spu4'][sampleIndex] = int(sample[PROBABILITY4]) - - data['nois1'][sampleIndex] = int(sample[AMBIENTNOISE1]) - data['nois2'][sampleIndex] = int(sample[AMBIENTNOISE2]) - data['nois3'][sampleIndex] = int(sample[AMBIENTNOISE3]) - data['nois4'][sampleIndex] = int(sample[AMBIENTNOISE4]) - - data['femax'][sampleIndex] = int(sample[CLUTTER]) - data['softw'][sampleIndex] = int(sample[SOFTWARE]) - - data['fe11'][sampleIndex] = int(sample[FREQUENCIES11]) - data['fe12'][sampleIndex] = int(sample[FREQUENCIES12]) - data['fe21'][sampleIndex] = int(sample[FREQUENCIES21]) - data['fe22'][sampleIndex] = int(sample[FREQUENCIES22]) - - data['snr1'][sampleIndex] = int(sample[SIGNALNOISE1]) - data['snr2'][sampleIndex] = int(sample[SIGNALNOISE2]) - data['snr3'][sampleIndex] = int(sample[SIGNALNOISE3]) - data['snr4'][sampleIndex] = int(sample[SIGNALNOISE4]) - - data['check'][sampleIndex] = int(sample[REFERENCE]) - data['jam'][sampleIndex] = int(sample[JAM]) + data['time'][sampleIndex] = procutil.dt2es(dt) + + data['val1'][sampleIndex] = int(sample['VAL1']) + data['val2'][sampleIndex] = int(sample['VAL2']) + data['val3'][sampleIndex] = int(sample['VAL3']) + data['val4'][sampleIndex] = int(sample['VAL4']) + + data['spu1'][sampleIndex] = int(sample['SPU1']) + data['spu2'][sampleIndex] = int(sample['SPU2']) + data['spu3'][sampleIndex] = int(sample['SPU3']) + data['spu4'][sampleIndex] = int(sample['SPU4']) + + data['nois1'][sampleIndex] = int(sample['NOIS1']) + data['nois2'][sampleIndex] = int(sample['NOIS2']) + data['nois3'][sampleIndex] = int(sample['NOIS3']) + data['nois4'][sampleIndex] = int(sample['NOIS4']) + + data['femax'][sampleIndex] = int(sample['FEMAX']) + data['softw'][sampleIndex] = int(sample['SOFTW']) + + data['fe11'][sampleIndex] = int(sample['FE11']) + data['fe12'][sampleIndex] = int(sample['FE12']) + data['fe21'][sampleIndex] = int(sample['FE21']) + data['fe22'][sampleIndex] = int(sample['FE22']) + + data['snr1'][sampleIndex] = int(sample['SNR1']) + data['snr2'][sampleIndex] = int(sample['SNR2']) + data['snr3'][sampleIndex] = int(sample['SNR3']) + data['snr4'][sampleIndex] = int(sample['SNR4']) + + data['check'][sampleIndex] = int(sample['CHECK']) + data['jam'][sampleIndex] = int(sample['JAM']) for altitude,alttitudeIndex in zip(altitudes, range(len(altitudes))): - echo = sample[altitude][ECHO] - radial = sample[altitude][RADIAL] - theta = sample[altitude][THETA] - vertical = sample[altitude][VERTICAL] + echo = sample[altitude]['CT'] + radial = sample[altitude]['SPEED'] + theta = sample[altitude]['DIR'] + vertical = sample[altitude]['W'] if radial != INVALID and theta != INVALID: @@ -191,14 +159,14 @@ 'metadata_url' : 'http://nccoos.unc.edu', 'references' : 'http://nccoos.unc.edu', - 'contact' : 'Sara Haines (haines@email.unc.edu)', + 'contact' : 'cbc (cbc@unc.edu)', # 'source' : 'fixed-profiler (acoustic doppler) observation', - 'history' : 'Data processed by NCCOOS', - 'comment' : 'File created using pycdf'+pycdfVersion()+' and numpy '+pycdfArrayPkg(), + 'history' : 'raw2proc using ' + sensor_info['process_module'], + 'comment' : 'File created using pycdf'+pycdf.pycdfVersion()+' and numpy '+pycdf.pycdfArrayPkg(), # conventions 'Conventions' : 'CF-1.0; SEACOOS-CDL-v2.0', # SEACOOS CDL codes 'format_category_code' : 'fixed-profiler', - 'institution_code' : platform_info['instituion'], + 'institution_code' : platform_info['institution'], 'platform_code' : platform_info['id'], 'package_code' : sensor_info['id'], @@ -207,10 +175,12 @@ 'project_url' : 'http://nccoos.unc.edu', # timeframe of data contained in file yyyy-mm-dd HH:MM:SS - 'start_date' : data['sample_dt'].strftime("%Y-%m-%d %H:%M:%S"), - 'end_date' : data['sample_dt'].strftime("%Y-%m-%d %H:%M:%S"), - 'release_date' : now.strftime("%Y-%m-%d %H:%M:%S"), + # first date in monthly file + 'start_date' : data['dt'][0].strftime("%Y-%m-%d %H:%M:%S"), + # last date in monthly file + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), # - 'creation_date' : now.strftime("%Y-%m-%d %H:%M:%S"), - 'modification_date' : now.strftime("%Y-%m-%d %H:%M:%S"), + 'creation_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + 'modification_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), 'process_level' : 'level1', # @@ -247,42 +217,137 @@ 'standard_name': 'height', 'reference':'zero at sea-surface', + 'positive' : 'up', 'units': 'm', 'axis': 'Z', }, # data variables - 'u': {'long_name': 'East/West Component of Current', - 'standard_name': 'eastward_current', - 'units': 'm s-1', - 'reference': 'clockwise from True East', - }, - 'v': {'long_name': 'North/South Component of Current', - 'standard_name': 'northward_current', - 'units': 'm s-1', - 'reference': 'clockwise from True North', - }, - 'w': {'long_name': 'Upward/Downward Component of Current', - 'standard_name': 'upward_current', - 'units': 'm s-1', - 'positive': 'up', - }, - 'back_scatter':{'long_name': 'Backscatter', - 'standard_name': 'back_scatter', - 'units': 'decibels', + 'u': {'short_name' : 'u', + 'long_name': 'East/West Component of Wind', + 'standard_name': 'eastward_wind', + 'units': 'cm s-1', + }, + 'v': {'short_name' : 'v', + 'long_name': 'North/South Component of Wind', + 'standard_name': 'northward_wind', + 'units': 'cm s-1', + }, + 'w': {'short_name' : 'w', + 'long_name': 'Vertical Component of Wind', + 'standard_name': 'upward_wind', + 'units': 'cm s-1', + }, + 'echo': {'short_name' : 'echo', + 'long_name': 'Echo Stength', + 'standard_name': 'echo_strenth', + }, + 'block' : {'short_name': 'block', + 'long_name': 'Block Number', + 'standard_name': 'block_number' + }, + 'val1': {'short_name' : 'val1', + 'long_name': 'Number of Beam Validations 1', + 'standard_name': 'validations_1', + }, + 'val2': {'short_name' : 'val2', + 'long_name': 'Number of Beam Validations 2', + 'standard_name': 'validations_2', + }, + 'val3': {'short_name' : 'val3', + 'long_name': 'Number of Beam Validations 3', + 'standard_name': 'validations_3', + }, + 'val4': {'short_name' : 'val4', + 'long_name': 'Number of Beam Validations 4', + 'standard_name': 'validations_4', + }, + 'spu1': {'short_name' : 'spu1', + 'long_name': 'Normalized Probability of False Signal 1', + 'standard_name': 'probability_1', + }, + 'spu2': {'short_name' : 'spu2', + 'long_name': 'Normalized Probability of False Signal 1', + 'standard_name': 'probability_2', + }, + 'spu3': {'short_name' : 'spu3', + 'long_name': 'Normalized Probability of False Signal 3', + 'standard_name': 'probability_3', + }, + 'spu4': {'short_name' : 'spu4', + 'long_name': 'Normalized Probability of False Signal 4', + 'standard_name': 'probability_4', + }, + 'nois1': {'short_name' : 'nois1', + 'long_name': 'Environmental Noise 1', + 'standard_name': 'ambient_1', + 'units': 'dB', + }, + 'nois2': {'short_name' : 'nois2', + 'long_name': 'Environmental Noise 2', + 'standard_name': 'ambient_2', + 'units': 'dB', + }, + 'nois3': {'short_name' : 'nois3', + 'long_name': 'Environmental Noise 3', + 'standard_name': 'ambient_3', + 'units': 'dB', + }, + 'nois4': {'short_name' : 'nois4', + 'long_name': 'Environmental Noise 4', + 'standard_name': 'ambient_4', + 'units': 'dB', + }, + 'femax': {'short_name': 'femax', + 'long_name': 'Maximum Ground Clutter', + 'standard_name': 'max_clutter', + }, + 'softw': {'short_name': 'softw', + 'long_name': 'Software Version', + 'standard_name': 'software', }, - 'wtemp': {'long_name': 'Water Temperature', - 'standard_name': 'water_temperature', - 'units': 'degrees Celsius', - }, + 'fe11': {'short_name': 'fe11', + 'long_name': 'Number of Frequencies Emitted 11', + 'standard_name': 'frequencies_11', + }, + 'fe12': {'short_name': 'fe12', + 'long_name': 'Number of Frequencies Emitted 12', + 'standard_name': 'frequencies_12', + }, + 'fe21': {'short_name': 'fe21', + 'long_name': 'Number of Frequencies Emitted 21', + 'standard_name': 'frequencies_21', + }, + 'fe22': {'short_name': 'fe22', + 'long_name': 'Number of Frequencies Emitted 22', + 'standard_name': 'frequencies_22', + }, + 'snr1': {'short_name' : 'snr1', + 'long_name': 'Average Signal To Noise Ratio 1', + 'standard_name': 'signal_to_noise_1', + 'units': 'dB', + }, + 'snr1': {'short_name' : 'snr2', + 'long_name': 'Average Signal To Noise Ratio 2', + 'standard_name': 'signal_to_noise_2', + 'units': 'dB', + }, + 'snr1': {'short_name' : 'snr3', + 'long_name': 'Average Signal To Noise Ratio 3', + 'standard_name': 'signal_to_noise_3', + 'units': 'dB', + }, + 'snr4': {'short_name' : 'snr4', + 'long_name': 'Average Signal To Noise Ratio 4', + 'standard_name': 'signal_to_noise_4', + 'units': 'dB', + }, } - - # integer values - ntime=NC.UNLIMITED - nlat=1 - nlon=1 - nz=sensor_info['nbins'] - # dimension names use tuple so order of initialization is maintained - dimensions = ('ntime', 'nlat', 'nlon', 'nz') + dim_inits = ( + ('ntime', pycdf.NC.UNLIMITED), + ('nlat', 1), + ('nlon', 1), + ('nz', sensor_info['num_altitudes']) + ) # using tuple of tuples so order of initialization is maintained @@ -292,49 +357,127 @@ var_inits = ( # coordinate variables - ('time', NC.INT, ('ntime',)), - ('lat', NC.FLOAT, ('nlat',)), - ('lon', NC.FLOAT, ('nlon',)), - ('z', NC.FLOAT, ('nz',)), + ('time', pycdf.NC.INT, ('ntime',)), + ('lat', pycdf.NC.FLOAT, ('nlat',)), + ('lon', pycdf.NC.FLOAT, ('nlon',)), + ('z', pycdf.NC.FLOAT, ('nz',)), # data variables - ('u', NC.FLOAT, ('ntime', 'nz')), - ('v', NC.FLOAT, ('ntime', 'nz')), - ('w', NC.FLOAT, ('ntime', 'nz')), - ('back_scatter', NC.FLOAT, ('ntime', 'nz')), - ('wtemp', NC.FLOAT, ('ntime',)), + ('u', pycdf.NC.FLOAT, ('ntime', 'nz')), + ('v', pycdf.NC.FLOAT, ('ntime', 'nz')), + ('w', pycdf.NC.FLOAT, ('ntime', 'nz')), + ('echo', pycdf.NC.FLOAT, ('ntime', 'nz')), + ('block', pycdf.NC.INT, ('ntime',)), + ('val1', pycdf.NC.INT, ('ntime',)), + ('val2', pycdf.NC.INT, ('ntime',)), + ('val3', pycdf.NC.INT, ('ntime',)), + ('val4', pycdf.NC.INT, ('ntime',)), + ('spu1', pycdf.NC.INT, ('ntime',)), + ('spu2', pycdf.NC.INT, ('ntime',)), + ('spu3', pycdf.NC.INT, ('ntime',)), + ('spu4', pycdf.NC.INT, ('ntime',)), + ('nois1', pycdf.NC.INT, ('ntime',)), + ('nois2', pycdf.NC.INT, ('ntime',)), + ('nois3', pycdf.NC.INT, ('ntime',)), + ('nois4', pycdf.NC.INT, ('ntime',)), + ('femax', pycdf.NC.INT, ('ntime',)), + ('softw', pycdf.NC.INT, ('ntime',)), + ('fe11', pycdf.NC.INT, ('ntime',)), + ('fe12', pycdf.NC.INT, ('ntime',)), + ('fe21', pycdf.NC.INT, ('ntime',)), + ('fe22', pycdf.NC.INT, ('ntime',)), + ('snr1', pycdf.NC.INT, ('ntime',)), + ('snr2', pycdf.NC.INT, ('ntime',)), + ('snr3', pycdf.NC.INT, ('ntime',)), + ('snr4', pycdf.NC.INT, ('ntime',)), ) + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] # var data var_data = ( - ('lat', platform_info['lat']), - ('lon', platform_info['lon']), - ('z', []), - ('u', []), - ('v', []), - ('w', []), - ('back_scatter', []), - ('wtemp', []), + ('time', data['time'][i]), + ('lat', platform_info['lat']), + ('lon', platform_info['lon']), + ('z', data['z']), + ('u', data['u'][i]), + ('v', data['v'][i]), + ('w', data['w'][i]), + ('echo', data['echo'][i]), + ('block', data['block'][i]), + ('val1', data['val1'][i]), + ('val2', data['val1'][i]), + ('val3', data['val1'][i]), + ('val4', data['val1'][i]), + ('spu1', data['spu1'][i]), + ('spu2', data['spu2'][i]), + ('spu3', data['spu3'][i]), + ('spu4', data['spu4'][i]), + ('nois1', data['nois1'][i]), + ('nois2', data['nois2'][i]), + ('nois3', data['nois3'][i]), + ('nois4', data['nois4'][i]), + ('femax', data['femax'][i]), + ('softw', data['softw'][i]), + ('fe11', data['fe11'][i]), + ('fe12', data['fe12'][i]), + ('fe21', data['fe21'][i]), + ('fe22', data['fe22'][i]), + ('snr1', data['snr1'][i]), + ('snr2', data['snr2'][i]), + ('snr3', data['snr3'][i]), + ('snr4', data['snr4'][i]), ) - return (global_atts, dimensions, var_inits, var_data) + return (global_atts, var_atts, dim_inits, var_inits, var_data) def updater(platform_info, sensor_info, data): # global_atts = { - # timeframe of data contained in file yyyy-mm-dd HH:MM:SS - 'end_date' : data['sample_dt'].strftime("%Y-%m-%d %H:%M:%S"), - 'release_date' : now.strftime("%Y-%m-%d %H:%M:%S"), + # update times of data contained in file (yyyy-mm-dd HH:MM:SS) + # last date in monthly file + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), # - 'creation_date' : now.strftime("%Y-%m-%d %H:%M:%S"), - 'modification_date' : now.strftime("%Y-%m-%d %H:%M:%S"), + 'modification_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), } - # var data + + # data variables + # update any variable attributes like range, min, max + var_atts = {} + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # data var_data = ( - ('u', data['u']), - ('v', data['v']), - ('w', data['w']), - ('back_scatter', data['back_scatter']), - ('wtemp', data['wtemp']), + ('time', data['time'][i]), + ('u', data['u'][i]), + ('v', data['v'][i]), + ('w', data['w'][i]), + ('echo', data['echo'][i]), + ('block', data['block'][i]), + ('val1', data['val1'][i]), + ('val2', data['val1'][i]), + ('val3', data['val1'][i]), + ('val4', data['val1'][i]), + ('spu1', data['spu1'][i]), + ('spu2', data['spu2'][i]), + ('spu3', data['spu3'][i]), + ('spu4', data['spu4'][i]), + ('nois1', data['nois1'][i]), + ('nois2', data['nois2'][i]), + ('nois3', data['nois3'][i]), + ('nois4', data['nois4'][i]), + ('femax', data['femax'][i]), + ('softw', data['softw'][i]), + ('fe11', data['fe11'][i]), + ('fe12', data['fe12'][i]), + ('fe21', data['fe21'][i]), + ('fe22', data['fe22'][i]), + ('snr1', data['snr1'][i]), + ('snr2', data['snr2'][i]), + ('snr3', data['snr3'][i]), + ('snr4', data['snr4'][i]), ) - return (global_atts, var_data) -# - + + return (global_atts, var_atts, var_data)