Index: raw2proc/trunk/raw2proc/largo_mini_andi_config_20100401.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/largo_mini_andi_config_20100401.py (revision 375) @@ -1,0 +1,1368 @@ +platform_info = { + # Used by raw2proc + 'id' : 'mini_andi', + 'config_start_date' : '2010-04-01 00:00:00', + 'config_end_date' : '2010-07-31 00:00:00', # None or yyyy-mm-dd HH:MM:SS + 'packages' : ('system', + 'turbidity', + 'optode_127', + 'optode_167', + 'optode_169', + 'optode_170', + 'optode_171', + 'optode_172', + 'optode_173', + 'optode_175', + 'optode_176', + 'optode_178', + 'optode_179', + 'conductivity', + 'pressure', + 'current', + ), + # Required + 'institution' : 'Unversity of North Carolina at Chapel Hill (UNC-CH)', + 'institution_url' : 'http://www.unc.edu', + 'institution_dods_url' : 'http://whewell.marine.unc.edu/dods', + 'contact' : 'Howard Mendlovitz (mendlovitz@unc.edu)', + 'conventions' : 'CF-1.0; SEACOOS-CDL-v2.0', + # Required by Scout + 'format_category_code' : 'fixed-point', + 'institution_code' : 'nccoos', + # Required by Version Tracking + 'format' : 'seacoos_rt', + 'seacoos_rt_version' : '2.0', + # Recommended + 'source' : 'Spongenet platform', + 'references' : 'http://ncccoos.org', + 'metadata_url' : 'http://whewell.marine.unc.edu/dods', + 'project' : 'UNC-CH Department of Marine Sciences', + 'project_url' : 'http://www.marine.unc.edu', + # Custom + 'id_number' : '4887-231', + 'description' : 'Platform', + 'serial_number' : '231', + 'product_number' : '4887', + 'product_name' : 'Seaguard Datalogger IW', + 'type' : 'Instrument', + 'protocol_version' : '4', + 'xmlns' : 'http://www.aadi.no/RTOutSchema', + 'location' : 'NURC', + 'vertical_position': '0', + 'owner' : 'UNC Chapel Hill', + 'status_map' : '0:OK', +} +sensor_info = { + 'system' : { 'id' : 'system', + # Used by raw2proc + 'raw_dir' : '/seacoos/data/nccoos/level0/' + 'largo/mini_andi/xml', + 'raw_file_glob' : '*.xml', + 'proc_dir' : '/seacoos/data/nccoos/level1/' + 'largo/mini_andi/nc', + 'process_module' : 'proc_mini_andi_system', + 'utc_offset' : 0, # hours offset to utc + 'location' : 'Florida Keys National Marine Sanctuary', + 'lat' : 24.9500, # degrees true (-) south, (+) north + 'lon' : -80.4536, # degrees true (-) west, (+) east + 'mvar' : -5.6333, # degrees (-) west, (+) east + 'elevation' : -20.0, # meters + # Custom + 'id_number' : 'SN100-0', + 'serial_number' : '0', + 'product_number' : 'SN100', + 'product_name' : 'System Node', + 'description' : 'System Parameters', + 'adr' : '-4', + 'protocol_version' : '3', + 'vertical_position' : '0', + 'status_map' : '0:OK', + # Custom for variables + 'voltage_id' : '0', + 'voltage_description' : 'Battery Voltage', + 'voltage_type' : 'VT_R4', + 'voltage_format' : '', + 'voltage_units' : 'V', + 'voltage_range_min' :'0', + 'voltage_range_max' : '15', + 'memory_id' : '1', + 'memory_description' : 'Memory Used', + 'memory_type' : 'VT_I4', + 'memory_format' : '', + 'memory_units' : 'Bytes', + 'memory_range_min' : '0', + 'memory_range_max' : '14024704', + 'interval_id' : '2', + 'interval_description' : 'Interval', + 'interval_type' : 'VT_I4', + 'interval_format' : '', + 'interval_units' : 'ms', + 'interval_range_min' : '', + 'interval_range_max' : '', + }, + 'turbidity' : { 'id' : 'turbidity', + # Used by raw2proc + 'raw_dir' : '/seacoos/data/nccoos/level0/' + 'largo/mini_andi/xml', + 'raw_file_glob' : '*.xml', + 'proc_dir' : '/seacoos/data/nccoos/level1/' + 'largo/mini_andi/nc', + 'process_module' : 'proc_mini_andi_turbidity', + 'utc_offset' : 0, # hours offset to utc + 'location' : 'Florida Keys National Marine Sanctuary', + 'lat' : 24.9500, # degrees true (-) south, (+) north + 'lon' : -80.4536, # degrees true (-) west, (+) east + 'mvar' : -5.6333, # degrees (-) west, (+) east + 'elevation' : -20.0, # meters + # Custom + 'id_number' : 'AN100-0', + 'serial_number' : '0', + 'product_number' : 'AN100', + 'product_name' : 'Analog Sensors', + 'description' : 'Turbidity #11692', + 'adr' : '-3', + 'protocol_version' : '3', + 'vertical_position' : '0', + 'status_map' : '0:OK', + # Custom for variables + 'turbidity_id' : '3', + 'turbidity_description' : 'Turbidity', + 'turbidity_type' : 'VT_R8', + 'turbidity_format' : '', + 'turbidity_units' : 'FTU', + 'turbidity_range_min' :'0', + 'turbidity_range_max' : '125', + }, + 'optode_127' : { 'id' : 'optode_127', + # Used by raw2proc + 'raw_dir' : '/seacoos/data/nccoos/level0/' + 'largo/mini_andi/xml', + 'raw_file_glob' : '*.xml', + 'proc_dir' : '/seacoos/data/nccoos/level1/' + 'largo/mini_andi/nc', + 'process_module' : 'proc_mini_andi_optode', + 'utc_offset' : 0, # hours offset to utc + 'location' : 'Florida Keys National Marine Sanctuary', + 'lat' : 24.9500, # degrees true (-) south, (+) north + 'lon' : -80.4536, # degrees true (-) west, (+) east + 'mvar' : -5.6333, # degrees (-) west, (+) east + 'elevation' : -20.0, # meters + # Custom + 'id_number' : '4330F-127', + 'serial_number' : '127', + 'product_number' : '4330F', + 'product_name' : 'Optode Sensor', + 'description' : 'Optode Sensor 4330F#127', + 'adr' : '340736', + 'protocol_version' : '1', + 'vertical_position' : '0', + 'status_map' : '0:OK', + # Custom for variables + 'o2concentration_id' : '0', + 'o2concentration_description' : 'O2Concentration', + 'o2concentration_type' : 'VT_R4', + 'o2concentration_format' : '%0.3f', + 'o2concentration_units' : 'uM', + 'o2concentration_range_min' :'0', + 'o2concentration_range_max' : '500', + 'airsaturation_id' : '1', + 'airsaturation_description' : 'AirSaturation', + 'airsaturation_type' : 'VT_R4', + 'airsaturation_format' : '%0.3f', + 'airsaturation_units' : '%', + 'airsaturation_range_min' :'0', + 'airsaturation_range_max' : '150', + 'temperature_id' : '2', + 'temperature_description' : 'Temperature', + 'temperature_type' : 'VT_R4', + 'temperature_format' : '%0.3f', + 'temperature_units' : 'Deg.C', + 'temperature_range_min' :'0', + 'temperature_range_max' : '40', + 'calphase_id' : '3', + 'calphase_description' : 'CalPhase', + 'calphase_type' : 'VT_R4', + 'calphase_format' : '%0.3f', + 'calphase_units' : 'Deg', + 'calphase_range_min' :'0', + 'calphase_range_max' : '90', + 'tcphase_id' : '4', + 'tcphase_description' : 'TCPhase', + 'tcphase_type' : 'VT_R4', + 'tcphase_format' : '%0.3f', + 'tcphase_units' : 'Deg', + 'tcphase_range_min' :'0', + 'tcphase_range_max' : '90', + 'c1rph_id' : '5', + 'c1rph_description' : 'C1RPh', + 'c1rph_type' : 'VT_R4', + 'c1rph_format' : '%0.3f', + 'c1rph_units' : 'Deg', + 'c1rph_range_min' :'0', + 'c1rph_range_max' : '90', + 'c2rph_id' : '6', + 'c2rph_description' : 'C2RPh', + 'c2rph_type' : 'VT_R4', + 'c2rph_format' : '%0.3f', + 'c2rph_units' : 'Deg', + 'c2rph_range_min' :'0', + 'c2rph_range_max' : '90', + 'c1amp_id' : '7', + 'c1amp_description' : 'C1Amp', + 'c1amp_type' : 'VT_R4', + 'c1amp_format' : '%0.1f', + 'c1amp_units' : 'mV', + 'c1amp_range_min' :'0', + 'c1amp_range_max' : '3300', + 'c2amp_id' : '8', + 'c2amp_description' : 'C2Amp', + 'c2amp_type' : 'VT_R4', + 'c2amp_format' : '%0.1f', + 'c2amp_units' : 'mV', + 'c2amp_range_min' :'0', + 'c2amp_range_max' : '3300', + 'rawtemp_id' : '9', + 'rawtemp_description' : 'RawTemp', + 'rawtemp_type' : 'VT_R4', + 'rawtemp_format' : '%0.1f', + 'rawtemp_units' : 'mV', + 'rawtemp_range_min' :'0', + 'rawtemp_range_max' : '1500', + }, + 'optode_167' : { 'id' : 'optode_167', + # Used by raw2proc + 'raw_dir' : '/seacoos/data/nccoos/level0/' + 'largo/mini_andi/xml', + 'raw_file_glob' : '*.xml', + 'proc_dir' : '/seacoos/data/nccoos/level1/' + 'largo/mini_andi/nc', + 'process_module' : 'proc_mini_andi_optode', + 'utc_offset' : 0, # hours offset to utc + 'location' : 'Florida Keys National Marine Sanctuary', + 'lat' : 24.9500, # degrees true (-) south, (+) north + 'lon' : -80.4536, # degrees true (-) west, (+) east + 'mvar' : -5.6333, # degrees (-) west, (+) east + 'elevation' : -20.0, # meters + # Custom + 'id_number' : '4330F-167', + 'serial_number' : '167', + 'product_number' : '4330F', + 'product_name' : 'Optode Sensor', + 'description' : 'Optode Sensor 4330F#167', + 'adr' : '352512', + 'protocol_version' : '1', + 'vertical_position' : '0', + 'status_map' : '0:OK', + # Custom for variables + 'o2concentration_id' : '0', + 'o2concentration_description' : 'O2Concentration', + 'o2concentration_type' : 'VT_R4', + 'o2concentration_format' : '%0.3f', + 'o2concentration_units' : 'uM', + 'o2concentration_range_min' :'0', + 'o2concentration_range_max' : '500', + 'airsaturation_id' : '1', + 'airsaturation_description' : 'AirSaturation', + 'airsaturation_type' : 'VT_R4', + 'airsaturation_format' : '%0.3f', + 'airsaturation_units' : '%', + 'airsaturation_range_min' :'0', + 'airsaturation_range_max' : '150', + 'temperature_id' : '2', + 'temperature_description' : 'Temperature', + 'temperature_type' : 'VT_R4', + 'temperature_format' : '%0.3f', + 'temperature_units' : 'Deg.C', + 'temperature_range_min' :'0', + 'temperature_range_max' : '40', + 'calphase_id' : '3', + 'calphase_description' : 'CalPhase', + 'calphase_type' : 'VT_R4', + 'calphase_format' : '%0.3f', + 'calphase_units' : 'Deg', + 'calphase_range_min' :'0', + 'calphase_range_max' : '90', + 'tcphase_id' : '4', + 'tcphase_description' : 'TCPhase', + 'tcphase_type' : 'VT_R4', + 'tcphase_format' : '%0.3f', + 'tcphase_units' : 'Deg', + 'tcphase_range_min' :'0', + 'tcphase_range_max' : '90', + 'c1rph_id' : '5', + 'c1rph_description' : 'C1RPh', + 'c1rph_type' : 'VT_R4', + 'c1rph_format' : '%0.3f', + 'c1rph_units' : 'Deg', + 'c1rph_range_min' :'0', + 'c1rph_range_max' : '90', + 'c2rph_id' : '6', + 'c2rph_description' : 'C2RPh', + 'c2rph_type' : 'VT_R4', + 'c2rph_format' : '%0.3f', + 'c2rph_units' : 'Deg', + 'c2rph_range_min' :'0', + 'c2rph_range_max' : '90', + 'c1amp_id' : '7', + 'c1amp_description' : 'C1Amp', + 'c1amp_type' : 'VT_R4', + 'c1amp_format' : '%0.1f', + 'c1amp_units' : 'mV', + 'c1amp_range_min' :'0', + 'c1amp_range_max' : '3300', + 'c2amp_id' : '8', + 'c2amp_description' : 'C2Amp', + 'c2amp_type' : 'VT_R4', + 'c2amp_format' : '%0.1f', + 'c2amp_units' : 'mV', + 'c2amp_range_min' :'0', + 'c2amp_range_max' : '3300', + 'rawtemp_id' : '9', + 'rawtemp_description' : 'RawTemp', + 'rawtemp_type' : 'VT_R4', + 'rawtemp_format' : '%0.1f', + 'rawtemp_units' : 'mV', + 'rawtemp_range_min' :'0', + 'rawtemp_range_max' : '1500', + }, + 'optode_169' : { 'id' : 'optode_169', + # Used by raw2proc + 'raw_dir' : '/seacoos/data/nccoos/level0/' + 'largo/mini_andi/xml', + 'raw_file_glob' : '*.xml', + 'proc_dir' : '/seacoos/data/nccoos/level1/' + 'largo/mini_andi/nc', + 'process_module' : 'proc_mini_andi_optode', + 'utc_offset' : 0, # hours offset to utc + 'location' : 'Florida Keys National Marine Sanctuary', + 'lat' : 24.9500, # degrees true (-) south, (+) north + 'lon' : -80.4536, # degrees true (-) west, (+) east + 'mvar' : -5.6333, # degrees (-) west, (+) east + 'elevation' : -20.0, # meters + # Custom + 'id_number' : '4330F-169', + 'serial_number' : '169', + 'product_number' : '4330F', + 'product_name' : 'Optode Sensor', + 'description' : 'Optode Sensor 4330F#169', + 'adr' : '353024', + 'protocol_version' : '1', + 'vertical_position' : '0', + 'status_map' : '0:OK', + # Custom for variables + 'o2concentration_id' : '0', + 'o2concentration_description' : 'O2Concentration', + 'o2concentration_type' : 'VT_R4', + 'o2concentration_format' : '%0.3f', + 'o2concentration_units' : 'uM', + 'o2concentration_range_min' :'0', + 'o2concentration_range_max' : '500', + 'airsaturation_id' : '1', + 'airsaturation_description' : 'AirSaturation', + 'airsaturation_type' : 'VT_R4', + 'airsaturation_format' : '%0.3f', + 'airsaturation_units' : '%', + 'airsaturation_range_min' :'0', + 'airsaturation_range_max' : '150', + 'temperature_id' : '2', + 'temperature_description' : 'Temperature', + 'temperature_type' : 'VT_R4', + 'temperature_format' : '%0.3f', + 'temperature_units' : 'Deg.C', + 'temperature_range_min' :'0', + 'temperature_range_max' : '40', + 'calphase_id' : '3', + 'calphase_description' : 'CalPhase', + 'calphase_type' : 'VT_R4', + 'calphase_format' : '%0.3f', + 'calphase_units' : 'Deg', + 'calphase_range_min' :'0', + 'calphase_range_max' : '90', + 'tcphase_id' : '4', + 'tcphase_description' : 'TCPhase', + 'tcphase_type' : 'VT_R4', + 'tcphase_format' : '%0.3f', + 'tcphase_units' : 'Deg', + 'tcphase_range_min' :'0', + 'tcphase_range_max' : '90', + 'c1rph_id' : '5', + 'c1rph_description' : 'C1RPh', + 'c1rph_type' : 'VT_R4', + 'c1rph_format' : '%0.3f', + 'c1rph_units' : 'Deg', + 'c1rph_range_min' :'0', + 'c1rph_range_max' : '90', + 'c2rph_id' : '6', + 'c2rph_description' : 'C2RPh', + 'c2rph_type' : 'VT_R4', + 'c2rph_format' : '%0.3f', + 'c2rph_units' : 'Deg', + 'c2rph_range_min' :'0', + 'c2rph_range_max' : '90', + 'c1amp_id' : '7', + 'c1amp_description' : 'C1Amp', + 'c1amp_type' : 'VT_R4', + 'c1amp_format' : '%0.1f', + 'c1amp_units' : 'mV', + 'c1amp_range_min' :'0', + 'c1amp_range_max' : '3300', + 'c2amp_id' : '8', + 'c2amp_description' : 'C2Amp', + 'c2amp_type' : 'VT_R4', + 'c2amp_format' : '%0.1f', + 'c2amp_units' : 'mV', + 'c2amp_range_min' :'0', + 'c2amp_range_max' : '3300', + 'rawtemp_id' : '9', + 'rawtemp_description' : 'RawTemp', + 'rawtemp_type' : 'VT_R4', + 'rawtemp_format' : '%0.1f', + 'rawtemp_units' : 'mV', + 'rawtemp_range_min' :'0', + 'rawtemp_range_max' : '1500', + }, + 'optode_170' : { 'id' : 'optode_170', + # Used by raw2proc + 'raw_dir' : '/seacoos/data/nccoos/level0/' + 'largo/mini_andi/xml', + 'raw_file_glob' : '*.xml', + 'proc_dir' : '/seacoos/data/nccoos/level1/' + 'largo/mini_andi/nc', + 'process_module' : 'proc_mini_andi_optode', + 'utc_offset' : 0, # hours offset to utc + 'location' : 'Florida Keys National Marine Sanctuary', + 'lat' : 24.9500, # degrees true (-) south, (+) north + 'lon' : -80.4536, # degrees true (-) west, (+) east + 'mvar' : -5.6333, # degrees (-) west, (+) east + 'elevation' : -20.0, # meters + # Custom + 'id_number' : '4330F-170', + 'serial_number' : '170', + 'product_number' : '4330F', + 'product_name' : 'Optode Sensor', + 'description' : 'Optode Sensor 4330F#170', + 'adr' : '353280', + 'protocol_version' : '1', + 'vertical_position' : '0', + 'status_map' : '0:OK', + # Custom for variables + 'o2concentration_id' : '0', + 'o2concentration_description' : 'O2Concentration', + 'o2concentration_type' : 'VT_R4', + 'o2concentration_format' : '%0.3f', + 'o2concentration_units' : 'uM', + 'o2concentration_range_min' :'0', + 'o2concentration_range_max' : '500', + 'airsaturation_id' : '1', + 'airsaturation_description' : 'AirSaturation', + 'airsaturation_type' : 'VT_R4', + 'airsaturation_format' : '%0.3f', + 'airsaturation_units' : '%', + 'airsaturation_range_min' :'0', + 'airsaturation_range_max' : '150', + 'temperature_id' : '2', + 'temperature_description' : 'Temperature', + 'temperature_type' : 'VT_R4', + 'temperature_format' : '%0.3f', + 'temperature_units' : 'Deg.C', + 'temperature_range_min' :'0', + 'temperature_range_max' : '40', + 'calphase_id' : '3', + 'calphase_description' : 'CalPhase', + 'calphase_type' : 'VT_R4', + 'calphase_format' : '%0.3f', + 'calphase_units' : 'Deg', + 'calphase_range_min' :'0', + 'calphase_range_max' : '90', + 'tcphase_id' : '4', + 'tcphase_description' : 'TCPhase', + 'tcphase_type' : 'VT_R4', + 'tcphase_format' : '%0.3f', + 'tcphase_units' : 'Deg', + 'tcphase_range_min' :'0', + 'tcphase_range_max' : '90', + 'c1rph_id' : '5', + 'c1rph_description' : 'C1RPh', + 'c1rph_type' : 'VT_R4', + 'c1rph_format' : '%0.3f', + 'c1rph_units' : 'Deg', + 'c1rph_range_min' :'0', + 'c1rph_range_max' : '90', + 'c2rph_id' : '6', + 'c2rph_description' : 'C2RPh', + 'c2rph_type' : 'VT_R4', + 'c2rph_format' : '%0.3f', + 'c2rph_units' : 'Deg', + 'c2rph_range_min' :'0', + 'c2rph_range_max' : '90', + 'c1amp_id' : '7', + 'c1amp_description' : 'C1Amp', + 'c1amp_type' : 'VT_R4', + 'c1amp_format' : '%0.1f', + 'c1amp_units' : 'mV', + 'c1amp_range_min' :'0', + 'c1amp_range_max' : '3300', + 'c2amp_id' : '8', + 'c2amp_description' : 'C2Amp', + 'c2amp_type' : 'VT_R4', + 'c2amp_format' : '%0.1f', + 'c2amp_units' : 'mV', + 'c2amp_range_min' :'0', + 'c2amp_range_max' : '3300', + 'rawtemp_id' : '9', + 'rawtemp_description' : 'RawTemp', + 'rawtemp_type' : 'VT_R4', + 'rawtemp_format' : '%0.1f', + 'rawtemp_units' : 'mV', + 'rawtemp_range_min' :'0', + 'rawtemp_range_max' : '1500', + }, + 'optode_171' : { 'id' : 'optode_171', + # Used by raw2proc + 'raw_dir' : '/seacoos/data/nccoos/level0/' + 'largo/mini_andi/xml', + 'raw_file_glob' : '*.xml', + 'proc_dir' : '/seacoos/data/nccoos/level1/' + 'largo/mini_andi/nc', + 'process_module' : 'proc_mini_andi_optode', + 'utc_offset' : 0, # hours offset to utc + 'location' : 'Florida Keys National Marine Sanctuary', + 'lat' : 24.9500, # degrees true (-) south, (+) north + 'lon' : -80.4536, # degrees true (-) west, (+) east + 'mvar' : -5.6333, # degrees (-) west, (+) east + 'elevation' : -20.0, # meters + # Custom + 'id_number' : '4330F-171', + 'serial_number' : '171', + 'product_number' : '4330F', + 'product_name' : 'Optode Sensor', + 'description' : 'Optode Sensor 4330F#171', + 'adr' : '353536', + 'protocol_version' : '1', + 'vertical_position' : '0', + 'status_map' : '0:OK', + # Custom for variables + 'o2concentration_id' : '0', + 'o2concentration_description' : 'O2Concentration', + 'o2concentration_type' : 'VT_R4', + 'o2concentration_format' : '%0.3f', + 'o2concentration_units' : 'uM', + 'o2concentration_range_min' :'0', + 'o2concentration_range_max' : '500', + 'airsaturation_id' : '1', + 'airsaturation_description' : 'AirSaturation', + 'airsaturation_type' : 'VT_R4', + 'airsaturation_format' : '%0.3f', + 'airsaturation_units' : '%', + 'airsaturation_range_min' :'0', + 'airsaturation_range_max' : '150', + 'temperature_id' : '2', + 'temperature_description' : 'Temperature', + 'temperature_type' : 'VT_R4', + 'temperature_format' : '%0.3f', + 'temperature_units' : 'Deg.C', + 'temperature_range_min' :'0', + 'temperature_range_max' : '40', + 'calphase_id' : '3', + 'calphase_description' : 'CalPhase', + 'calphase_type' : 'VT_R4', + 'calphase_format' : '%0.3f', + 'calphase_units' : 'Deg', + 'calphase_range_min' :'0', + 'calphase_range_max' : '90', + 'tcphase_id' : '4', + 'tcphase_description' : 'TCPhase', + 'tcphase_type' : 'VT_R4', + 'tcphase_format' : '%0.3f', + 'tcphase_units' : 'Deg', + 'tcphase_range_min' :'0', + 'tcphase_range_max' : '90', + 'c1rph_id' : '5', + 'c1rph_description' : 'C1RPh', + 'c1rph_type' : 'VT_R4', + 'c1rph_format' : '%0.3f', + 'c1rph_units' : 'Deg', + 'c1rph_range_min' :'0', + 'c1rph_range_max' : '90', + 'c2rph_id' : '6', + 'c2rph_description' : 'C2RPh', + 'c2rph_type' : 'VT_R4', + 'c2rph_format' : '%0.3f', + 'c2rph_units' : 'Deg', + 'c2rph_range_min' :'0', + 'c2rph_range_max' : '90', + 'c1amp_id' : '7', + 'c1amp_description' : 'C1Amp', + 'c1amp_type' : 'VT_R4', + 'c1amp_format' : '%0.1f', + 'c1amp_units' : 'mV', + 'c1amp_range_min' :'0', + 'c1amp_range_max' : '3300', + 'c2amp_id' : '8', + 'c2amp_description' : 'C2Amp', + 'c2amp_type' : 'VT_R4', + 'c2amp_format' : '%0.1f', + 'c2amp_units' : 'mV', + 'c2amp_range_min' :'0', + 'c2amp_range_max' : '3300', + 'rawtemp_id' : '9', + 'rawtemp_description' : 'RawTemp', + 'rawtemp_type' : 'VT_R4', + 'rawtemp_format' : '%0.1f', + 'rawtemp_units' : 'mV', + 'rawtemp_range_min' :'0', + 'rawtemp_range_max' : '1500', + }, + 'optode_172' : { 'id' : 'optode_172', + # Used by raw2proc + 'raw_dir' : '/seacoos/data/nccoos/level0/' + 'largo/mini_andi/xml', + 'raw_file_glob' : '*.xml', + 'proc_dir' : '/seacoos/data/nccoos/level1/' + 'largo/mini_andi/nc', + 'process_module' : 'proc_mini_andi_optode', + 'utc_offset' : 0, # hours offset to utc + 'location' : 'Florida Keys National Marine Sanctuary', + 'lat' : 24.9500, # degrees true (-) south, (+) north + 'lon' : -80.4536, # degrees true (-) west, (+) east + 'mvar' : -5.6333, # degrees (-) west, (+) east + 'elevation' : -20.0, # meters + # Custom + 'id_number' : '4330F-172', + 'serial_number' : '172', + 'product_number' : '4330F', + 'product_name' : 'Optode Sensor', + 'description' : 'Optode Sensor 4330F#172', + 'adr' : '353792', + 'protocol_version' : '1', + 'vertical_position' : '0', + 'status_map' : '0:OK', + # Custom for variables + 'o2concentration_id' : '0', + 'o2concentration_description' : 'O2Concentration', + 'o2concentration_type' : 'VT_R4', + 'o2concentration_format' : '%0.3f', + 'o2concentration_units' : 'uM', + 'o2concentration_range_min' :'0', + 'o2concentration_range_max' : '500', + 'airsaturation_id' : '1', + 'airsaturation_description' : 'AirSaturation', + 'airsaturation_type' : 'VT_R4', + 'airsaturation_format' : '%0.3f', + 'airsaturation_units' : '%', + 'airsaturation_range_min' :'0', + 'airsaturation_range_max' : '150', + 'temperature_id' : '2', + 'temperature_description' : 'Temperature', + 'temperature_type' : 'VT_R4', + 'temperature_format' : '%0.3f', + 'temperature_units' : 'Deg.C', + 'temperature_range_min' :'0', + 'temperature_range_max' : '40', + 'calphase_id' : '3', + 'calphase_description' : 'CalPhase', + 'calphase_type' : 'VT_R4', + 'calphase_format' : '%0.3f', + 'calphase_units' : 'Deg', + 'calphase_range_min' :'0', + 'calphase_range_max' : '90', + 'tcphase_id' : '4', + 'tcphase_description' : 'TCPhase', + 'tcphase_type' : 'VT_R4', + 'tcphase_format' : '%0.3f', + 'tcphase_units' : 'Deg', + 'tcphase_range_min' :'0', + 'tcphase_range_max' : '90', + 'c1rph_id' : '5', + 'c1rph_description' : 'C1RPh', + 'c1rph_type' : 'VT_R4', + 'c1rph_format' : '%0.3f', + 'c1rph_units' : 'Deg', + 'c1rph_range_min' :'0', + 'c1rph_range_max' : '90', + 'c2rph_id' : '6', + 'c2rph_description' : 'C2RPh', + 'c2rph_type' : 'VT_R4', + 'c2rph_format' : '%0.3f', + 'c2rph_units' : 'Deg', + 'c2rph_range_min' :'0', + 'c2rph_range_max' : '90', + 'c1amp_id' : '7', + 'c1amp_description' : 'C1Amp', + 'c1amp_type' : 'VT_R4', + 'c1amp_format' : '%0.1f', + 'c1amp_units' : 'mV', + 'c1amp_range_min' :'0', + 'c1amp_range_max' : '3300', + 'c2amp_id' : '8', + 'c2amp_description' : 'C2Amp', + 'c2amp_type' : 'VT_R4', + 'c2amp_format' : '%0.1f', + 'c2amp_units' : 'mV', + 'c2amp_range_min' :'0', + 'c2amp_range_max' : '3300', + 'rawtemp_id' : '9', + 'rawtemp_description' : 'RawTemp', + 'rawtemp_type' : 'VT_R4', + 'rawtemp_format' : '%0.1f', + 'rawtemp_units' : 'mV', + 'rawtemp_range_min' :'0', + 'rawtemp_range_max' : '1500', + }, + 'optode_173' : { 'id' : 'optode_173', + # Used by raw2proc + 'raw_dir' : '/seacoos/data/nccoos/level0/' + 'largo/mini_andi/xml', + 'raw_file_glob' : '*.xml', + 'proc_dir' : '/seacoos/data/nccoos/level1/' + 'largo/mini_andi/nc', + 'process_module' : 'proc_mini_andi_optode', + 'utc_offset' : 0, # hours offset to utc + 'location' : 'Florida Keys National Marine Sanctuary', + 'lat' : 24.9500, # degrees true (-) south, (+) north + 'lon' : -80.4536, # degrees true (-) west, (+) east + 'mvar' : -5.6333, # degrees (-) west, (+) east + 'elevation' : -20.0, # meters + # Custom + 'id_number' : '4330F-173', + 'serial_number' : '173', + 'product_number' : '4330F', + 'product_name' : 'Optode Sensor', + 'description' : 'Optode Sensor 4330F#173', + 'adr' : '354048', + 'protocol_version' : '1', + 'vertical_position' : '0', + 'status_map' : '0:OK', + # Custom for variables + 'o2concentration_id' : '0', + 'o2concentration_description' : 'O2Concentration', + 'o2concentration_type' : 'VT_R4', + 'o2concentration_format' : '%0.3f', + 'o2concentration_units' : 'uM', + 'o2concentration_range_min' :'0', + 'o2concentration_range_max' : '500', + 'airsaturation_id' : '1', + 'airsaturation_description' : 'AirSaturation', + 'airsaturation_type' : 'VT_R4', + 'airsaturation_format' : '%0.3f', + 'airsaturation_units' : '%', + 'airsaturation_range_min' :'0', + 'airsaturation_range_max' : '150', + 'temperature_id' : '2', + 'temperature_description' : 'Temperature', + 'temperature_type' : 'VT_R4', + 'temperature_format' : '%0.3f', + 'temperature_units' : 'Deg.C', + 'temperature_range_min' :'0', + 'temperature_range_max' : '40', + 'calphase_id' : '3', + 'calphase_description' : 'CalPhase', + 'calphase_type' : 'VT_R4', + 'calphase_format' : '%0.3f', + 'calphase_units' : 'Deg', + 'calphase_range_min' :'0', + 'calphase_range_max' : '90', + 'tcphase_id' : '4', + 'tcphase_description' : 'TCPhase', + 'tcphase_type' : 'VT_R4', + 'tcphase_format' : '%0.3f', + 'tcphase_units' : 'Deg', + 'tcphase_range_min' :'0', + 'tcphase_range_max' : '90', + 'c1rph_id' : '5', + 'c1rph_description' : 'C1RPh', + 'c1rph_type' : 'VT_R4', + 'c1rph_format' : '%0.3f', + 'c1rph_units' : 'Deg', + 'c1rph_range_min' :'0', + 'c1rph_range_max' : '90', + 'c2rph_id' : '6', + 'c2rph_description' : 'C2RPh', + 'c2rph_type' : 'VT_R4', + 'c2rph_format' : '%0.3f', + 'c2rph_units' : 'Deg', + 'c2rph_range_min' :'0', + 'c2rph_range_max' : '90', + 'c1amp_id' : '7', + 'c1amp_description' : 'C1Amp', + 'c1amp_type' : 'VT_R4', + 'c1amp_format' : '%0.1f', + 'c1amp_units' : 'mV', + 'c1amp_range_min' :'0', + 'c1amp_range_max' : '3300', + 'c2amp_id' : '8', + 'c2amp_description' : 'C2Amp', + 'c2amp_type' : 'VT_R4', + 'c2amp_format' : '%0.1f', + 'c2amp_units' : 'mV', + 'c2amp_range_min' :'0', + 'c2amp_range_max' : '3300', + 'rawtemp_id' : '9', + 'rawtemp_description' : 'RawTemp', + 'rawtemp_type' : 'VT_R4', + 'rawtemp_format' : '%0.1f', + 'rawtemp_units' : 'mV', + 'rawtemp_range_min' :'0', + 'rawtemp_range_max' : '1500', + }, + 'optode_175' : { 'id' : 'optode_175', + # Used by raw2proc + 'raw_dir' : '/seacoos/data/nccoos/level0/' + 'largo/mini_andi/xml', + 'raw_file_glob' : '*.xml', + 'proc_dir' : '/seacoos/data/nccoos/level1/' + 'largo/mini_andi/nc', + 'process_module' : 'proc_mini_andi_optode', + 'utc_offset' : 0, # hours offset to utc + 'location' : 'Florida Keys National Marine Sanctuary', + 'lat' : 24.9500, # degrees true (-) south, (+) north + 'lon' : -80.4536, # degrees true (-) west, (+) east + 'mvar' : -5.6333, # degrees (-) west, (+) east + 'elevation' : -20.0, # meters + # Custom + 'id_number' : '4330F-175', + 'serial_number' : '175', + 'product_number' : '4330F', + 'product_name' : 'Optode Sensor', + 'description' : 'Optode Sensor 4330F#175', + 'adr' : '354560', + 'protocol_version' : '1', + 'vertical_position' : '0', + 'status_map' : '0:OK', + # Custom for variables + 'o2concentration_id' : '0', + 'o2concentration_description' : 'O2Concentration', + 'o2concentration_type' : 'VT_R4', + 'o2concentration_format' : '%0.3f', + 'o2concentration_units' : 'uM', + 'o2concentration_range_min' :'0', + 'o2concentration_range_max' : '500', + 'airsaturation_id' : '1', + 'airsaturation_description' : 'AirSaturation', + 'airsaturation_type' : 'VT_R4', + 'airsaturation_format' : '%0.3f', + 'airsaturation_units' : '%', + 'airsaturation_range_min' :'0', + 'airsaturation_range_max' : '150', + 'temperature_id' : '2', + 'temperature_description' : 'Temperature', + 'temperature_type' : 'VT_R4', + 'temperature_format' : '%0.3f', + 'temperature_units' : 'Deg.C', + 'temperature_range_min' :'0', + 'temperature_range_max' : '40', + 'calphase_id' : '3', + 'calphase_description' : 'CalPhase', + 'calphase_type' : 'VT_R4', + 'calphase_format' : '%0.3f', + 'calphase_units' : 'Deg', + 'calphase_range_min' :'0', + 'calphase_range_max' : '90', + 'tcphase_id' : '4', + 'tcphase_description' : 'TCPhase', + 'tcphase_type' : 'VT_R4', + 'tcphase_format' : '%0.3f', + 'tcphase_units' : 'Deg', + 'tcphase_range_min' :'0', + 'tcphase_range_max' : '90', + 'c1rph_id' : '5', + 'c1rph_description' : 'C1RPh', + 'c1rph_type' : 'VT_R4', + 'c1rph_format' : '%0.3f', + 'c1rph_units' : 'Deg', + 'c1rph_range_min' :'0', + 'c1rph_range_max' : '90', + 'c2rph_id' : '6', + 'c2rph_description' : 'C2RPh', + 'c2rph_type' : 'VT_R4', + 'c2rph_format' : '%0.3f', + 'c2rph_units' : 'Deg', + 'c2rph_range_min' :'0', + 'c2rph_range_max' : '90', + 'c1amp_id' : '7', + 'c1amp_description' : 'C1Amp', + 'c1amp_type' : 'VT_R4', + 'c1amp_format' : '%0.1f', + 'c1amp_units' : 'mV', + 'c1amp_range_min' :'0', + 'c1amp_range_max' : '3300', + 'c2amp_id' : '8', + 'c2amp_description' : 'C2Amp', + 'c2amp_type' : 'VT_R4', + 'c2amp_format' : '%0.1f', + 'c2amp_units' : 'mV', + 'c2amp_range_min' :'0', + 'c2amp_range_max' : '3300', + 'rawtemp_id' : '9', + 'rawtemp_description' : 'RawTemp', + 'rawtemp_type' : 'VT_R4', + 'rawtemp_format' : '%0.1f', + 'rawtemp_units' : 'mV', + 'rawtemp_range_min' :'0', + 'rawtemp_range_max' : '1500', + }, + 'optode_176' : { 'id' : 'optode_176', + # Used by raw2proc + 'raw_dir' : '/seacoos/data/nccoos/level0/' + 'largo/mini_andi/xml', + 'raw_file_glob' : '*.xml', + 'proc_dir' : '/seacoos/data/nccoos/level1/' + 'largo/mini_andi/nc', + 'process_module' : 'proc_mini_andi_optode', + 'utc_offset' : 0, # hours offset to utc + 'location' : 'Florida Keys National Marine Sanctuary', + 'lat' : 24.9500, # degrees true (-) south, (+) north + 'lon' : -80.4536, # degrees true (-) west, (+) east + 'mvar' : -5.6333, # degrees (-) west, (+) east + 'elevation' : -20.0, # meters + # Custom + 'id_number' : '4330F-176', + 'serial_number' : '176', + 'product_number' : '4330F', + 'product_name' : 'Optode Sensor', + 'description' : 'Optode Sensor 4330F#176', + 'adr' : '354816', + 'protocol_version' : '1', + 'vertical_position' : '0', + 'status_map' : '0:OK', + # Custom for variables + 'o2concentration_id' : '0', + 'o2concentration_description' : 'O2Concentration', + 'o2concentration_type' : 'VT_R4', + 'o2concentration_format' : '%0.3f', + 'o2concentration_units' : 'uM', + 'o2concentration_range_min' :'0', + 'o2concentration_range_max' : '500', + 'airsaturation_id' : '1', + 'airsaturation_description' : 'AirSaturation', + 'airsaturation_type' : 'VT_R4', + 'airsaturation_format' : '%0.3f', + 'airsaturation_units' : '%', + 'airsaturation_range_min' :'0', + 'airsaturation_range_max' : '150', + 'temperature_id' : '2', + 'temperature_description' : 'Temperature', + 'temperature_type' : 'VT_R4', + 'temperature_format' : '%0.3f', + 'temperature_units' : 'Deg.C', + 'temperature_range_min' :'0', + 'temperature_range_max' : '40', + 'calphase_id' : '3', + 'calphase_description' : 'CalPhase', + 'calphase_type' : 'VT_R4', + 'calphase_format' : '%0.3f', + 'calphase_units' : 'Deg', + 'calphase_range_min' :'0', + 'calphase_range_max' : '90', + 'tcphase_id' : '4', + 'tcphase_description' : 'TCPhase', + 'tcphase_type' : 'VT_R4', + 'tcphase_format' : '%0.3f', + 'tcphase_units' : 'Deg', + 'tcphase_range_min' :'0', + 'tcphase_range_max' : '90', + 'c1rph_id' : '5', + 'c1rph_description' : 'C1RPh', + 'c1rph_type' : 'VT_R4', + 'c1rph_format' : '%0.3f', + 'c1rph_units' : 'Deg', + 'c1rph_range_min' :'0', + 'c1rph_range_max' : '90', + 'c2rph_id' : '6', + 'c2rph_description' : 'C2RPh', + 'c2rph_type' : 'VT_R4', + 'c2rph_format' : '%0.3f', + 'c2rph_units' : 'Deg', + 'c2rph_range_min' :'0', + 'c2rph_range_max' : '90', + 'c1amp_id' : '7', + 'c1amp_description' : 'C1Amp', + 'c1amp_type' : 'VT_R4', + 'c1amp_format' : '%0.1f', + 'c1amp_units' : 'mV', + 'c1amp_range_min' :'0', + 'c1amp_range_max' : '3300', + 'c2amp_id' : '8', + 'c2amp_description' : 'C2Amp', + 'c2amp_type' : 'VT_R4', + 'c2amp_format' : '%0.1f', + 'c2amp_units' : 'mV', + 'c2amp_range_min' :'0', + 'c2amp_range_max' : '3300', + 'rawtemp_id' : '9', + 'rawtemp_description' : 'RawTemp', + 'rawtemp_type' : 'VT_R4', + 'rawtemp_format' : '%0.1f', + 'rawtemp_units' : 'mV', + 'rawtemp_range_min' :'0', + 'rawtemp_range_max' : '1500', + }, + 'optode_178' : { 'id' : 'optode_178', + # Used by raw2proc + 'raw_dir' : '/seacoos/data/nccoos/level0/' + 'largo/mini_andi/xml', + 'raw_file_glob' : '*.xml', + 'proc_dir' : '/seacoos/data/nccoos/level1/' + 'largo/mini_andi/nc', + 'process_module' : 'proc_mini_andi_optode', + 'utc_offset' : 0, # hours offset to utc + 'location' : 'Florida Keys National Marine Sanctuary', + 'lat' : 24.9500, # degrees true (-) south, (+) north + 'lon' : -80.4536, # degrees true (-) west, (+) east + 'mvar' : -5.6333, # degrees (-) west, (+) east + 'elevation' : -20.0, # meters + # Custom + 'id_number' : '4330F-178', + 'serial_number' : '178', + 'product_number' : '4330F', + 'product_name' : 'Optode Sensor', + 'description' : 'Optode Sensor 4330F#178', + 'adr' : '355328', + 'protocol_version' : '1', + 'vertical_position' : '0', + 'status_map' : '0:OK', + # Custom for variables + 'o2concentration_id' : '0', + 'o2concentration_description' : 'O2Concentration', + 'o2concentration_type' : 'VT_R4', + 'o2concentration_format' : '%0.3f', + 'o2concentration_units' : 'uM', + 'o2concentration_range_min' :'0', + 'o2concentration_range_max' : '500', + 'airsaturation_id' : '1', + 'airsaturation_description' : 'AirSaturation', + 'airsaturation_type' : 'VT_R4', + 'airsaturation_format' : '%0.3f', + 'airsaturation_units' : '%', + 'airsaturation_range_min' :'0', + 'airsaturation_range_max' : '150', + 'temperature_id' : '2', + 'temperature_description' : 'Temperature', + 'temperature_type' : 'VT_R4', + 'temperature_format' : '%0.3f', + 'temperature_units' : 'Deg.C', + 'temperature_range_min' :'0', + 'temperature_range_max' : '40', + 'calphase_id' : '3', + 'calphase_description' : 'CalPhase', + 'calphase_type' : 'VT_R4', + 'calphase_format' : '%0.3f', + 'calphase_units' : 'Deg', + 'calphase_range_min' :'0', + 'calphase_range_max' : '90', + 'tcphase_id' : '4', + 'tcphase_description' : 'TCPhase', + 'tcphase_type' : 'VT_R4', + 'tcphase_format' : '%0.3f', + 'tcphase_units' : 'Deg', + 'tcphase_range_min' :'0', + 'tcphase_range_max' : '90', + 'c1rph_id' : '5', + 'c1rph_description' : 'C1RPh', + 'c1rph_type' : 'VT_R4', + 'c1rph_format' : '%0.3f', + 'c1rph_units' : 'Deg', + 'c1rph_range_min' :'0', + 'c1rph_range_max' : '90', + 'c2rph_id' : '6', + 'c2rph_description' : 'C2RPh', + 'c2rph_type' : 'VT_R4', + 'c2rph_format' : '%0.3f', + 'c2rph_units' : 'Deg', + 'c2rph_range_min' :'0', + 'c2rph_range_max' : '90', + 'c1amp_id' : '7', + 'c1amp_description' : 'C1Amp', + 'c1amp_type' : 'VT_R4', + 'c1amp_format' : '%0.1f', + 'c1amp_units' : 'mV', + 'c1amp_range_min' :'0', + 'c1amp_range_max' : '3300', + 'c2amp_id' : '8', + 'c2amp_description' : 'C2Amp', + 'c2amp_type' : 'VT_R4', + 'c2amp_format' : '%0.1f', + 'c2amp_units' : 'mV', + 'c2amp_range_min' :'0', + 'c2amp_range_max' : '3300', + 'rawtemp_id' : '9', + 'rawtemp_description' : 'RawTemp', + 'rawtemp_type' : 'VT_R4', + 'rawtemp_format' : '%0.1f', + 'rawtemp_units' : 'mV', + 'rawtemp_range_min' :'0', + 'rawtemp_range_max' : '1500', + }, + 'optode_179' : { 'id' : 'optode_179', + # Used by raw2proc + 'raw_dir' : '/seacoos/data/nccoos/level0/' + 'largo/mini_andi/xml', + 'raw_file_glob' : '*.xml', + 'proc_dir' : '/seacoos/data/nccoos/level1/' + 'largo/mini_andi/nc', + 'process_module' : 'proc_mini_andi_optode', + 'utc_offset' : 0, # hours offset to utc + 'location' : 'Florida Keys National Marine Sanctuary', + 'lat' : 24.9500, # degrees true (-) south, (+) north + 'lon' : -80.4536, # degrees true (-) west, (+) east + 'mvar' : -5.6333, # degrees (-) west, (+) east + 'elevation' : -20.0, # meters + # Custom + 'id_number' : '4330F-179', + 'serial_number' : '179', + 'product_number' : '4330F', + 'product_name' : 'Optode Sensor', + 'description' : 'Optode Sensor 4330F#179', + 'adr' : '355584', + 'protocol_version' : '1', + 'vertical_position' : '0', + 'status_map' : '0:OK', + # Custom for variables + 'o2concentration_id' : '0', + 'o2concentration_description' : 'O2Concentration', + 'o2concentration_type' : 'VT_R4', + 'o2concentration_format' : '%0.3f', + 'o2concentration_units' : 'uM', + 'o2concentration_range_min' :'0', + 'o2concentration_range_max' : '500', + 'airsaturation_id' : '1', + 'airsaturation_description' : 'AirSaturation', + 'airsaturation_type' : 'VT_R4', + 'airsaturation_format' : '%0.3f', + 'airsaturation_units' : '%', + 'airsaturation_range_min' :'0', + 'airsaturation_range_max' : '150', + 'temperature_id' : '2', + 'temperature_description' : 'Temperature', + 'temperature_type' : 'VT_R4', + 'temperature_format' : '%0.3f', + 'temperature_units' : 'Deg.C', + 'temperature_range_min' :'0', + 'temperature_range_max' : '40', + 'calphase_id' : '3', + 'calphase_description' : 'CalPhase', + 'calphase_type' : 'VT_R4', + 'calphase_format' : '%0.3f', + 'calphase_units' : 'Deg', + 'calphase_range_min' :'0', + 'calphase_range_max' : '90', + 'tcphase_id' : '4', + 'tcphase_description' : 'TCPhase', + 'tcphase_type' : 'VT_R4', + 'tcphase_format' : '%0.3f', + 'tcphase_units' : 'Deg', + 'tcphase_range_min' :'0', + 'tcphase_range_max' : '90', + 'c1rph_id' : '5', + 'c1rph_description' : 'C1RPh', + 'c1rph_type' : 'VT_R4', + 'c1rph_format' : '%0.3f', + 'c1rph_units' : 'Deg', + 'c1rph_range_min' :'0', + 'c1rph_range_max' : '90', + 'c2rph_id' : '6', + 'c2rph_description' : 'C2RPh', + 'c2rph_type' : 'VT_R4', + 'c2rph_format' : '%0.3f', + 'c2rph_units' : 'Deg', + 'c2rph_range_min' :'0', + 'c2rph_range_max' : '90', + 'c1amp_id' : '7', + 'c1amp_description' : 'C1Amp', + 'c1amp_type' : 'VT_R4', + 'c1amp_format' : '%0.1f', + 'c1amp_units' : 'mV', + 'c1amp_range_min' :'0', + 'c1amp_range_max' : '3300', + 'c2amp_id' : '8', + 'c2amp_description' : 'C2Amp', + 'c2amp_type' : 'VT_R4', + 'c2amp_format' : '%0.1f', + 'c2amp_units' : 'mV', + 'c2amp_range_min' :'0', + 'c2amp_range_max' : '3300', + 'rawtemp_id' : '9', + 'rawtemp_description' : 'RawTemp', + 'rawtemp_type' : 'VT_R4', + 'rawtemp_format' : '%0.1f', + 'rawtemp_units' : 'mV', + 'rawtemp_range_min' :'0', + 'rawtemp_range_max' : '1500', + }, + 'conductivity' : { 'id' : 'conductivity', + # Used by raw2proc + 'raw_dir' : '/seacoos/data/nccoos/level0/' + 'largo/mini_andi/xml', + 'raw_file_glob' : '*.xml', + 'proc_dir' : '/seacoos/data/nccoos/level1/' + 'largo/mini_andi/nc', + 'process_module' : 'proc_mini_andi_conductivity', + 'utc_offset' : 0, # hours offset to utc + 'location' : 'Florida Keys National Marine Sanctuary', + 'lat' : 24.9500, # degrees true (-) south, (+) north + 'lon' : -80.4536, # degrees true (-) west, (+) east + 'mvar' : -5.6333, # degrees (-) west, (+) east + 'elevation' : -20.0, # meters + # Custom + 'id_number' : '4319-217', + 'serial_number' : '217', + 'product_number' : '4319', + 'product_name' : 'Conductivity Sensor', + 'description' : 'Conductivity #217', + 'adr' : '362752', + 'protocol_version' : '1', + 'vertical_position' : '0', + 'status_map' : '0:OK', + # Custom for variables + 'conductivity_id' : '0', + 'conductivity_description' : 'Conductivity', + 'conductivity_type' : 'VT_R4', + 'conductivity_format' : '%0.3f', + 'conductivity_units' : 'mS/cm', + 'conductivity_range_min' :'0', + 'conductivity_range_max' : '75', + 'temperature_id' : '1', + 'temperature_description' : 'Temperature', + 'temperature_type' : 'VT_R4', + 'temperature_format' : '%0.3f', + 'temperature_units' : 'Deg.C', + 'temperature_range_min' :'-5', + 'temperature_range_max' : '35', + }, + 'pressure' : { 'id' : 'pressure', + # Used by raw2proc + 'raw_dir' : '/seacoos/data/nccoos/level0/' + 'largo/mini_andi/xml', + 'raw_file_glob' : '*.xml', + 'proc_dir' : '/seacoos/data/nccoos/level1/' + 'largo/mini_andi/nc', + 'process_module' : 'proc_mini_andi_pressure', + 'utc_offset' : 0, # hours offset to utc + 'location' : 'Florida Keys National Marine Sanctuary', + 'lat' : 24.9500, # degrees true (-) south, (+) north + 'lon' : -80.4536, # degrees true (-) west, (+) east + 'mvar' : -5.6333, # degrees (-) west, (+) east + 'elevation' : -20.0, # meters + # Custom + 'id_number' : '4117D-235', + 'serial_number' : '235', + 'product_number' : '4117D', + 'product_name' : 'Pressure Sensor', + 'description' : 'Pressure #235', + 'adr' : '366848', + 'protocol_version' : '1', + 'vertical_position' : '0', + 'status_map' : '0:OK', + # Custom for variables + 'pressure_id' : '0', + 'pressure_description' : 'Pressure', + 'pressure_type' : 'VT_R4', + 'pressure_format' : '%0.3f', + 'pressure_units' : 'kPa', + 'pressure_range_min' :'0', + 'pressure_range_max' : '1000', + 'temperature_id' : '1', + 'temperature_description' : 'Temperature', + 'temperature_type' : 'VT_R4', + 'temperature_format' : '%0.3f', + 'temperature_units' : 'DegC', + 'temperature_range_min' :'-5', + 'temperature_range_max' : '35', + }, + 'current' : { 'id' : 'current', + # Used by raw2proc + 'raw_dir' : '/seacoos/data/nccoos/level0/' + 'largo/mini_andi/xml', + 'raw_file_glob' : '*.xml', + 'proc_dir' : '/seacoos/data/nccoos/level1/' + 'largo/mini_andi/nc', + 'process_module' : 'proc_mini_andi_current', + 'utc_offset' : 0, # hours offset to utc + 'location' : 'Florida Keys National Marine Sanctuary', + 'lat' : 24.9500, # degrees true (-) south, (+) north + 'lon' : -80.4536, # degrees true (-) west, (+) east + 'mvar' : -5.6333, # degrees (-) west, (+) east + 'elevation' : -20.0, # meters + # Custom + 'id_number' : '4520-275', + 'serial_number' : '275', + 'product_number' : '4520', + 'product_name' : 'Doppler Current Sensor', + 'description' : 'DCS #275', + 'adr' : '591360', + 'protocol_version' : '1', + 'vertical_position' : '0', + 'status_map' : '0:OK', + # Custom for variables + 'abs_speed_id' : '0', + 'abs_speed_description' : 'Abs Speed', + 'abs_speed_type' : 'VT_R4', + 'abs_speed_format' : '%0.3f', + 'abs_speed_units' : 'cm/s', + 'abs_speed_range_min' :'0', + 'abs_speed_range_max' : '300', + 'direction_id' : '1', + 'direction_description' : 'Direction', + 'direction_type' : 'VT_R4', + 'direction_format' : '%0.3f', + 'direction_units' : 'Deg.M', + 'direction_range_min' :'0', + 'direction_range_max' : '360', + 'v_id' : '2', + 'v_description' : 'North', + 'v_type' : 'VT_R4', + 'v_format' : '%0.3f', + 'v_units' : 'cm/s', + 'v_range_min' :'-300', + 'v_range_max' : '300', + 'u_id' : '3', + 'u_description' : 'East', + 'u_type' : 'VT_R4', + 'u_format' : '%0.3f', + 'u_units' : 'cm/s', + 'u_range_min' :'-300', + 'u_range_max' : '300', + 'heading_id' : '4', + 'heading_description' : 'Heading', + 'heading_type' : 'VT_R4', + 'heading_format' : '%0.3f', + 'heading_units' : 'Deg.M', + 'heading_range_min' :'0', + 'heading_range_max' : '360', + 'tiltx_id' : '5', + 'tiltx_description' : 'Tilt X', + 'tiltx_type' : 'VT_R4', + 'tiltx_format' : '%0.3f', + 'tiltx_units' : 'Deg', + 'tiltx_range_min' :'-45', + 'tiltx_range_max' : '45', + 'tilty_id' : '6', + 'tilty_description' : 'Tilt Y', + 'tilty_type' : 'VT_R4', + 'tilty_format' : '%0.3f', + 'tilty_units' : 'Deg', + 'tilty_range_min' :'-45', + 'tilty_range_max' : '45', + 'std_speed_id' : '7', + 'std_speed_description' : 'SP Std', + 'std_speed_type' : 'VT_R4', + 'std_speed_format' : '%0.3f', + 'std_speed_units' : 'cm/s', + 'std_speed_range_min' :'0', + 'std_speed_range_max' : '20', + 'strength_id' : '8', + 'strength_description' : 'Strength', + 'strength_type' : 'VT_R4', + 'strength_format' : '%0.3f', + 'strength_units' : 'dB', + 'strength_range_min' : '-80', + 'strength_range_max' : '0', + 'pings_id' : '9', + 'pings_description' : 'Ping Count', + 'pings_type' : 'VT_12', + 'pings_format' : '%d', + 'pings_units' : '', + 'pings_range_min' :'', + 'pings_range_max' : '', + }, +} Index: raw2proc/trunk/raw2proc/proc_mini_andi_conductivity.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/proc_mini_andi_conductivity.py (revision 375) @@ -1,0 +1,386 @@ +""" +Parse data and assert what data creates and updates monthly NetCDF files. + +Spongenet mini_andi conductivity parameters sponge data. +""" + +import math +import numpy as n +import pycdf +import datetime +import procutil +from spongenet.parse import Data + +nowDt = datetime.datetime.utcnow().replace(microsecond=0) + +def parser(platform_info, sensor_info, lines): + """ + Parse and assign sponge data from XML file. + """ + + _data = Data(''.join(lines)) + + # Each Device tag represents a time sample. + num_samples = len(_data.devices) + + data = { + 'dt' : n.array(n.ones((num_samples,)) * n.nan, dtype=object), + 'time' : n.array(n.ones((num_samples,)) * n.nan, dtype=long), + 'pdt' : n.array(n.ones((num_samples,)) * n.nan, dtype=object), + 'ptime' : n.array(n.ones((num_samples,)) * n.nan, dtype=long), + 'ds' : n.array(n.ones((num_samples,)) * n.nan, dtype=object), + 'session' : n.array(n.ones((num_samples,)) * n.nan, dtype=long), + 'pds' : n.array(n.ones((num_samples,)) * n.nan, dtype=object), + 'psession' : n.array(n.ones((num_samples,)) * n.nan, dtype=long), + 'record' : n.array(n.ones((num_samples,)) * n.nan, dtype=int), + 'status' : n.array(n.ones((num_samples,)) * n.nan, dtype=int), + 'pstatus' : n.array(n.ones((num_samples,)) * n.nan, dtype=int), + 'conductivity' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + 'temperature' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + } + + for (sample_index, sample) in enumerate(_data.devices): + # sample time at the platform + dt = {'month' : int(sample['time'][5:7]), + 'day' : int(sample['time'][8:10]), + 'year' : int(sample['time'][0:4]), + 'hour' : int(sample['time'][11:13]), + 'min' : int(sample['time'][14:16]), + 'sec' : int(sample['time'][17:19]), + } + dt = '%(month)02d-%(day)02d-%(year)04d %(hour)02d:%(min)02d:%(sec)02d' \ + % dt + dt = procutil.scanf_datetime(dt, fmt='%m-%d-%Y %H:%M:%S') + if sensor_info['utc_offset']: + dt = dt + datetime.timedelta(hours=sensor_info['utc_offset']) + data['dt'][sample_index] = dt + data['time'][sample_index] = procutil.dt2es(dt) + + # sample time at the package + package_dt = {'month' : int(sample['data_time'][5:7]), + 'day' : int(sample['data_time'][8:10]), + 'year' : int(sample['data_time'][0:4]), + 'hour' : int(sample['data_time'][11:13]), + 'min' : int(sample['data_time'][14:16]), + 'sec' : int(sample['data_time'][17:19]), + } + package_dt = ('%(month)02d-%(day)02d-%(year)04d ' + + '%(hour)02d:%(min)02d:%(sec)02d') \ + % package_dt + package_dt = procutil.scanf_datetime(package_dt, fmt='%m-%d-%Y %H:%M:%S') + if sensor_info['utc_offset']: + package_dt = package_dt + \ + datetime.timedelta(hours=sensor_info['utc_offset']) + data['pdt'][sample_index] = package_dt + data['ptime'][sample_index] = procutil.dt2es(package_dt) + + # platform session time + ds = {'month' : int(sample['sessionid'][14:16]), + 'day' : int(sample['sessionid'][17:19]), + 'year' : int(sample['sessionid'][9:13]), + 'hour' : int(sample['sessionid'][20:22]), + 'min' : int(sample['sessionid'][23:25]), + 'sec' : int(sample['sessionid'][26:28]), + } + ds = '%(month)02d-%(day)02d-%(year)04d %(hour)02d:%(min)02d:%(sec)02d' \ + % ds + ds = procutil.scanf_datetime(ds, fmt='%m-%d-%Y %H:%M:%S') + if sensor_info['utc_offset']: + ds = ds + datetime.timedelta(hours=sensor_info['utc_offset']) + data['ds'][sample_index] = ds + data['session'][sample_index] = procutil.dt2es(ds) + + # package session time + package_ds = {'month' : int(sample['data_sessionid'][5:7]), + 'day' : int(sample['data_sessionid'][8:10]), + 'year' : int(sample['data_sessionid'][0:4]), + 'hour' : int(sample['data_sessionid'][11:13]), + 'min' : int(sample['data_sessionid'][14:16]), + 'sec' : int(sample['data_sessionid'][17:19]), + } + package_ds = ('%(month)02d-%(day)02d-%(year)04d ' + + '%(hour)02d:%(min)02d:%(sec)02d') \ + % package_ds + package_ds = procutil.scanf_datetime(package_ds, fmt='%m-%d-%Y %H:%M:%S') + if sensor_info['utc_offset']: + package_ds = package_ds + \ + datetime.timedelta(hours=sensor_info['utc_offset']) + data['pds'][sample_index] = package_ds + data['psession'][sample_index] = procutil.dt2es(package_ds) + + # platform variables + try: + data['record'][sample_index] = int(sample["recordnumber"]) + except KeyError: + pass + + try: + data['status'][sample_index] = int(sample["status"]. + partition(":")[0]) + except (KeyError, AttributeError, ): + pass + + # package variables + try: + data['pstatus'][sample_index] = int(sample.sensors + [sensor_info["id_number"]] + ["status"]. + partition(":")[0]) + except (KeyError, AttributeError, ): + pass + + try: + data['conductivity'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["conductivity_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + try: + data['temperature'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["temperature_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + return data + +def creator(platform_info, sensor_info, data): + # + # + title_str = sensor_info['description']+' at '+ sensor_info['location'] + global_atts = { + # Required + 'title' : title_str, + 'institution' : platform_info['institution'], + 'institution_url' : platform_info['institution_url'], + 'institution_dods_url' : platform_info['institution_dods_url'], + 'contact' : platform_info['contact'], + 'Conventions' : platform_info['conventions'], + # Required by Scout + 'format_category_code' : platform_info['format_category_code'], + 'institution_code' : platform_info['institution_code'], + 'platform_code' : platform_info['id'], + 'package_code' : sensor_info['id'], + # Required by Version tracking + 'format' : platform_info['format'], + 'seacoos_rt_version' : platform_info['seacoos_rt_version'], + # Recommended + '_FillValue' : n.nan, + 'missing_value' : n.nan, + 'source' : platform_info['source'], + 'references' : platform_info['references'], + 'metadata_url' : platform_info['metadata_url'], + 'history' : 'raw2proc using ' + sensor_info['process_module'], + 'comment' : 'File created using pycdf ' + \ + pycdf.pycdfVersion() + ' and ' + \ + pycdf.pycdfArrayPkg() + ' ' + \ + n.__version__, + 'project' : platform_info['project'], + 'project_url' : platform_info['project_url'], + # timeframe of data contained in file yyyy-mm-dd HH:MM:SS + # first date in monthly file + 'start_date' : data['dt'][0].strftime("%Y-%m-%d %H:%M:%S"), + # last date in monthly file + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + 'creation_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + 'modification_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + 'process_level' : 'level1', + # Custom + 'id_number' : platform_info['id_number'], + 'description' : platform_info['description'], + 'serial_number' : platform_info['serial_number'], + 'product_number' : platform_info['product_number'], + 'product_name' : platform_info['product_name'], + 'type' : platform_info['type'], + 'protocol_version' : platform_info['protocol_version'], + 'xmlns' : platform_info['xmlns'], + 'location' : platform_info['location'], + 'vertical_position': platform_info['vertical_position'], + 'owner' : platform_info['owner'], + 'package_id_number' : sensor_info['id_number'], + 'package_description' : sensor_info['description'], + 'package_serial_number' : sensor_info['serial_number'], + 'package_product_number' : sensor_info['product_number'], + 'package_product_name' : sensor_info['product_name'], + 'package_adr' : sensor_info['adr'], + 'package_protocol_version' : sensor_info['protocol_version'], + 'package_vertical_position' : sensor_info['vertical_position'], + } + + var_atts = { + # coordinate variables + 'time' : {'short_name': 'time', + 'long_name': 'Time', + 'standard_name': 'time', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + 'axis': 'T', + }, + 'lat' : {'short_name': 'lat', + 'long_name': 'Latitude', + 'standard_name': 'latitude', + 'reference':'geographic coordinates', + 'units': 'degrees_north', + 'valid_range':(-90.,90.), + 'axis': 'Y', + }, + 'lon' : {'short_name': 'lon', + 'long_name': 'Longtitude', + 'standard_name': 'longtitude', + 'reference':'geographic coordinates', + 'units': 'degrees_east', + 'valid_range':(-180.,180.), + 'axis': 'Y', + }, + 'z' : {'short_name': 'z', + 'long_name': 'Height', + 'standard_name': 'height', + 'reference':'zero at sea-surface', + 'positive' : 'up', + 'units': 'meters', + 'axis': 'Z', + }, + # data variables + 'ptime' : {'short_name': 'ptime', + 'long_name': 'Package Time', + 'standard_name': 'none', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + }, + 'session' : {'short_name': 'session', + 'long_name': 'Session ID', + 'standard_name': 'none', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + }, + 'psession' : {'short_name': 'ptime', + 'long_name': 'Package Session ID', + 'standard_name': 'none', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + }, + 'record' : {'short_name': 'record', + 'long_name': 'Record Number', + 'standard_name': 'none', + 'units' : 'none', + }, + 'status': {'short_name' : 'status', + 'long_name': 'Platform Status Code', + 'standard_name': 'none', + 'units' : 'none', + 'value_map' : platform_info['status_map'], + }, + 'pstatus': {'short_name' : 'pstatus', + 'long_name': 'Package Status Code', + 'standard_name': 'none', + 'units' : 'none', + 'value_map' : sensor_info['status_map'], + }, + 'conductivity' : {'short_name' : 'conductivity', + 'long_name': sensor_info['conductivity_description'], + 'standard_name': 'conductivity', + 'units': 'none', + 'id_number' : sensor_info['conductivity_id'], + 'type' : sensor_info['conductivity_type'], + 'format' : sensor_info['conductivity_format'], + 'non_standard_units' : sensor_info['conductivity_units'], + 'range_min' : sensor_info['conductivity_range_min'], + 'range_max' : sensor_info['conductivity_range_max'], + }, + 'temperature' : {'short_name' : 'temperature', + 'long_name': sensor_info['temperature_description'], + 'standard_name': 'water_temperature', + 'units': 'celsius', + 'id_number' : sensor_info['temperature_id'], + 'type' : sensor_info['temperature_type'], + 'format' : sensor_info['temperature_format'], + 'non_standard_units' : sensor_info['temperature_units'], + 'range_min' : sensor_info['temperature_range_min'], + 'range_max' : sensor_info['temperature_range_max'], + }, + } + + # dimension names use tuple so order of initialization is maintained + dim_inits = ( + ('ntime', pycdf.NC.UNLIMITED), + ('nlat', 1), + ('nlon', 1), + ('nz', 1), + ) + + # using tuple of tuples so order of initialization is maintained + # using dict for attributes order of init not important + # use dimension names not values + # (varName, varType, (dimName1, [dimName2], ...)) + var_inits = ( + # coordinate variables + ('time', pycdf.NC.INT, ('ntime',)), + ('lat', pycdf.NC.FLOAT, ('nlat',)), + ('lon', pycdf.NC.FLOAT, ('nlon',)), + ('z', pycdf.NC.FLOAT, ('nz',)), + # data variables + ('ptime', pycdf.NC.INT, ('ntime',)), + ('session', pycdf.NC.INT, ('ntime',)), + ('psession', pycdf.NC.INT, ('ntime',)), + ('record', pycdf.NC.INT, ('ntime',)), + ('status', pycdf.NC.INT, ('ntime',)), + ('pstatus', pycdf.NC.INT, ('ntime',)), + ('conductivity', pycdf.NC.FLOAT, ('ntime',)), + ('temperature', pycdf.NC.FLOAT, ('ntime',)), + ) + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # var data + var_data = ( + ('time', data['time'][i]), + ('lat', sensor_info['lat']), + ('lon', sensor_info['lat']), + ('z', sensor_info['elevation']), + ('ptime', data['ptime'][i]), + ('session', data['session'][i]), + ('psession', data['psession'][i]), + ('record', data['record'][i]), + ('status', data['status'][i]), + ('pstatus', data['pstatus'][i]), + ('conductivity', data['conductivity'][i]), + ('temperature', data['temperature'][i]), + ) + + return (global_atts, var_atts, dim_inits, var_inits, var_data) + +def updater(platform_info, sensor_info, data): + # + global_atts = { + # update times of data contained in file (yyyy-mm-dd HH:MM:SS) + # last date in monthly file + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + # + 'modification_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + } + + # data variables + # update any variable attributes like range, min, max + var_atts = {} + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # data + var_data = ( + ('time', data['time'][i]), + ('ptime', data['ptime'][i]), + ('session', data['session'][i]), + ('psession', data['psession'][i]), + ('record', data['record'][i]), + ('status', data['status'][i]), + ('pstatus', data['pstatus'][i]), + ('conductivity', data['conductivity'][i]), + ('temperature', data['temperature'][i]), + ) + + return (global_atts, var_atts, var_data) Index: raw2proc/trunk/raw2proc/proc_mini_andi_current.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/proc_mini_andi_current.py (revision 375) @@ -1,0 +1,579 @@ +""" +Parse data and assert what data creates and updates monthly NetCDF files. + +Spongenet mini_andi current parameters sponge data. +""" + +import math +import numpy as n +import pycdf +import datetime +import procutil +from spongenet.parse import Data + +nowDt = datetime.datetime.utcnow().replace(microsecond=0) + +def parser(platform_info, sensor_info, lines): + """ + Parse and assign sponge data from XML file. + """ + + _data = Data(''.join(lines)) + + # Each Device tag represents a time sample. + num_samples = len(_data.devices) + + data = { + 'dt' : n.array(n.ones((num_samples,)) * n.nan, dtype=object), + 'time' : n.array(n.ones((num_samples,)) * n.nan, dtype=long), + 'pdt' : n.array(n.ones((num_samples,)) * n.nan, dtype=object), + 'ptime' : n.array(n.ones((num_samples,)) * n.nan, dtype=long), + 'ds' : n.array(n.ones((num_samples,)) * n.nan, dtype=object), + 'session' : n.array(n.ones((num_samples,)) * n.nan, dtype=long), + 'pds' : n.array(n.ones((num_samples,)) * n.nan, dtype=object), + 'psession' : n.array(n.ones((num_samples,)) * n.nan, dtype=long), + 'record' : n.array(n.ones((num_samples,)) * n.nan, dtype=int), + 'status' : n.array(n.ones((num_samples,)) * n.nan, dtype=int), + 'pstatus' : n.array(n.ones((num_samples,)) * n.nan, dtype=int), + 'abs_speed' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + 'direction' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + 'v' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + 'u' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + 'heading' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + 'tiltx' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + 'tilty' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + 'std_speed' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + 'strength' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + 'pings' : n.array(n.ones((num_samples,)) * n.nan, dtype=int), + } + + for (sample_index, sample) in enumerate(_data.devices): + # sample time at the platform + dt = {'month' : int(sample['time'][5:7]), + 'day' : int(sample['time'][8:10]), + 'year' : int(sample['time'][0:4]), + 'hour' : int(sample['time'][11:13]), + 'min' : int(sample['time'][14:16]), + 'sec' : int(sample['time'][17:19]), + } + dt = '%(month)02d-%(day)02d-%(year)04d %(hour)02d:%(min)02d:%(sec)02d' \ + % dt + dt = procutil.scanf_datetime(dt, fmt='%m-%d-%Y %H:%M:%S') + if sensor_info['utc_offset']: + dt = dt + datetime.timedelta(hours=sensor_info['utc_offset']) + data['dt'][sample_index] = dt + data['time'][sample_index] = procutil.dt2es(dt) + + # sample time at the package + package_dt = {'month' : int(sample['data_time'][5:7]), + 'day' : int(sample['data_time'][8:10]), + 'year' : int(sample['data_time'][0:4]), + 'hour' : int(sample['data_time'][11:13]), + 'min' : int(sample['data_time'][14:16]), + 'sec' : int(sample['data_time'][17:19]), + } + package_dt = ('%(month)02d-%(day)02d-%(year)04d ' + + '%(hour)02d:%(min)02d:%(sec)02d') \ + % package_dt + package_dt = procutil.scanf_datetime(package_dt, fmt='%m-%d-%Y %H:%M:%S') + if sensor_info['utc_offset']: + package_dt = package_dt + \ + datetime.timedelta(hours=sensor_info['utc_offset']) + data['pdt'][sample_index] = package_dt + data['ptime'][sample_index] = procutil.dt2es(package_dt) + + # platform session time + ds = {'month' : int(sample['sessionid'][14:16]), + 'day' : int(sample['sessionid'][17:19]), + 'year' : int(sample['sessionid'][9:13]), + 'hour' : int(sample['sessionid'][20:22]), + 'min' : int(sample['sessionid'][23:25]), + 'sec' : int(sample['sessionid'][26:28]), + } + ds = '%(month)02d-%(day)02d-%(year)04d %(hour)02d:%(min)02d:%(sec)02d' \ + % ds + ds = procutil.scanf_datetime(ds, fmt='%m-%d-%Y %H:%M:%S') + if sensor_info['utc_offset']: + ds = ds + datetime.timedelta(hours=sensor_info['utc_offset']) + data['ds'][sample_index] = ds + data['session'][sample_index] = procutil.dt2es(ds) + + # package session time + package_ds = {'month' : int(sample['data_sessionid'][5:7]), + 'day' : int(sample['data_sessionid'][8:10]), + 'year' : int(sample['data_sessionid'][0:4]), + 'hour' : int(sample['data_sessionid'][11:13]), + 'min' : int(sample['data_sessionid'][14:16]), + 'sec' : int(sample['data_sessionid'][17:19]), + } + package_ds = ('%(month)02d-%(day)02d-%(year)04d ' + + '%(hour)02d:%(min)02d:%(sec)02d') \ + % package_ds + package_ds = procutil.scanf_datetime(package_ds, fmt='%m-%d-%Y %H:%M:%S') + if sensor_info['utc_offset']: + package_ds = package_ds + \ + datetime.timedelta(hours=sensor_info['utc_offset']) + data['pds'][sample_index] = package_ds + data['psession'][sample_index] = procutil.dt2es(package_ds) + + # platform variables + try: + data['record'][sample_index] = int(sample["recordnumber"]) + except KeyError: + pass + + try: + data['status'][sample_index] = int(sample["status"]. + partition(":")[0]) + except (KeyError, AttributeError, ): + pass + + # package variables + try: + data['pstatus'][sample_index] = int(sample.sensors + [sensor_info["id_number"]] + ["status"]. + partition(":")[0]) + except (KeyError, AttributeError, ): + pass + + try: + data['abs_speed'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["abs_speed_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + try: + data['direction'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["direction_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + try: + data['v'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["v_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + try: + data['u'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["u_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + try: + data['heading'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["heading_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + try: + data['tiltx'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["tiltx_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + try: + data['tilty'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["tilty_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + try: + data['std_speed'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["std_speed_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + try: + data['strength'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["strength_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + try: + data['pings'][sample_index] = int(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["pings_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + return data + +def creator(platform_info, sensor_info, data): + # + # + title_str = sensor_info['description']+' at '+ sensor_info['location'] + global_atts = { + # Required + 'title' : title_str, + 'institution' : platform_info['institution'], + 'institution_url' : platform_info['institution_url'], + 'institution_dods_url' : platform_info['institution_dods_url'], + 'contact' : platform_info['contact'], + 'Conventions' : platform_info['conventions'], + # Required by Scout + 'format_category_code' : platform_info['format_category_code'], + 'institution_code' : platform_info['institution_code'], + 'platform_code' : platform_info['id'], + 'package_code' : sensor_info['id'], + # Required by Version tracking + 'format' : platform_info['format'], + 'seacoos_rt_version' : platform_info['seacoos_rt_version'], + # Recommended + '_FillValue' : n.nan, + 'missing_value' : n.nan, + 'source' : platform_info['source'], + 'references' : platform_info['references'], + 'metadata_url' : platform_info['metadata_url'], + 'history' : 'raw2proc using ' + sensor_info['process_module'], + 'comment' : 'File created using pycdf ' + \ + pycdf.pycdfVersion() + ' and ' + \ + pycdf.pycdfArrayPkg() + ' ' + \ + n.__version__, + 'project' : platform_info['project'], + 'project_url' : platform_info['project_url'], + # timeframe of data contained in file yyyy-mm-dd HH:MM:SS + # first date in monthly file + 'start_date' : data['dt'][0].strftime("%Y-%m-%d %H:%M:%S"), + # last date in monthly file + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + 'creation_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + 'modification_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + 'process_level' : 'level1', + # Custom + 'id_number' : platform_info['id_number'], + 'description' : platform_info['description'], + 'serial_number' : platform_info['serial_number'], + 'product_number' : platform_info['product_number'], + 'product_name' : platform_info['product_name'], + 'type' : platform_info['type'], + 'protocol_version' : platform_info['protocol_version'], + 'xmlns' : platform_info['xmlns'], + 'location' : platform_info['location'], + 'vertical_position': platform_info['vertical_position'], + 'owner' : platform_info['owner'], + 'package_id_number' : sensor_info['id_number'], + 'package_description' : sensor_info['description'], + 'package_serial_number' : sensor_info['serial_number'], + 'package_product_number' : sensor_info['product_number'], + 'package_product_name' : sensor_info['product_name'], + 'package_adr' : sensor_info['adr'], + 'package_protocol_version' : sensor_info['protocol_version'], + 'package_vertical_position' : sensor_info['vertical_position'], + } + + var_atts = { + # coordinate variables + 'time' : {'short_name': 'time', + 'long_name': 'Time', + 'standard_name': 'time', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + 'axis': 'T', + }, + 'lat' : {'short_name': 'lat', + 'long_name': 'Latitude', + 'standard_name': 'latitude', + 'reference':'geographic coordinates', + 'units': 'degrees_north', + 'valid_range':(-90.,90.), + 'axis': 'Y', + }, + 'lon' : {'short_name': 'lon', + 'long_name': 'Longtitude', + 'standard_name': 'longtitude', + 'reference':'geographic coordinates', + 'units': 'degrees_east', + 'valid_range':(-180.,180.), + 'axis': 'Y', + }, + 'z' : {'short_name': 'z', + 'long_name': 'Height', + 'standard_name': 'height', + 'reference':'zero at sea-surface', + 'positive' : 'up', + 'units': 'meters', + 'axis': 'Z', + }, + # data variables + 'ptime' : {'short_name': 'ptime', + 'long_name': 'Package Time', + 'standard_name': 'none', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + }, + 'session' : {'short_name': 'session', + 'long_name': 'Session ID', + 'standard_name': 'none', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + }, + 'psession' : {'short_name': 'ptime', + 'long_name': 'Package Session ID', + 'standard_name': 'none', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + }, + 'record' : {'short_name': 'record', + 'long_name': 'Record Number', + 'standard_name': 'none', + 'units' : 'none', + }, + 'status': {'short_name' : 'status', + 'long_name': 'Platform Status Code', + 'standard_name': 'none', + 'units' : 'none', + 'value_map' : platform_info['status_map'], + }, + 'pstatus': {'short_name' : 'pstatus', + 'long_name': 'Package Status Code', + 'standard_name': 'none', + 'units' : 'none', + 'value_map' : sensor_info['status_map'], + }, + 'abs_speed' : {'short_name' : 'c_spd', + 'long_name': sensor_info['abs_speed_description'], + 'standard_name': 'current_speed', + 'units': 'cm s-1', + 'id_number' : sensor_info['abs_speed_id'], + 'type' : sensor_info['abs_speed_type'], + 'format' : sensor_info['abs_speed_format'], + 'non_standard_units' : sensor_info['abs_speed_units'], + 'range_min' : sensor_info['abs_speed_range_min'], + 'range_max' : sensor_info['abs_speed_range_max'], + }, + 'direction' : {'short_name' : 'c_dir', + 'long_name': sensor_info['direction_description'], + 'standard_name': 'current_to_direction', + 'units': 'degrees_true', + 'id_number' : sensor_info['direction_id'], + 'type' : sensor_info['direction_type'], + 'format' : sensor_info['direction_format'], + 'non_standard_units' : sensor_info['direction_units'], + 'range_min' : sensor_info['direction_range_min'], + 'range_max' : sensor_info['direction_range_max'], + }, + 'v' : {'short_name' : 'water_v', + 'long_name': sensor_info['v_description'], + 'standard_name': 'northward_current', + 'units': 'cm s-1', + 'id_number' : sensor_info['v_id'], + 'type' : sensor_info['v_type'], + 'format' : sensor_info['v_format'], + 'non_standard_units' : sensor_info['v_units'], + 'range_min' : sensor_info['v_range_min'], + 'range_max' : sensor_info['v_range_max'], + }, + 'u' : {'short_name' : 'water_u', + 'long_name': sensor_info['u_description'], + 'standard_name': 'eastward_current', + 'units': 'cm s-1', + 'id_number' : sensor_info['u_id'], + 'type' : sensor_info['u_type'], + 'format' : sensor_info['u_format'], + 'non_standard_units' : sensor_info['u_units'], + 'range_min' : sensor_info['u_range_min'], + 'range_max' : sensor_info['u_range_max'], + }, + 'heading' : {'short_name' : 'heading', + 'long_name': sensor_info['heading_description'], + 'standard_name': 'none', + 'units': 'none', + 'id_number' : sensor_info['heading_id'], + 'type' : sensor_info['heading_type'], + 'format' : sensor_info['heading_format'], + 'non_standard_units' : sensor_info['heading_units'], + 'range_min' : sensor_info['heading_range_min'], + 'range_max' : sensor_info['heading_range_max'], + }, + 'tiltx' : {'short_name' : 'tiltx', + 'long_name': sensor_info['tiltx_description'], + 'standard_name': 'none', + 'units': 'none', + 'id_number' : sensor_info['tiltx_id'], + 'type' : sensor_info['tiltx_type'], + 'format' : sensor_info['tiltx_format'], + 'non_standard_units' : sensor_info['tiltx_units'], + 'range_min' : sensor_info['tiltx_range_min'], + 'range_max' : sensor_info['tiltx_range_max'], + }, + 'tilty' : {'short_name' : 'tilty', + 'long_name': sensor_info['tilty_description'], + 'standard_name': 'none', + 'units': 'none', + 'id_number' : sensor_info['tilty_id'], + 'type' : sensor_info['tilty_type'], + 'format' : sensor_info['tilty_format'], + 'non_standard_units' : sensor_info['tilty_units'], + 'range_min' : sensor_info['tilty_range_min'], + 'range_max' : sensor_info['tilty_range_max'], + }, + 'std_speed' : {'short_name' : 'beam_vel', + 'long_name': sensor_info['std_speed_description'], + 'standard_name': 'beam_velocity', + 'units': 'cm s-1', + 'id_number' : sensor_info['std_speed_id'], + 'type' : sensor_info['std_speed_type'], + 'format' : sensor_info['std_speed_format'], + 'non_standard_units' : sensor_info['std_speed_units'], + 'range_min' : sensor_info['std_speed_range_min'], + 'range_max' : sensor_info['std_speed_range_max'], + }, + 'strength' : {'short_name' : 'beam_echo', + 'long_name': sensor_info['strength_description'], + 'standard_name': 'beam_echo_intensity', + 'units': 'db', + 'id_number' : sensor_info['strength_id'], + 'type' : sensor_info['strength_type'], + 'format' : sensor_info['strength_format'], + 'non_standard_units' : sensor_info['strength_units'], + 'range_min' : sensor_info['strength_range_min'], + 'range_max' : sensor_info['strength_range_max'], + }, + 'pings' : {'short_name' : 'pings', + 'long_name': sensor_info['pings_description'], + 'standard_name': 'none', + 'units': 'none', + 'id_number' : sensor_info['pings_id'], + 'type' : sensor_info['pings_type'], + 'format' : sensor_info['pings_format'], + 'non_standard_units' : sensor_info['pings_units'], + 'range_min' : sensor_info['pings_range_min'], + 'range_max' : sensor_info['pings_range_max'], + }, + } + + # dimension names use tuple so order of initialization is maintained + dim_inits = ( + ('ntime', pycdf.NC.UNLIMITED), + ('nlat', 1), + ('nlon', 1), + ('nz', 1), + ) + + # using tuple of tuples so order of initialization is maintained + # using dict for attributes order of init not important + # use dimension names not values + # (varName, varType, (dimName1, [dimName2], ...)) + var_inits = ( + # coordinate variables + ('time', pycdf.NC.INT, ('ntime',)), + ('lat', pycdf.NC.FLOAT, ('nlat',)), + ('lon', pycdf.NC.FLOAT, ('nlon',)), + ('z', pycdf.NC.FLOAT, ('nz',)), + # data variables + ('ptime', pycdf.NC.INT, ('ntime',)), + ('session', pycdf.NC.INT, ('ntime',)), + ('psession', pycdf.NC.INT, ('ntime',)), + ('record', pycdf.NC.INT, ('ntime',)), + ('status', pycdf.NC.INT, ('ntime',)), + ('pstatus', pycdf.NC.INT, ('ntime',)), + ('abs_speed', pycdf.NC.FLOAT, ('ntime',)), + ('direction', pycdf.NC.FLOAT, ('ntime',)), + ('v', pycdf.NC.FLOAT, ('ntime',)), + ('u', pycdf.NC.FLOAT, ('ntime',)), + ('heading', pycdf.NC.FLOAT, ('ntime',)), + ('tiltx', pycdf.NC.FLOAT, ('ntime',)), + ('tilty', pycdf.NC.FLOAT, ('ntime',)), + ('std_speed', pycdf.NC.FLOAT, ('ntime',)), + ('strength', pycdf.NC.FLOAT, ('ntime',)), + ('pings', pycdf.NC.INT, ('ntime',)), + ) + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # var data + var_data = ( + ('time', data['time'][i]), + ('lat', sensor_info['lat']), + ('lon', sensor_info['lat']), + ('z', sensor_info['elevation']), + ('ptime', data['ptime'][i]), + ('session', data['session'][i]), + ('psession', data['psession'][i]), + ('record', data['record'][i]), + ('status', data['status'][i]), + ('pstatus', data['pstatus'][i]), + ('abs_speed', data['abs_speed'][i]), + ('direction', data['direction'][i]), + ('v', data['v'][i]), + ('u', data['u'][i]), + ('heading', data['heading'][i]), + ('tiltx', data['tiltx'][i]), + ('tilty', data['tilty'][i]), + ('std_speed', data['std_speed'][i]), + ('strength', data['strength'][i]), + ('pings', data['pings'][i]), + ) + + return (global_atts, var_atts, dim_inits, var_inits, var_data) + +def updater(platform_info, sensor_info, data): + # + global_atts = { + # update times of data contained in file (yyyy-mm-dd HH:MM:SS) + # last date in monthly file + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + # + 'modification_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + } + + # data variables + # update any variable attributes like range, min, max + var_atts = {} + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # data + var_data = ( + ('time', data['time'][i]), + ('ptime', data['ptime'][i]), + ('session', data['session'][i]), + ('psession', data['psession'][i]), + ('record', data['record'][i]), + ('status', data['status'][i]), + ('pstatus', data['pstatus'][i]), + ('abs_speed', data['abs_speed'][i]), + ('direction', data['direction'][i]), + ('v', data['v'][i]), + ('u', data['u'][i]), + ('heading', data['heading'][i]), + ('tiltx', data['tiltx'][i]), + ('tilty', data['tilty'][i]), + ('std_speed', data['std_speed'][i]), + ('strength', data['strength'][i]), + ('pings', data['pings'][i]), + ) + + return (global_atts, var_atts, var_data) + Index: raw2proc/trunk/raw2proc/proc_mini_andi_optode.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/proc_mini_andi_optode.py (revision 375) @@ -1,0 +1,579 @@ +""" +Parse data and assert what data creates and updates monthly NetCDF files. + +Spongenet mini_andi optode parameters sponge data. +""" + +import math +import numpy as n +import pycdf +import datetime +import procutil +from spongenet.parse import Data + +nowDt = datetime.datetime.utcnow().replace(microsecond=0) + +def parser(platform_info, sensor_info, lines): + """ + Parse and assign sponge data from XML file. + """ + + _data = Data(''.join(lines)) + + # Each Device tag represents a time sample. + num_samples = len(_data.devices) + + data = { + 'dt' : n.array(n.ones((num_samples,)) * n.nan, dtype=object), + 'time' : n.array(n.ones((num_samples,)) * n.nan, dtype=long), + 'pdt' : n.array(n.ones((num_samples,)) * n.nan, dtype=object), + 'ptime' : n.array(n.ones((num_samples,)) * n.nan, dtype=long), + 'ds' : n.array(n.ones((num_samples,)) * n.nan, dtype=object), + 'session' : n.array(n.ones((num_samples,)) * n.nan, dtype=long), + 'pds' : n.array(n.ones((num_samples,)) * n.nan, dtype=object), + 'psession' : n.array(n.ones((num_samples,)) * n.nan, dtype=long), + 'record' : n.array(n.ones((num_samples,)) * n.nan, dtype=int), + 'status' : n.array(n.ones((num_samples,)) * n.nan, dtype=int), + 'pstatus' : n.array(n.ones((num_samples,)) * n.nan, dtype=int), + 'o2concentration' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + 'airsaturation' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + 'temperature' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + 'calphase' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + 'tcphase' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + 'c1rph' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + 'c2rph' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + 'c1amp' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + 'c2amp' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + 'rawtemp' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + } + + for (sample_index, sample) in enumerate(_data.devices): + # sample time at the platform + dt = {'month' : int(sample['time'][5:7]), + 'day' : int(sample['time'][8:10]), + 'year' : int(sample['time'][0:4]), + 'hour' : int(sample['time'][11:13]), + 'min' : int(sample['time'][14:16]), + 'sec' : int(sample['time'][17:19]), + } + dt = '%(month)02d-%(day)02d-%(year)04d %(hour)02d:%(min)02d:%(sec)02d' \ + % dt + dt = procutil.scanf_datetime(dt, fmt='%m-%d-%Y %H:%M:%S') + if sensor_info['utc_offset']: + dt = dt + datetime.timedelta(hours=sensor_info['utc_offset']) + data['dt'][sample_index] = dt + data['time'][sample_index] = procutil.dt2es(dt) + + # sample time at the package + package_dt = {'month' : int(sample['data_time'][5:7]), + 'day' : int(sample['data_time'][8:10]), + 'year' : int(sample['data_time'][0:4]), + 'hour' : int(sample['data_time'][11:13]), + 'min' : int(sample['data_time'][14:16]), + 'sec' : int(sample['data_time'][17:19]), + } + package_dt = ('%(month)02d-%(day)02d-%(year)04d ' + + '%(hour)02d:%(min)02d:%(sec)02d') \ + % package_dt + package_dt = procutil.scanf_datetime(package_dt, fmt='%m-%d-%Y %H:%M:%S') + if sensor_info['utc_offset']: + package_dt = package_dt + \ + datetime.timedelta(hours=sensor_info['utc_offset']) + data['pdt'][sample_index] = package_dt + data['ptime'][sample_index] = procutil.dt2es(package_dt) + + # platform session time + ds = {'month' : int(sample['sessionid'][14:16]), + 'day' : int(sample['sessionid'][17:19]), + 'year' : int(sample['sessionid'][9:13]), + 'hour' : int(sample['sessionid'][20:22]), + 'min' : int(sample['sessionid'][23:25]), + 'sec' : int(sample['sessionid'][26:28]), + } + ds = '%(month)02d-%(day)02d-%(year)04d %(hour)02d:%(min)02d:%(sec)02d' \ + % ds + ds = procutil.scanf_datetime(ds, fmt='%m-%d-%Y %H:%M:%S') + if sensor_info['utc_offset']: + ds = ds + datetime.timedelta(hours=sensor_info['utc_offset']) + data['ds'][sample_index] = ds + data['session'][sample_index] = procutil.dt2es(ds) + + # package session time + package_ds = {'month' : int(sample['data_sessionid'][5:7]), + 'day' : int(sample['data_sessionid'][8:10]), + 'year' : int(sample['data_sessionid'][0:4]), + 'hour' : int(sample['data_sessionid'][11:13]), + 'min' : int(sample['data_sessionid'][14:16]), + 'sec' : int(sample['data_sessionid'][17:19]), + } + package_ds = ('%(month)02d-%(day)02d-%(year)04d ' + + '%(hour)02d:%(min)02d:%(sec)02d') \ + % package_ds + package_ds = procutil.scanf_datetime(package_ds, fmt='%m-%d-%Y %H:%M:%S') + if sensor_info['utc_offset']: + package_ds = package_ds + \ + datetime.timedelta(hours=sensor_info['utc_offset']) + data['pds'][sample_index] = package_ds + data['psession'][sample_index] = procutil.dt2es(package_ds) + + # platform variables + try: + data['record'][sample_index] = int(sample["recordnumber"]) + except KeyError: + pass + + try: + data['status'][sample_index] = int(sample["status"]. + partition(":")[0]) + except (KeyError, AttributeError, ): + pass + + # package variables + try: + data['pstatus'][sample_index] = int(sample.sensors + [sensor_info["id_number"]] + ["status"]. + partition(":")[0]) + except (KeyError, AttributeError, ): + pass + + try: + data['o2concentration'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["o2concentration_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + try: + data['airsaturation'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["airsaturation_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + try: + data['temperature'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["temperature_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + try: + data['calphase'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["calphase_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + try: + data['tcphase'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["tcphase_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + try: + data['c1rph'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["c1rph_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + try: + data['c2rph'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["c2rph_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + try: + data['c1amp'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["c1amp_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + try: + data['c2amp'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["c2amp_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + try: + data['rawtemp'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["rawtemp_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + return data + +def creator(platform_info, sensor_info, data): + # + # + title_str = sensor_info['description']+' at '+ sensor_info['location'] + global_atts = { + # Required + 'title' : title_str, + 'institution' : platform_info['institution'], + 'institution_url' : platform_info['institution_url'], + 'institution_dods_url' : platform_info['institution_dods_url'], + 'contact' : platform_info['contact'], + 'Conventions' : platform_info['conventions'], + # Required by Scout + 'format_category_code' : platform_info['format_category_code'], + 'institution_code' : platform_info['institution_code'], + 'platform_code' : platform_info['id'], + 'package_code' : sensor_info['id'], + # Required by Version tracking + 'format' : platform_info['format'], + 'seacoos_rt_version' : platform_info['seacoos_rt_version'], + # Recommended + '_FillValue' : n.nan, + 'missing_value' : n.nan, + 'source' : platform_info['source'], + 'references' : platform_info['references'], + 'metadata_url' : platform_info['metadata_url'], + 'history' : 'raw2proc using ' + sensor_info['process_module'], + 'comment' : 'File created using pycdf ' + \ + pycdf.pycdfVersion() + ' and ' + \ + pycdf.pycdfArrayPkg() + ' ' + \ + n.__version__, + 'project' : platform_info['project'], + 'project_url' : platform_info['project_url'], + # timeframe of data contained in file yyyy-mm-dd HH:MM:SS + # first date in monthly file + 'start_date' : data['dt'][0].strftime("%Y-%m-%d %H:%M:%S"), + # last date in monthly file + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + 'creation_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + 'modification_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + 'process_level' : 'level1', + # Custom + 'id_number' : platform_info['id_number'], + 'description' : platform_info['description'], + 'serial_number' : platform_info['serial_number'], + 'product_number' : platform_info['product_number'], + 'product_name' : platform_info['product_name'], + 'type' : platform_info['type'], + 'protocol_version' : platform_info['protocol_version'], + 'xmlns' : platform_info['xmlns'], + 'location' : platform_info['location'], + 'vertical_position': platform_info['vertical_position'], + 'owner' : platform_info['owner'], + 'package_id_number' : sensor_info['id_number'], + 'package_description' : sensor_info['description'], + 'package_serial_number' : sensor_info['serial_number'], + 'package_product_number' : sensor_info['product_number'], + 'package_product_name' : sensor_info['product_name'], + 'package_adr' : sensor_info['adr'], + 'package_protocol_version' : sensor_info['protocol_version'], + 'package_vertical_position' : sensor_info['vertical_position'], + } + + var_atts = { + # coordinate variables + 'time' : {'short_name': 'time', + 'long_name': 'Time', + 'standard_name': 'time', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + 'axis': 'T', + }, + 'lat' : {'short_name': 'lat', + 'long_name': 'Latitude', + 'standard_name': 'latitude', + 'reference':'geographic coordinates', + 'units': 'degrees_north', + 'valid_range':(-90.,90.), + 'axis': 'Y', + }, + 'lon' : {'short_name': 'lon', + 'long_name': 'Longtitude', + 'standard_name': 'longtitude', + 'reference':'geographic coordinates', + 'units': 'degrees_east', + 'valid_range':(-180.,180.), + 'axis': 'Y', + }, + 'z' : {'short_name': 'z', + 'long_name': 'Height', + 'standard_name': 'height', + 'reference':'zero at sea-surface', + 'positive' : 'up', + 'units': 'meters', + 'axis': 'Z', + }, + # data variables + 'ptime' : {'short_name': 'ptime', + 'long_name': 'Package Time', + 'standard_name': 'none', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + }, + 'session' : {'short_name': 'session', + 'long_name': 'Session ID', + 'standard_name': 'none', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + }, + 'psession' : {'short_name': 'ptime', + 'long_name': 'Package Session ID', + 'standard_name': 'none', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + }, + 'record' : {'short_name': 'record', + 'long_name': 'Record Number', + 'standard_name': 'none', + 'units' : 'none', + }, + 'status': {'short_name' : 'status', + 'long_name': 'Platform Status Code', + 'standard_name': 'none', + 'units' : 'none', + 'value_map' : platform_info['status_map'], + }, + 'pstatus': {'short_name' : 'pstatus', + 'long_name': 'Package Status Code', + 'standard_name': 'none', + 'units' : 'none', + 'value_map' : sensor_info['status_map'], + }, + 'o2concentration' : {'short_name' : 'o2concentration', + 'long_name': sensor_info['o2concentration_description'], + 'standard_name': 'oxygen_concentration', + 'units': 'none', + 'id_number' : sensor_info['o2concentration_id'], + 'type' : sensor_info['o2concentration_type'], + 'format' : sensor_info['o2concentration_format'], + 'non_standard_units' : sensor_info['o2concentration_units'], + 'range_min' : sensor_info['o2concentration_range_min'], + 'range_max' : sensor_info['o2concentration_range_max'], + }, + 'airsaturation' : {'short_name' : 'airsaturation', + 'long_name': sensor_info['airsaturation_description'], + 'standard_name': 'oxygen_saturation', + 'units': 'none', + 'id_number' : sensor_info['airsaturation_id'], + 'type' : sensor_info['airsaturation_type'], + 'format' : sensor_info['airsaturation_format'], + 'non_standard_units' : sensor_info['airsaturation_units'], + 'range_min' : sensor_info['airsaturation_range_min'], + 'range_max' : sensor_info['airsaturation_range_max'], + }, + 'temperature' : {'short_name' : 'temperature', + 'long_name': sensor_info['temperature_description'], + 'standard_name': 'water_temperature', + 'units': 'celsius', + 'id_number' : sensor_info['temperature_id'], + 'type' : sensor_info['temperature_type'], + 'format' : sensor_info['temperature_format'], + 'non_standard_units' : sensor_info['temperature_units'], + 'range_min' : sensor_info['temperature_range_min'], + 'range_max' : sensor_info['temperature_range_max'], + }, + 'calphase' : {'short_name' : 'calphase', + 'long_name': sensor_info['calphase_description'], + 'standard_name': 'none', + 'units': 'none', + 'id_number' : sensor_info['calphase_id'], + 'type' : sensor_info['calphase_type'], + 'format' : sensor_info['calphase_format'], + 'non_standard_units' : sensor_info['calphase_units'], + 'range_min' : sensor_info['calphase_range_min'], + 'range_max' : sensor_info['calphase_range_max'], + }, + 'tcphase' : {'short_name' : 'tcphase', + 'long_name': sensor_info['tcphase_description'], + 'standard_name': 'none', + 'units': 'none', + 'id_number' : sensor_info['tcphase_id'], + 'type' : sensor_info['tcphase_type'], + 'format' : sensor_info['tcphase_format'], + 'non_standard_units' : sensor_info['tcphase_units'], + 'range_min' : sensor_info['tcphase_range_min'], + 'range_max' : sensor_info['tcphase_range_max'], + }, + 'c1rph' : {'short_name' : 'c1rph', + 'long_name': sensor_info['c1rph_description'], + 'standard_name': 'none', + 'units': 'none', + 'id_number' : sensor_info['c1rph_id'], + 'type' : sensor_info['c1rph_type'], + 'format' : sensor_info['c1rph_format'], + 'non_standard_units' : sensor_info['c1rph_units'], + 'range_min' : sensor_info['c1rph_range_min'], + 'range_max' : sensor_info['c1rph_range_max'], + }, + 'c2rph' : {'short_name' : 'c2rph', + 'long_name': sensor_info['c2rph_description'], + 'standard_name': 'none', + 'units': 'none', + 'id_number' : sensor_info['c2rph_id'], + 'type' : sensor_info['c2rph_type'], + 'format' : sensor_info['c2rph_format'], + 'non_standard_units' : sensor_info['c2rph_units'], + 'range_min' : sensor_info['c2rph_range_min'], + 'range_max' : sensor_info['c2rph_range_max'], + }, + 'c1amp' : {'short_name' : 'c1amp', + 'long_name': sensor_info['c1amp_description'], + 'standard_name': 'none', + 'units': 'none', + 'id_number' : sensor_info['c1amp_id'], + 'type' : sensor_info['c1amp_type'], + 'format' : sensor_info['c1amp_format'], + 'non_standard_units' : sensor_info['c1amp_units'], + 'range_min' : sensor_info['c1amp_range_min'], + 'range_max' : sensor_info['c1amp_range_max'], + }, + 'c2amp' : {'short_name' : 'c2amp', + 'long_name': sensor_info['c2amp_description'], + 'standard_name': 'none', + 'units': 'none', + 'id_number' : sensor_info['c2amp_id'], + 'type' : sensor_info['c2amp_type'], + 'format' : sensor_info['c2amp_format'], + 'non_standard_units' : sensor_info['c2amp_units'], + 'range_min' : sensor_info['c2amp_range_min'], + 'range_max' : sensor_info['c2amp_range_max'], + }, + 'rawtemp' : {'short_name' : 'rawtemp', + 'long_name': sensor_info['rawtemp_description'], + 'standard_name': 'none', + 'units': 'none', + 'id_number' : sensor_info['rawtemp_id'], + 'type' : sensor_info['rawtemp_type'], + 'format' : sensor_info['rawtemp_format'], + 'non_standard_units' : sensor_info['rawtemp_units'], + 'range_min' : sensor_info['rawtemp_range_min'], + 'range_max' : sensor_info['rawtemp_range_max'], + }, + } + + # dimension names use tuple so order of initialization is maintained + dim_inits = ( + ('ntime', pycdf.NC.UNLIMITED), + ('nlat', 1), + ('nlon', 1), + ('nz', 1), + ) + + # using tuple of tuples so order of initialization is maintained + # using dict for attributes order of init not important + # use dimension names not values + # (varName, varType, (dimName1, [dimName2], ...)) + var_inits = ( + # coordinate variables + ('time', pycdf.NC.INT, ('ntime',)), + ('lat', pycdf.NC.FLOAT, ('nlat',)), + ('lon', pycdf.NC.FLOAT, ('nlon',)), + ('z', pycdf.NC.FLOAT, ('nz',)), + # data variables + ('ptime', pycdf.NC.INT, ('ntime',)), + ('session', pycdf.NC.INT, ('ntime',)), + ('psession', pycdf.NC.INT, ('ntime',)), + ('record', pycdf.NC.INT, ('ntime',)), + ('status', pycdf.NC.INT, ('ntime',)), + ('pstatus', pycdf.NC.INT, ('ntime',)), + ('o2concentration', pycdf.NC.FLOAT, ('ntime',)), + ('airsaturation', pycdf.NC.FLOAT, ('ntime',)), + ('temperature', pycdf.NC.FLOAT, ('ntime',)), + ('calphase', pycdf.NC.FLOAT, ('ntime',)), + ('tcphase', pycdf.NC.FLOAT, ('ntime',)), + ('c1rph', pycdf.NC.FLOAT, ('ntime',)), + ('c2rph', pycdf.NC.FLOAT, ('ntime',)), + ('c1amp', pycdf.NC.FLOAT, ('ntime',)), + ('c2amp', pycdf.NC.FLOAT, ('ntime',)), + ('rawtemp', pycdf.NC.FLOAT, ('ntime',)), + ) + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # var data + var_data = ( + ('time', data['time'][i]), + ('lat', sensor_info['lat']), + ('lon', sensor_info['lat']), + ('z', sensor_info['elevation']), + ('ptime', data['ptime'][i]), + ('session', data['session'][i]), + ('psession', data['psession'][i]), + ('record', data['record'][i]), + ('status', data['status'][i]), + ('pstatus', data['pstatus'][i]), + ('o2concentration', data['o2concentration'][i]), + ('airsaturation', data['airsaturation'][i]), + ('temperature', data['temperature'][i]), + ('calphase', data['calphase'][i]), + ('tcphase', data['tcphase'][i]), + ('c1rph', data['c1rph'][i]), + ('c2rph', data['c2rph'][i]), + ('c1amp', data['c1amp'][i]), + ('c2amp', data['c2amp'][i]), + ('rawtemp', data['rawtemp'][i]), + ) + + return (global_atts, var_atts, dim_inits, var_inits, var_data) + +def updater(platform_info, sensor_info, data): + # + global_atts = { + # update times of data contained in file (yyyy-mm-dd HH:MM:SS) + # last date in monthly file + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + # + 'modification_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + } + + # data variables + # update any variable attributes like range, min, max + var_atts = {} + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # data + var_data = ( + ('time', data['time'][i]), + ('ptime', data['ptime'][i]), + ('session', data['session'][i]), + ('psession', data['psession'][i]), + ('record', data['record'][i]), + ('status', data['status'][i]), + ('pstatus', data['pstatus'][i]), + ('o2concentration', data['o2concentration'][i]), + ('airsaturation', data['airsaturation'][i]), + ('temperature', data['temperature'][i]), + ('calphase', data['calphase'][i]), + ('tcphase', data['tcphase'][i]), + ('c1rph', data['c1rph'][i]), + ('c2rph', data['c2rph'][i]), + ('c1amp', data['c1amp'][i]), + ('c2amp', data['c2amp'][i]), + ('rawtemp', data['rawtemp'][i]), + ) + + return (global_atts, var_atts, var_data) + Index: raw2proc/trunk/raw2proc/proc_mini_andi_pressure.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/proc_mini_andi_pressure.py (revision 375) @@ -1,0 +1,386 @@ +""" +Parse data and assert what data creates and updates monthly NetCDF files. + +Spongenet mini_andi pressure parameters sponge data. +""" + +import math +import numpy as n +import pycdf +import datetime +import procutil +from spongenet.parse import Data + +nowDt = datetime.datetime.utcnow().replace(microsecond=0) + +def parser(platform_info, sensor_info, lines): + """ + Parse and assign sponge data from XML file. + """ + + _data = Data(''.join(lines)) + + # Each Device tag represents a time sample. + num_samples = len(_data.devices) + + data = { + 'dt' : n.array(n.ones((num_samples,)) * n.nan, dtype=object), + 'time' : n.array(n.ones((num_samples,)) * n.nan, dtype=long), + 'pdt' : n.array(n.ones((num_samples,)) * n.nan, dtype=object), + 'ptime' : n.array(n.ones((num_samples,)) * n.nan, dtype=long), + 'ds' : n.array(n.ones((num_samples,)) * n.nan, dtype=object), + 'session' : n.array(n.ones((num_samples,)) * n.nan, dtype=long), + 'pds' : n.array(n.ones((num_samples,)) * n.nan, dtype=object), + 'psession' : n.array(n.ones((num_samples,)) * n.nan, dtype=long), + 'record' : n.array(n.ones((num_samples,)) * n.nan, dtype=int), + 'status' : n.array(n.ones((num_samples,)) * n.nan, dtype=int), + 'pstatus' : n.array(n.ones((num_samples,)) * n.nan, dtype=int), + 'pressure' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + 'temperature' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + } + + for (sample_index, sample) in enumerate(_data.devices): + # sample time at the platform + dt = {'month' : int(sample['time'][5:7]), + 'day' : int(sample['time'][8:10]), + 'year' : int(sample['time'][0:4]), + 'hour' : int(sample['time'][11:13]), + 'min' : int(sample['time'][14:16]), + 'sec' : int(sample['time'][17:19]), + } + dt = '%(month)02d-%(day)02d-%(year)04d %(hour)02d:%(min)02d:%(sec)02d' \ + % dt + dt = procutil.scanf_datetime(dt, fmt='%m-%d-%Y %H:%M:%S') + if sensor_info['utc_offset']: + dt = dt + datetime.timedelta(hours=sensor_info['utc_offset']) + data['dt'][sample_index] = dt + data['time'][sample_index] = procutil.dt2es(dt) + + # sample time at the package + package_dt = {'month' : int(sample['data_time'][5:7]), + 'day' : int(sample['data_time'][8:10]), + 'year' : int(sample['data_time'][0:4]), + 'hour' : int(sample['data_time'][11:13]), + 'min' : int(sample['data_time'][14:16]), + 'sec' : int(sample['data_time'][17:19]), + } + package_dt = ('%(month)02d-%(day)02d-%(year)04d ' + + '%(hour)02d:%(min)02d:%(sec)02d') \ + % package_dt + package_dt = procutil.scanf_datetime(package_dt, fmt='%m-%d-%Y %H:%M:%S') + if sensor_info['utc_offset']: + package_dt = package_dt + \ + datetime.timedelta(hours=sensor_info['utc_offset']) + data['pdt'][sample_index] = package_dt + data['ptime'][sample_index] = procutil.dt2es(package_dt) + + # platform session time + ds = {'month' : int(sample['sessionid'][14:16]), + 'day' : int(sample['sessionid'][17:19]), + 'year' : int(sample['sessionid'][9:13]), + 'hour' : int(sample['sessionid'][20:22]), + 'min' : int(sample['sessionid'][23:25]), + 'sec' : int(sample['sessionid'][26:28]), + } + ds = '%(month)02d-%(day)02d-%(year)04d %(hour)02d:%(min)02d:%(sec)02d' \ + % ds + ds = procutil.scanf_datetime(ds, fmt='%m-%d-%Y %H:%M:%S') + if sensor_info['utc_offset']: + ds = ds + datetime.timedelta(hours=sensor_info['utc_offset']) + data['ds'][sample_index] = ds + data['session'][sample_index] = procutil.dt2es(ds) + + # package session time + package_ds = {'month' : int(sample['data_sessionid'][5:7]), + 'day' : int(sample['data_sessionid'][8:10]), + 'year' : int(sample['data_sessionid'][0:4]), + 'hour' : int(sample['data_sessionid'][11:13]), + 'min' : int(sample['data_sessionid'][14:16]), + 'sec' : int(sample['data_sessionid'][17:19]), + } + package_ds = ('%(month)02d-%(day)02d-%(year)04d ' + + '%(hour)02d:%(min)02d:%(sec)02d') \ + % package_ds + package_ds = procutil.scanf_datetime(package_ds, fmt='%m-%d-%Y %H:%M:%S') + if sensor_info['utc_offset']: + package_ds = package_ds + \ + datetime.timedelta(hours=sensor_info['utc_offset']) + data['pds'][sample_index] = package_ds + data['psession'][sample_index] = procutil.dt2es(package_ds) + + # platform variables + try: + data['record'][sample_index] = int(sample["recordnumber"]) + except KeyError: + pass + + try: + data['status'][sample_index] = int(sample["status"]. + partition(":")[0]) + except (KeyError, AttributeError, ): + pass + + # package variables + try: + data['pstatus'][sample_index] = int(sample.sensors + [sensor_info["id_number"]] + ["status"]. + partition(":")[0]) + except (KeyError, AttributeError, ): + pass + + try: + data['pressure'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["pressure_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + try: + data['temperature'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["temperature_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + return data + +def creator(platform_info, sensor_info, data): + # + # + title_str = sensor_info['description']+' at '+ sensor_info['location'] + global_atts = { + # Required + 'title' : title_str, + 'institution' : platform_info['institution'], + 'institution_url' : platform_info['institution_url'], + 'institution_dods_url' : platform_info['institution_dods_url'], + 'contact' : platform_info['contact'], + 'Conventions' : platform_info['conventions'], + # Required by Scout + 'format_category_code' : platform_info['format_category_code'], + 'institution_code' : platform_info['institution_code'], + 'platform_code' : platform_info['id'], + 'package_code' : sensor_info['id'], + # Required by Version tracking + 'format' : platform_info['format'], + 'seacoos_rt_version' : platform_info['seacoos_rt_version'], + # Recommended + '_FillValue' : n.nan, + 'missing_value' : n.nan, + 'source' : platform_info['source'], + 'references' : platform_info['references'], + 'metadata_url' : platform_info['metadata_url'], + 'history' : 'raw2proc using ' + sensor_info['process_module'], + 'comment' : 'File created using pycdf ' + \ + pycdf.pycdfVersion() + ' and ' + \ + pycdf.pycdfArrayPkg() + ' ' + \ + n.__version__, + 'project' : platform_info['project'], + 'project_url' : platform_info['project_url'], + # timeframe of data contained in file yyyy-mm-dd HH:MM:SS + # first date in monthly file + 'start_date' : data['dt'][0].strftime("%Y-%m-%d %H:%M:%S"), + # last date in monthly file + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + 'creation_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + 'modification_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + 'process_level' : 'level1', + # Custom + 'id_number' : platform_info['id_number'], + 'description' : platform_info['description'], + 'serial_number' : platform_info['serial_number'], + 'product_number' : platform_info['product_number'], + 'product_name' : platform_info['product_name'], + 'type' : platform_info['type'], + 'protocol_version' : platform_info['protocol_version'], + 'xmlns' : platform_info['xmlns'], + 'location' : platform_info['location'], + 'vertical_position': platform_info['vertical_position'], + 'owner' : platform_info['owner'], + 'package_id_number' : sensor_info['id_number'], + 'package_description' : sensor_info['description'], + 'package_serial_number' : sensor_info['serial_number'], + 'package_product_number' : sensor_info['product_number'], + 'package_product_name' : sensor_info['product_name'], + 'package_adr' : sensor_info['adr'], + 'package_protocol_version' : sensor_info['protocol_version'], + 'package_vertical_position' : sensor_info['vertical_position'], + } + + var_atts = { + # coordinate variables + 'time' : {'short_name': 'time', + 'long_name': 'Time', + 'standard_name': 'time', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + 'axis': 'T', + }, + 'lat' : {'short_name': 'lat', + 'long_name': 'Latitude', + 'standard_name': 'latitude', + 'reference':'geographic coordinates', + 'units': 'degrees_north', + 'valid_range':(-90.,90.), + 'axis': 'Y', + }, + 'lon' : {'short_name': 'lon', + 'long_name': 'Longtitude', + 'standard_name': 'longtitude', + 'reference':'geographic coordinates', + 'units': 'degrees_east', + 'valid_range':(-180.,180.), + 'axis': 'Y', + }, + 'z' : {'short_name': 'z', + 'long_name': 'Height', + 'standard_name': 'height', + 'reference':'zero at sea-surface', + 'positive' : 'up', + 'units': 'meters', + 'axis': 'Z', + }, + # data variables + 'ptime' : {'short_name': 'ptime', + 'long_name': 'Package Time', + 'standard_name': 'none', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + }, + 'session' : {'short_name': 'session', + 'long_name': 'Session ID', + 'standard_name': 'none', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + }, + 'psession' : {'short_name': 'ptime', + 'long_name': 'Package Session ID', + 'standard_name': 'none', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + }, + 'record' : {'short_name': 'record', + 'long_name': 'Record Number', + 'standard_name': 'none', + 'units' : 'none', + }, + 'status': {'short_name' : 'status', + 'long_name': 'Platform Status Code', + 'standard_name': 'none', + 'units' : 'none', + 'value_map' : platform_info['status_map'], + }, + 'pstatus': {'short_name' : 'pstatus', + 'long_name': 'Package Status Code', + 'standard_name': 'none', + 'units' : 'none', + 'value_map' : sensor_info['status_map'], + }, + 'pressure' : {'short_name' : 'pressure', + 'long_name': sensor_info['pressure_description'], + 'standard_name': 'water_pressure', + 'units': 'none', + 'id_number' : sensor_info['pressure_id'], + 'type' : sensor_info['pressure_type'], + 'format' : sensor_info['pressure_format'], + 'non_standard_units' : sensor_info['pressure_units'], + 'range_min' : sensor_info['pressure_range_min'], + 'range_max' : sensor_info['pressure_range_max'], + }, + 'temperature' : {'short_name' : 'temperature', + 'long_name': sensor_info['temperature_description'], + 'standard_name': 'water_temperature', + 'units': 'celsius', + 'id_number' : sensor_info['temperature_id'], + 'type' : sensor_info['temperature_type'], + 'format' : sensor_info['temperature_format'], + 'non_standard_units' : sensor_info['temperature_units'], + 'range_min' : sensor_info['temperature_range_min'], + 'range_max' : sensor_info['temperature_range_max'], + }, + } + + # dimension names use tuple so order of initialization is maintained + dim_inits = ( + ('ntime', pycdf.NC.UNLIMITED), + ('nlat', 1), + ('nlon', 1), + ('nz', 1), + ) + + # using tuple of tuples so order of initialization is maintained + # using dict for attributes order of init not important + # use dimension names not values + # (varName, varType, (dimName1, [dimName2], ...)) + var_inits = ( + # coordinate variables + ('time', pycdf.NC.INT, ('ntime',)), + ('lat', pycdf.NC.FLOAT, ('nlat',)), + ('lon', pycdf.NC.FLOAT, ('nlon',)), + ('z', pycdf.NC.FLOAT, ('nz',)), + # data variables + ('ptime', pycdf.NC.INT, ('ntime',)), + ('session', pycdf.NC.INT, ('ntime',)), + ('psession', pycdf.NC.INT, ('ntime',)), + ('record', pycdf.NC.INT, ('ntime',)), + ('status', pycdf.NC.INT, ('ntime',)), + ('pstatus', pycdf.NC.INT, ('ntime',)), + ('pressure', pycdf.NC.FLOAT, ('ntime',)), + ('temperature', pycdf.NC.FLOAT, ('ntime',)), + ) + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # var data + var_data = ( + ('time', data['time'][i]), + ('lat', sensor_info['lat']), + ('lon', sensor_info['lat']), + ('z', sensor_info['elevation']), + ('ptime', data['ptime'][i]), + ('session', data['session'][i]), + ('psession', data['psession'][i]), + ('record', data['record'][i]), + ('status', data['status'][i]), + ('pstatus', data['pstatus'][i]), + ('pressure', data['pressure'][i]), + ('temperature', data['temperature'][i]), + ) + + return (global_atts, var_atts, dim_inits, var_inits, var_data) + +def updater(platform_info, sensor_info, data): + # + global_atts = { + # update times of data contained in file (yyyy-mm-dd HH:MM:SS) + # last date in monthly file + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + # + 'modification_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + } + + # data variables + # update any variable attributes like range, min, max + var_atts = {} + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # data + var_data = ( + ('time', data['time'][i]), + ('ptime', data['ptime'][i]), + ('session', data['session'][i]), + ('psession', data['psession'][i]), + ('record', data['record'][i]), + ('status', data['status'][i]), + ('pstatus', data['pstatus'][i]), + ('pressure', data['pressure'][i]), + ('temperature', data['temperature'][i]), + ) + + return (global_atts, var_atts, var_data) Index: raw2proc/trunk/raw2proc/proc_mini_andi_system.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/proc_mini_andi_system.py (revision 375) @@ -1,0 +1,411 @@ +""" +Parse data and assert what data creates and updates monthly NetCDF files. + +Spongenet mini_andi system parameters sponge data. +""" + +import math +import numpy as n +import pycdf +import datetime +import procutil +from spongenet.parse import Data + +nowDt = datetime.datetime.utcnow().replace(microsecond=0) + +def parser(platform_info, sensor_info, lines): + """ + Parse and assign sponge data from XML file. + """ + + _data = Data(''.join(lines)) + + # Each Device tag represents a time sample. + num_samples = len(_data.devices) + + data = { + 'dt' : n.array(n.ones((num_samples,)) * n.nan, dtype=object), + 'time' : n.array(n.ones((num_samples,)) * n.nan, dtype=long), + 'pdt' : n.array(n.ones((num_samples,)) * n.nan, dtype=object), + 'ptime' : n.array(n.ones((num_samples,)) * n.nan, dtype=long), + 'ds' : n.array(n.ones((num_samples,)) * n.nan, dtype=object), + 'session' : n.array(n.ones((num_samples,)) * n.nan, dtype=long), + 'pds' : n.array(n.ones((num_samples,)) * n.nan, dtype=object), + 'psession' : n.array(n.ones((num_samples,)) * n.nan, dtype=long), + 'record' : n.array(n.ones((num_samples,)) * n.nan, dtype=int), + 'status' : n.array(n.ones((num_samples,)) * n.nan, dtype=int), + 'pstatus' : n.array(n.ones((num_samples,)) * n.nan, dtype=int), + 'voltage' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + 'memory' : n.array(n.ones((num_samples,)) * n.nan, dtype=int), + 'interval' : n.array(n.ones((num_samples,)) * n.nan, dtype=int), + } + + for (sample_index, sample) in enumerate(_data.devices): + # sample time at the platform + dt = {'month' : int(sample['time'][5:7]), + 'day' : int(sample['time'][8:10]), + 'year' : int(sample['time'][0:4]), + 'hour' : int(sample['time'][11:13]), + 'min' : int(sample['time'][14:16]), + 'sec' : int(sample['time'][17:19]), + } + dt = '%(month)02d-%(day)02d-%(year)04d %(hour)02d:%(min)02d:%(sec)02d' \ + % dt + dt = procutil.scanf_datetime(dt, fmt='%m-%d-%Y %H:%M:%S') + if sensor_info['utc_offset']: + dt = dt + datetime.timedelta(hours=sensor_info['utc_offset']) + data['dt'][sample_index] = dt + data['time'][sample_index] = procutil.dt2es(dt) + + # sample time at the package + package_dt = {'month' : int(sample['data_time'][5:7]), + 'day' : int(sample['data_time'][8:10]), + 'year' : int(sample['data_time'][0:4]), + 'hour' : int(sample['data_time'][11:13]), + 'min' : int(sample['data_time'][14:16]), + 'sec' : int(sample['data_time'][17:19]), + } + package_dt = ('%(month)02d-%(day)02d-%(year)04d ' + + '%(hour)02d:%(min)02d:%(sec)02d') \ + % package_dt + package_dt = procutil.scanf_datetime(package_dt, fmt='%m-%d-%Y %H:%M:%S') + if sensor_info['utc_offset']: + package_dt = package_dt + \ + datetime.timedelta(hours=sensor_info['utc_offset']) + data['pdt'][sample_index] = package_dt + data['ptime'][sample_index] = procutil.dt2es(package_dt) + + # platform session time + ds = {'month' : int(sample['sessionid'][14:16]), + 'day' : int(sample['sessionid'][17:19]), + 'year' : int(sample['sessionid'][9:13]), + 'hour' : int(sample['sessionid'][20:22]), + 'min' : int(sample['sessionid'][23:25]), + 'sec' : int(sample['sessionid'][26:28]), + } + ds = '%(month)02d-%(day)02d-%(year)04d %(hour)02d:%(min)02d:%(sec)02d' \ + % ds + ds = procutil.scanf_datetime(ds, fmt='%m-%d-%Y %H:%M:%S') + if sensor_info['utc_offset']: + ds = ds + datetime.timedelta(hours=sensor_info['utc_offset']) + data['ds'][sample_index] = ds + data['session'][sample_index] = procutil.dt2es(ds) + + # package session time + package_ds = {'month' : int(sample['data_sessionid'][5:7]), + 'day' : int(sample['data_sessionid'][8:10]), + 'year' : int(sample['data_sessionid'][0:4]), + 'hour' : int(sample['data_sessionid'][11:13]), + 'min' : int(sample['data_sessionid'][14:16]), + 'sec' : int(sample['data_sessionid'][17:19]), + } + package_ds = ('%(month)02d-%(day)02d-%(year)04d ' + + '%(hour)02d:%(min)02d:%(sec)02d') \ + % package_ds + package_ds = procutil.scanf_datetime(package_ds, fmt='%m-%d-%Y %H:%M:%S') + if sensor_info['utc_offset']: + package_ds = package_ds + \ + datetime.timedelta(hours=sensor_info['utc_offset']) + data['pds'][sample_index] = package_ds + data['psession'][sample_index] = procutil.dt2es(package_ds) + + # platform variables + try: + data['record'][sample_index] = int(sample["recordnumber"]) + except KeyError: + pass + + try: + data['status'][sample_index] = int(sample["status"]. + partition(":")[0]) + except (KeyError, AttributeError, ): + pass + + # package variables + try: + data['pstatus'][sample_index] = int(sample.sensors + [sensor_info["id_number"]] + ["status"]. + partition(":")[0]) + except (KeyError, AttributeError, ): + pass + + try: + data['voltage'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["voltage_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + try: + data['memory'][sample_index] = int(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["memory_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + try: + data['interval'][sample_index] = int(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["interval_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + return data + +def creator(platform_info, sensor_info, data): + # + # + title_str = sensor_info['description']+' at '+ sensor_info['location'] + global_atts = { + # Required + 'title' : title_str, + 'institution' : platform_info['institution'], + 'institution_url' : platform_info['institution_url'], + 'institution_dods_url' : platform_info['institution_dods_url'], + 'contact' : platform_info['contact'], + 'Conventions' : platform_info['conventions'], + # Required by Scout + 'format_category_code' : platform_info['format_category_code'], + 'institution_code' : platform_info['institution_code'], + 'platform_code' : platform_info['id'], + 'package_code' : sensor_info['id'], + # Required by Version tracking + 'format' : platform_info['format'], + 'seacoos_rt_version' : platform_info['seacoos_rt_version'], + # Recommended + '_FillValue' : n.nan, + 'missing_value' : n.nan, + 'source' : platform_info['source'], + 'references' : platform_info['references'], + 'metadata_url' : platform_info['metadata_url'], + 'history' : 'raw2proc using ' + sensor_info['process_module'], + 'comment' : 'File created using pycdf ' + \ + pycdf.pycdfVersion() + ' and ' + \ + pycdf.pycdfArrayPkg() + ' ' + \ + n.__version__, + 'project' : platform_info['project'], + 'project_url' : platform_info['project_url'], + # timeframe of data contained in file yyyy-mm-dd HH:MM:SS + # first date in monthly file + 'start_date' : data['dt'][0].strftime("%Y-%m-%d %H:%M:%S"), + # last date in monthly file + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + 'creation_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + 'modification_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + 'process_level' : 'level1', + # Custom + 'id_number' : platform_info['id_number'], + 'description' : platform_info['description'], + 'serial_number' : platform_info['serial_number'], + 'product_number' : platform_info['product_number'], + 'product_name' : platform_info['product_name'], + 'type' : platform_info['type'], + 'protocol_version' : platform_info['protocol_version'], + 'xmlns' : platform_info['xmlns'], + 'location' : platform_info['location'], + 'vertical_position': platform_info['vertical_position'], + 'owner' : platform_info['owner'], + 'package_id_number' : sensor_info['id_number'], + 'package_description' : sensor_info['description'], + 'package_serial_number' : sensor_info['serial_number'], + 'package_product_number' : sensor_info['product_number'], + 'package_product_name' : sensor_info['product_name'], + 'package_adr' : sensor_info['adr'], + 'package_protocol_version' : sensor_info['protocol_version'], + 'package_vertical_position' : sensor_info['vertical_position'], + } + + var_atts = { + # coordinate variables + 'time' : {'short_name': 'time', + 'long_name': 'Time', + 'standard_name': 'time', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + 'axis': 'T', + }, + 'lat' : {'short_name': 'lat', + 'long_name': 'Latitude', + 'standard_name': 'latitude', + 'reference':'geographic coordinates', + 'units': 'degrees_north', + 'valid_range':(-90.,90.), + 'axis': 'Y', + }, + 'lon' : {'short_name': 'lon', + 'long_name': 'Longtitude', + 'standard_name': 'longtitude', + 'reference':'geographic coordinates', + 'units': 'degrees_east', + 'valid_range':(-180.,180.), + 'axis': 'Y', + }, + 'z' : {'short_name': 'z', + 'long_name': 'Height', + 'standard_name': 'height', + 'reference':'zero at sea-surface', + 'positive' : 'up', + 'units': 'meters', + 'axis': 'Z', + }, + # data variables + 'ptime' : {'short_name': 'ptime', + 'long_name': 'Package Time', + 'standard_name': 'none', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + }, + 'session' : {'short_name': 'session', + 'long_name': 'Session ID', + 'standard_name': 'none', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + }, + 'psession' : {'short_name': 'ptime', + 'long_name': 'Package Session ID', + 'standard_name': 'none', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + }, + 'record' : {'short_name': 'record', + 'long_name': 'Record Number', + 'standard_name': 'none', + 'units' : 'none', + }, + 'status': {'short_name' : 'status', + 'long_name': 'Platform Status Code', + 'standard_name': 'none', + 'units' : 'none', + 'value_map' : platform_info['status_map'], + }, + 'pstatus': {'short_name' : 'pstatus', + 'long_name': 'Package Status Code', + 'standard_name': 'none', + 'units' : 'none', + 'value_map' : sensor_info['status_map'], + }, + 'voltage': {'short_name' : 'voltage', + 'long_name': sensor_info['voltage_description'], + 'standard_name': 'none', + 'units': 'volts', + 'id_number' : sensor_info['voltage_id'], + 'type' : sensor_info['voltage_type'], + 'format' : sensor_info['voltage_format'], + 'non_standard_units' : sensor_info['voltage_units'], + 'range_min' : sensor_info['voltage_range_min'], + 'range_max' : sensor_info['voltage_range_max'], + }, + 'memory': {'short_name' : 'memory', + 'long_name': sensor_info['memory_description'], + 'standard_name': 'none', + 'units' : 'none', + 'id_number' : sensor_info['memory_id'], + 'type' : sensor_info['memory_type'], + 'format' : sensor_info['memory_format'], + 'non_standard_units' : sensor_info['memory_units'], + 'range_min' : sensor_info['memory_range_min'], + 'range_max' : sensor_info['memory_range_max'], + }, + 'interval': {'short_name' : 'interval', + 'long_name': 'Averaging Interval', + 'standard_name': 'averaging_interval', + 'units': 'seconds/1000', + 'id_number' : sensor_info['interval_id'], + 'type' : sensor_info['interval_type'], + 'format' : sensor_info['interval_format'], + 'non_standard_units' : sensor_info['interval_units'], + 'range_min' : sensor_info['interval_range_min'], + 'range_max' : sensor_info['interval_range_max'], + }, + } + + # dimension names use tuple so order of initialization is maintained + dim_inits = ( + ('ntime', pycdf.NC.UNLIMITED), + ('nlat', 1), + ('nlon', 1), + ('nz', 1), + ) + + # using tuple of tuples so order of initialization is maintained + # using dict for attributes order of init not important + # use dimension names not values + # (varName, varType, (dimName1, [dimName2], ...)) + var_inits = ( + # coordinate variables + ('time', pycdf.NC.INT, ('ntime',)), + ('lat', pycdf.NC.FLOAT, ('nlat',)), + ('lon', pycdf.NC.FLOAT, ('nlon',)), + ('z', pycdf.NC.FLOAT, ('nz',)), + # data variables + ('ptime', pycdf.NC.INT, ('ntime',)), + ('session', pycdf.NC.INT, ('ntime',)), + ('psession', pycdf.NC.INT, ('ntime',)), + ('record', pycdf.NC.INT, ('ntime',)), + ('status', pycdf.NC.INT, ('ntime',)), + ('pstatus', pycdf.NC.INT, ('ntime',)), + ('voltage', pycdf.NC.FLOAT, ('ntime',)), + ('memory', pycdf.NC.INT, ('ntime',)), + ('interval', pycdf.NC.INT, ('ntime',)), + ) + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # var data + var_data = ( + ('time', data['time'][i]), + ('lat', sensor_info['lat']), + ('lon', sensor_info['lat']), + ('z', sensor_info['elevation']), + ('ptime', data['ptime'][i]), + ('session', data['session'][i]), + ('psession', data['psession'][i]), + ('record', data['record'][i]), + ('status', data['status'][i]), + ('pstatus', data['pstatus'][i]), + ('voltage', data['voltage'][i]), + ('memory', data['memory'][i]), + ('interval', data['interval'][i]), + ) + + return (global_atts, var_atts, dim_inits, var_inits, var_data) + +def updater(platform_info, sensor_info, data): + # + global_atts = { + # update times of data contained in file (yyyy-mm-dd HH:MM:SS) + # last date in monthly file + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + # + 'modification_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + } + + # data variables + # update any variable attributes like range, min, max + var_atts = {} + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # data + var_data = ( + ('time', data['time'][i]), + ('ptime', data['ptime'][i]), + ('session', data['session'][i]), + ('psession', data['psession'][i]), + ('record', data['record'][i]), + ('status', data['status'][i]), + ('pstatus', data['pstatus'][i]), + ('voltage', data['voltage'][i]), + ('memory', data['memory'][i]), + ('interval', data['interval'][i]), + ) + + return (global_atts, var_atts, var_data) + Index: raw2proc/trunk/raw2proc/proc_mini_andi_turbidity.py =================================================================== --- (revision ) +++ raw2proc/trunk/raw2proc/proc_mini_andi_turbidity.py (revision 375) @@ -1,0 +1,363 @@ +""" +Parse data and assert what data creates and updates monthly NetCDF files. + +Spongenet mini_andi turbidity parameters sponge data. +""" + +import math +import numpy as n +import pycdf +import datetime +import procutil +from spongenet.parse import Data + +nowDt = datetime.datetime.utcnow().replace(microsecond=0) + +def parser(platform_info, sensor_info, lines): + """ + Parse and assign sponge data from XML file. + """ + + _data = Data(''.join(lines)) + + # Each Device tag represents a time sample. + num_samples = len(_data.devices) + + data = { + 'dt' : n.array(n.ones((num_samples,)) * n.nan, dtype=object), + 'time' : n.array(n.ones((num_samples,)) * n.nan, dtype=long), + 'pdt' : n.array(n.ones((num_samples,)) * n.nan, dtype=object), + 'ptime' : n.array(n.ones((num_samples,)) * n.nan, dtype=long), + 'ds' : n.array(n.ones((num_samples,)) * n.nan, dtype=object), + 'session' : n.array(n.ones((num_samples,)) * n.nan, dtype=long), + 'pds' : n.array(n.ones((num_samples,)) * n.nan, dtype=object), + 'psession' : n.array(n.ones((num_samples,)) * n.nan, dtype=long), + 'record' : n.array(n.ones((num_samples,)) * n.nan, dtype=int), + 'status' : n.array(n.ones((num_samples,)) * n.nan, dtype=int), + 'pstatus' : n.array(n.ones((num_samples,)) * n.nan, dtype=int), + 'turbidity' : n.array(n.ones((num_samples,)) * n.nan, dtype=float), + } + + for (sample_index, sample) in enumerate(_data.devices): + # sample time at the platform + dt = {'month' : int(sample['time'][5:7]), + 'day' : int(sample['time'][8:10]), + 'year' : int(sample['time'][0:4]), + 'hour' : int(sample['time'][11:13]), + 'min' : int(sample['time'][14:16]), + 'sec' : int(sample['time'][17:19]), + } + dt = '%(month)02d-%(day)02d-%(year)04d %(hour)02d:%(min)02d:%(sec)02d' \ + % dt + dt = procutil.scanf_datetime(dt, fmt='%m-%d-%Y %H:%M:%S') + if sensor_info['utc_offset']: + dt = dt + datetime.timedelta(hours=sensor_info['utc_offset']) + data['dt'][sample_index] = dt + data['time'][sample_index] = procutil.dt2es(dt) + + # sample time at the package + package_dt = {'month' : int(sample['data_time'][5:7]), + 'day' : int(sample['data_time'][8:10]), + 'year' : int(sample['data_time'][0:4]), + 'hour' : int(sample['data_time'][11:13]), + 'min' : int(sample['data_time'][14:16]), + 'sec' : int(sample['data_time'][17:19]), + } + package_dt = ('%(month)02d-%(day)02d-%(year)04d ' + + '%(hour)02d:%(min)02d:%(sec)02d') \ + % package_dt + package_dt = procutil.scanf_datetime(package_dt, fmt='%m-%d-%Y %H:%M:%S') + if sensor_info['utc_offset']: + package_dt = package_dt + \ + datetime.timedelta(hours=sensor_info['utc_offset']) + data['pdt'][sample_index] = package_dt + data['ptime'][sample_index] = procutil.dt2es(package_dt) + + # platform session time + ds = {'month' : int(sample['sessionid'][14:16]), + 'day' : int(sample['sessionid'][17:19]), + 'year' : int(sample['sessionid'][9:13]), + 'hour' : int(sample['sessionid'][20:22]), + 'min' : int(sample['sessionid'][23:25]), + 'sec' : int(sample['sessionid'][26:28]), + } + ds = '%(month)02d-%(day)02d-%(year)04d %(hour)02d:%(min)02d:%(sec)02d' \ + % ds + ds = procutil.scanf_datetime(ds, fmt='%m-%d-%Y %H:%M:%S') + if sensor_info['utc_offset']: + ds = ds + datetime.timedelta(hours=sensor_info['utc_offset']) + data['ds'][sample_index] = ds + data['session'][sample_index] = procutil.dt2es(ds) + + # package session time + package_ds = {'month' : int(sample['data_sessionid'][5:7]), + 'day' : int(sample['data_sessionid'][8:10]), + 'year' : int(sample['data_sessionid'][0:4]), + 'hour' : int(sample['data_sessionid'][11:13]), + 'min' : int(sample['data_sessionid'][14:16]), + 'sec' : int(sample['data_sessionid'][17:19]), + } + package_ds = ('%(month)02d-%(day)02d-%(year)04d ' + + '%(hour)02d:%(min)02d:%(sec)02d') \ + % package_ds + package_ds = procutil.scanf_datetime(package_ds, fmt='%m-%d-%Y %H:%M:%S') + if sensor_info['utc_offset']: + package_ds = package_ds + \ + datetime.timedelta(hours=sensor_info['utc_offset']) + data['pds'][sample_index] = package_ds + data['psession'][sample_index] = procutil.dt2es(package_ds) + + # platform variables + try: + data['record'][sample_index] = int(sample["recordnumber"]) + except KeyError: + pass + + try: + data['status'][sample_index] = int(sample["status"]. + partition(":")[0]) + except (KeyError, AttributeError, ): + pass + + # package variables + try: + data['pstatus'][sample_index] = int(sample.sensors + [sensor_info["id_number"]] + ["status"]. + partition(":")[0]) + except (KeyError, AttributeError, ): + pass + + try: + data['turbidity'][sample_index] = float(sample.sensors + [sensor_info["id_number"]]. + points[sensor_info + ["turbidity_description"]] + ["value"]) + except (KeyError, AttributeError, ): + pass + + return data + +def creator(platform_info, sensor_info, data): + # + # + title_str = sensor_info['description']+' at '+ sensor_info['location'] + global_atts = { + # Required + 'title' : title_str, + 'institution' : platform_info['institution'], + 'institution_url' : platform_info['institution_url'], + 'institution_dods_url' : platform_info['institution_dods_url'], + 'contact' : platform_info['contact'], + 'Conventions' : platform_info['conventions'], + # Required by Scout + 'format_category_code' : platform_info['format_category_code'], + 'institution_code' : platform_info['institution_code'], + 'platform_code' : platform_info['id'], + 'package_code' : sensor_info['id'], + # Required by Version tracking + 'format' : platform_info['format'], + 'seacoos_rt_version' : platform_info['seacoos_rt_version'], + # Recommended + '_FillValue' : n.nan, + 'missing_value' : n.nan, + 'source' : platform_info['source'], + 'references' : platform_info['references'], + 'metadata_url' : platform_info['metadata_url'], + 'history' : 'raw2proc using ' + sensor_info['process_module'], + 'comment' : 'File created using pycdf ' + \ + pycdf.pycdfVersion() + ' and ' + \ + pycdf.pycdfArrayPkg() + ' ' + \ + n.__version__, + 'project' : platform_info['project'], + 'project_url' : platform_info['project_url'], + # timeframe of data contained in file yyyy-mm-dd HH:MM:SS + # first date in monthly file + 'start_date' : data['dt'][0].strftime("%Y-%m-%d %H:%M:%S"), + # last date in monthly file + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + 'creation_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + 'modification_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + 'process_level' : 'level1', + # Custom + 'id_number' : platform_info['id_number'], + 'description' : platform_info['description'], + 'serial_number' : platform_info['serial_number'], + 'product_number' : platform_info['product_number'], + 'product_name' : platform_info['product_name'], + 'type' : platform_info['type'], + 'protocol_version' : platform_info['protocol_version'], + 'xmlns' : platform_info['xmlns'], + 'location' : platform_info['location'], + 'vertical_position': platform_info['vertical_position'], + 'owner' : platform_info['owner'], + 'package_id_number' : sensor_info['id_number'], + 'package_description' : sensor_info['description'], + 'package_serial_number' : sensor_info['serial_number'], + 'package_product_number' : sensor_info['product_number'], + 'package_product_name' : sensor_info['product_name'], + 'package_adr' : sensor_info['adr'], + 'package_protocol_version' : sensor_info['protocol_version'], + 'package_vertical_position' : sensor_info['vertical_position'], + } + + var_atts = { + # coordinate variables + 'time' : {'short_name': 'time', + 'long_name': 'Time', + 'standard_name': 'time', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + 'axis': 'T', + }, + 'lat' : {'short_name': 'lat', + 'long_name': 'Latitude', + 'standard_name': 'latitude', + 'reference':'geographic coordinates', + 'units': 'degrees_north', + 'valid_range':(-90.,90.), + 'axis': 'Y', + }, + 'lon' : {'short_name': 'lon', + 'long_name': 'Longtitude', + 'standard_name': 'longtitude', + 'reference':'geographic coordinates', + 'units': 'degrees_east', + 'valid_range':(-180.,180.), + 'axis': 'Y', + }, + 'z' : {'short_name': 'z', + 'long_name': 'Height', + 'standard_name': 'height', + 'reference':'zero at sea-surface', + 'positive' : 'up', + 'units': 'meters', + 'axis': 'Z', + }, + # data variables + 'ptime' : {'short_name': 'ptime', + 'long_name': 'Package Time', + 'standard_name': 'none', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + }, + 'session' : {'short_name': 'session', + 'long_name': 'Session ID', + 'standard_name': 'none', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + }, + 'psession' : {'short_name': 'ptime', + 'long_name': 'Package Session ID', + 'standard_name': 'none', + 'units': 'seconds since 1970-1-1 00:00:00 -0', # UTC + }, + 'record' : {'short_name': 'record', + 'long_name': 'Record Number', + 'standard_name': 'none', + 'units' : 'none', + }, + 'status': {'short_name' : 'status', + 'long_name': 'Platform Status Code', + 'standard_name': 'none', + 'units' : 'none', + 'value_map' : platform_info['status_map'], + }, + 'pstatus': {'short_name' : 'pstatus', + 'long_name': 'Package Status Code', + 'standard_name': 'none', + 'units' : 'none', + 'value_map' : sensor_info['status_map'], + }, + 'turbidity' : {'short_name' : 'turbidity', + 'long_name': sensor_info['turbidity_description'], + 'standard_name': 'turbidity', + 'units': 'none', + 'id_number' : sensor_info['turbidity_id'], + 'type' : sensor_info['turbidity_type'], + 'format' : sensor_info['turbidity_format'], + 'non_standard_units' : sensor_info['turbidity_units'], + 'range_min' : sensor_info['turbidity_range_min'], + 'range_max' : sensor_info['turbidity_range_max'], + }, + } + + # dimension names use tuple so order of initialization is maintained + dim_inits = ( + ('ntime', pycdf.NC.UNLIMITED), + ('nlat', 1), + ('nlon', 1), + ('nz', 1), + ) + + # using tuple of tuples so order of initialization is maintained + # using dict for attributes order of init not important + # use dimension names not values + # (varName, varType, (dimName1, [dimName2], ...)) + var_inits = ( + # coordinate variables + ('time', pycdf.NC.INT, ('ntime',)), + ('lat', pycdf.NC.FLOAT, ('nlat',)), + ('lon', pycdf.NC.FLOAT, ('nlon',)), + ('z', pycdf.NC.FLOAT, ('nz',)), + # data variables + ('ptime', pycdf.NC.INT, ('ntime',)), + ('session', pycdf.NC.INT, ('ntime',)), + ('psession', pycdf.NC.INT, ('ntime',)), + ('record', pycdf.NC.INT, ('ntime',)), + ('status', pycdf.NC.INT, ('ntime',)), + ('pstatus', pycdf.NC.INT, ('ntime',)), + ('turbidity', pycdf.NC.FLOAT, ('ntime',)), + ) + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # var data + var_data = ( + ('time', data['time'][i]), + ('lat', sensor_info['lat']), + ('lon', sensor_info['lat']), + ('z', sensor_info['elevation']), + ('ptime', data['ptime'][i]), + ('session', data['session'][i]), + ('psession', data['psession'][i]), + ('record', data['record'][i]), + ('status', data['status'][i]), + ('pstatus', data['pstatus'][i]), + ('turbidity', data['turbidity'][i]), + ) + + return (global_atts, var_atts, dim_inits, var_inits, var_data) + +def updater(platform_info, sensor_info, data): + # + global_atts = { + # update times of data contained in file (yyyy-mm-dd HH:MM:SS) + # last date in monthly file + 'end_date' : data['dt'][-1].strftime("%Y-%m-%d %H:%M:%S"), + 'release_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + # + 'modification_date' : nowDt.strftime("%Y-%m-%d %H:%M:%S"), + } + + # data variables + # update any variable attributes like range, min, max + var_atts = {} + + # subset data only to month being processed (see raw2proc.process()) + i = data['in'] + + # data + var_data = ( + ('time', data['time'][i]), + ('ptime', data['ptime'][i]), + ('session', data['session'][i]), + ('psession', data['psession'][i]), + ('record', data['record'][i]), + ('status', data['status'][i]), + ('pstatus', data['pstatus'][i]), + ('turbidity', data['turbidity'][i]), + ) + + return (global_atts, var_atts, var_data) +