Index: spongenet/trunk/spongenet/query.py =================================================================== --- spongenet/trunk/spongenet/query.py (revision 378) +++ spongenet/trunk/spongenet/query.py (revision 380) @@ -23,7 +23,6 @@ import os import glob -import zipfile -import shutil -import fileinput +import csv +from pycdf import CDF, CDFError import doctest import unittest @@ -227,6 +226,5 @@ "[0-9][0-9][0-9][0-9]_[0-9][0-9].nc" True - >>> csvfile_pattern == "%(location)s_%(platform)s_%(package)s_" \ - "[0-9][0-9][0-9][0-9]_[0-9][0-9].csv" + >>> csvfile_pattern == "%(location)s_%(platform)s_%(month)s.csv" True >>> location == "location" @@ -234,20 +232,132 @@ >>> platform == "platform" True - >>> packages == ('system', - ... 'turbidity', - ... 'optode_127', - ... 'optode_167', - ... 'optode_169', - ... 'optode_170', - ... 'optode_171', - ... 'optode_172', - ... 'optode_173', - ... 'optode_175', - ... 'optode_176', - ... 'optode_178', - ... 'optode_179', - ... 'conductivity', - ... 'pressure', - ... 'current', + >>> packages == (('system', ('voltage', + ... 'memory', + ... 'interval', )), + ... ('turbidity', ('turbidity', )), + ... ('optode_127', ('o2concentration', + ... 'airsaturation', + ... 'temperature', + ... 'calphase', + ... 'tcphase', + ... 'c1rph', + ... 'c2rph', + ... 'c1amp', + ... 'c2amp', + ... 'rawtemp', )), + ... ('optode_167', ('o2concentration', + ... 'airsaturation', + ... 'temperature', + ... 'calphase', + ... 'tcphase', + ... 'c1rph', + ... 'c2rph', + ... 'c1amp', + ... 'c2amp', + ... 'rawtemp', )), + ... ('optode_169', ('o2concentration', + ... 'airsaturation', + ... 'temperature', + ... 'calphase', + ... 'tcphase', + ... 'c1rph', + ... 'c2rph', + ... 'c1amp', + ... 'c2amp', + ... 'rawtemp', )), + ... ('optode_170', ('o2concentration', + ... 'airsaturation', + ... 'temperature', + ... 'calphase', + ... 'tcphase', + ... 'c1rph', + ... 'c2rph', + ... 'c1amp', + ... 'c2amp', + ... 'rawtemp', )), + ... ('optode_171', ('o2concentration', + ... 'airsaturation', + ... 'temperature', + ... 'calphase', + ... 'tcphase', + ... 'c1rph', + ... 'c2rph', + ... 'c1amp', + ... 'c2amp', + ... 'rawtemp', )), + ... ('optode_172', ('o2concentration', + ... 'airsaturation', + ... 'temperature', + ... 'calphase', + ... 'tcphase', + ... 'c1rph', + ... 'c2rph', + ... 'c1amp', + ... 'c2amp', + ... 'rawtemp', )), + ... ('optode_173', ('o2concentration', + ... 'airsaturation', + ... 'temperature', + ... 'calphase', + ... 'tcphase', + ... 'c1rph', + ... 'c2rph', + ... 'c1amp', + ... 'c2amp', + ... 'rawtemp', )), + ... ('optode_175', ('o2concentration', + ... 'airsaturation', + ... 'temperature', + ... 'calphase', + ... 'tcphase', + ... 'c1rph', + ... 'c2rph', + ... 'c1amp', + ... 'c2amp', + ... 'rawtemp', )), + ... ('optode_176', ('o2concentration', + ... 'airsaturation', + ... 'temperature', + ... 'calphase', + ... 'tcphase', + ... 'c1rph', + ... 'c2rph', + ... 'c1amp', + ... 'c2amp', + ... 'rawtemp', )), + ... ('optode_178', ('o2concentration', + ... 'airsaturation', + ... 'temperature', + ... 'calphase', + ... 'tcphase', + ... 'c1rph', + ... 'c2rph', + ... 'c1amp', + ... 'c2amp', + ... 'rawtemp', )), + ... ('optode_179', ('o2concentration', + ... 'airsaturation', + ... 'temperature', + ... 'calphase', + ... 'tcphase', + ... 'c1rph', + ... 'c2rph', + ... 'c1amp', + ... 'c2amp', + ... 'rawtemp', )), + ... ('conductivity', ('conductivity', + ... 'temperature', )), + ... ('pressure', ('pressure', + ... 'temperature', )), + ... ('current', ('abs_speed', + ... 'direction', + ... 'v', + ... 'u', + ... 'heading', + ... 'tiltx', + ... 'tilty', + ... 'std_speed', + ... 'strength', + ... 'pings', )), ... ) True @@ -278,4 +388,37 @@ +def create(csvpath, ncdir, ncfile_pattern, + location, platform, month, packages): + """ + Create a specific month of CSV spreadsheet from sponge data NetCDF files. + """ + + columns = ["time", ] + columns.extend([".".join([package, variable]) + for package, variables in packages + for variable in variables]) + + for package, variables in packages: + ncpath = ncfile_pattern % {"location": location, + "platform": platform, + "package": package, } + ncpath = ncpath[:-34] + month + ".nc" + ncpath = os.path.join(ncdir, ncpath) + try: + cdf = CDF(ncpath) + for variable in variables: + cdf.var(variable).get() + except CDFError: + pass + + handle = open(csvpath, "wb") + writer = csv.DictWriter(handle, columns) + headers = dict([(column,column) for column in columns]) + writer.writerow(headers) + handle.close() + + return columns + + def query(ncdir, csvdir, ncfile_pattern, csvfile_pattern, @@ -284,4 +427,50 @@ Create CSV spreadsheets from sponge data NetCDF files. """ + + # Find all monthly NetCDF files for location/platform + ncfile_patterns = [(package, + ncfile_pattern % {"location":location, + "platform":platform, + "package":package, }, + ) for package, variables in packages] + paths = dict([(package, + [month + for month in glob.glob(os.path.join(ncdir, pattern)) + if os.path.isfile(month)], + ) for package, pattern in ncfile_patterns]) + + # Find all months for location/platform + months = list(set([os.path.splitext(path)[0][-7:] + for package, variables in packages + for path in paths[package]])) + months.sort() + + # Create a directory for monthly CSV files. + if not os.path.exists(csvdir): + os.mkdir(csvdir, 0755) + elif not os.path.isdir(csvdir): + raise IOError("CSV directory name " + \ + csvdir + \ + " exists and is not a directory.") + + # Create a CSV file for each month + for month in months: + path = csvfile_pattern % {"location": location, + "platform": platform, + "month": month} + path = os.path.join(csvdir, path) + if not os.path.exists(path): + create(path, ncdir, ncfile_pattern, + location, platform, month, packages) + elif not os.path.isfile(path): + raise IOError("CSV file name " + \ + path + \ + " exists and is not a file.") + elif month == months[-1]: + # Always create last month + create(path + "new", ncdir, ncfile_pattern, + location, platform, month, packages) + os.remove(path) + os.rename(path + "new", path) return @@ -315,4 +504,5 @@ else: print USAGE + return Index: spongenet/trunk/spongenet/tests/query/bad_config.py =================================================================== --- spongenet/trunk/spongenet/tests/query/bad_config.py (revision 378) +++ spongenet/trunk/spongenet/tests/query/bad_config.py (revision 380) @@ -14,23 +14,134 @@ NCFILE_PATTERN = "%(location)s_%(platform)s_%(package)s_" \ "[0-9][0-9][0-9][0-9]_[0-9][0-9].nc -CSVFILE_PATTERN = "%(location)s_%(platform)s_%(package)s_" \ - "[0-9][0-9][0-9][0-9]_[0-9][0-9].csv" +CSVFILE_PATTERN = "%(location)s_%(platform)s_%(month)s.csv" LOCATION = "location" PLATFORM = "platform" -PACKAGES = ('system', - 'turbidity', - 'optode_127', - 'optode_167', - 'optode_169', - 'optode_170', - 'optode_171', - 'optode_172', - 'optode_173', - 'optode_175', - 'optode_176', - 'optode_178', - 'optode_179', - 'conductivity', - 'pressure', - 'current', +PACKAGES = (('system', ('voltage', + 'memory', + 'interval', )), + ('turbidity', ('turbidity', )), + ('optode_127', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_167', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_169', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_170', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_171', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_172', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_173', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_175', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_176', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_178', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_179', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('conductivity', ('conductivity', + 'temperature', )), + ('pressure', ('pressure', + 'temperature', )), + ('current', ('abs_speed', + 'direction', + 'v', + 'u', + 'heading', + 'tiltx', + 'tilty', + 'std_speed', + 'strength', + 'pings', )), ) Index: spongenet/trunk/spongenet/tests/query/config.py =================================================================== --- spongenet/trunk/spongenet/tests/query/config.py (revision 378) +++ spongenet/trunk/spongenet/tests/query/config.py (revision 380) @@ -13,23 +13,134 @@ NCFILE_PATTERN = "%(location)s_%(platform)s_%(package)s_" \ "[0-9][0-9][0-9][0-9]_[0-9][0-9].nc" -CSVFILE_PATTERN = "%(location)s_%(platform)s_%(package)s_" \ - "[0-9][0-9][0-9][0-9]_[0-9][0-9].csv" +CSVFILE_PATTERN = "%(location)s_%(platform)s_%(month)s.csv" LOCATION = "location" PLATFORM = "platform" -PACKAGES = ('system', - 'turbidity', - 'optode_127', - 'optode_167', - 'optode_169', - 'optode_170', - 'optode_171', - 'optode_172', - 'optode_173', - 'optode_175', - 'optode_176', - 'optode_178', - 'optode_179', - 'conductivity', - 'pressure', - 'current', +PACKAGES = (('system', ('voltage', + 'memory', + 'interval', )), + ('turbidity', ('turbidity', )), + ('optode_127', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_167', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_169', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_170', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_171', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_172', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_173', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_175', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_176', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_178', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_179', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('conductivity', ('conductivity', + 'temperature', )), + ('pressure', ('pressure', + 'temperature', )), + ('current', ('abs_speed', + 'direction', + 'v', + 'u', + 'heading', + 'tiltx', + 'tilty', + 'std_speed', + 'strength', + 'pings', )), ) Index: spongenet/trunk/spongenet/tests/query/incomplete_config.py =================================================================== --- spongenet/trunk/spongenet/tests/query/incomplete_config.py (revision 378) +++ spongenet/trunk/spongenet/tests/query/incomplete_config.py (revision 380) @@ -13,23 +13,134 @@ NCFILE_PATTERN = "%(location)s_%(platform)s_%(package)s_" \ "[0-9][0-9][0-9][0-9]_[0-9][0-9].nc" -CSVFILE_PATTERN = "%(location)s_%(platform)s_%(package)s_" \ - "[0-9][0-9][0-9][0-9]_[0-9][0-9].csv" +CSVFILE_PATTERN = "%(location)s_%(platform)s_%(month)s.csv" LOCATION = "location" PLATFORM = "platform" -PACKAGES = ('system', - 'turbidity', - 'optode_127', - 'optode_167', - 'optode_169', - 'optode_170', - 'optode_171', - 'optode_172', - 'optode_173', - 'optode_175', - 'optode_176', - 'optode_178', - 'optode_179', - 'conductivity', - 'pressure', - 'current', +PACKAGES = (('system', ('voltage', + 'memory', + 'interval', )), + ('turbidity', ('turbidity', )), + ('optode_127', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_167', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_169', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_170', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_171', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_172', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_173', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_175', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_176', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_178', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('optode_179', ('o2concentration', + 'airsaturation', + 'temperature', + 'calphase', + 'tcphase', + 'c1rph', + 'c2rph', + 'c1amp', + 'c2amp', + 'rawtemp', )), + ('conductivity', ('conductivity', + 'temperature', )), + ('pressure', ('pressure', + 'temperature', )), + ('current', ('abs_speed', + 'direction', + 'v', + 'u', + 'heading', + 'tiltx', + 'tilty', + 'std_speed', + 'strength', + 'pings', )), )