Index: spongenet/trunk/spongenet/query.py =================================================================== --- spongenet/trunk/spongenet/query.py (revision 380) +++ spongenet/trunk/spongenet/query.py (revision 381) @@ -20,8 +20,12 @@ __license__ = "GPL2" +import warnings +warnings.simplefilter("ignore", DeprecationWarning) + import sys import os import glob import csv +import warnings from pycdf import CDF, CDFError import doctest @@ -232,123 +236,123 @@ >>> platform == "platform" True - >>> packages == (('system', ('voltage', + >>> packages == (('system', ['voltage', ... 'memory', - ... 'interval', )), - ... ('turbidity', ('turbidity', )), - ... ('optode_127', ('o2concentration', - ... 'airsaturation', - ... 'temperature', - ... 'calphase', - ... 'tcphase', - ... 'c1rph', - ... 'c2rph', - ... 'c1amp', - ... 'c2amp', - ... 'rawtemp', )), - ... ('optode_167', ('o2concentration', - ... 'airsaturation', - ... 'temperature', - ... 'calphase', - ... 'tcphase', - ... 'c1rph', - ... 'c2rph', - ... 'c1amp', - ... 'c2amp', - ... 'rawtemp', )), - ... ('optode_169', ('o2concentration', - ... 'airsaturation', - ... 'temperature', - ... 'calphase', - ... 'tcphase', - ... 'c1rph', - ... 'c2rph', - ... 'c1amp', - ... 'c2amp', - ... 'rawtemp', )), - ... ('optode_170', ('o2concentration', - ... 'airsaturation', - ... 'temperature', - ... 'calphase', - ... 'tcphase', - ... 'c1rph', - ... 'c2rph', - ... 'c1amp', - ... 'c2amp', - ... 'rawtemp', )), - ... ('optode_171', ('o2concentration', - ... 'airsaturation', - ... 'temperature', - ... 'calphase', - ... 'tcphase', - ... 'c1rph', - ... 'c2rph', - ... 'c1amp', - ... 'c2amp', - ... 'rawtemp', )), - ... ('optode_172', ('o2concentration', - ... 'airsaturation', - ... 'temperature', - ... 'calphase', - ... 'tcphase', - ... 'c1rph', - ... 'c2rph', - ... 'c1amp', - ... 'c2amp', - ... 'rawtemp', )), - ... ('optode_173', ('o2concentration', - ... 'airsaturation', - ... 'temperature', - ... 'calphase', - ... 'tcphase', - ... 'c1rph', - ... 'c2rph', - ... 'c1amp', - ... 'c2amp', - ... 'rawtemp', )), - ... ('optode_175', ('o2concentration', - ... 'airsaturation', - ... 'temperature', - ... 'calphase', - ... 'tcphase', - ... 'c1rph', - ... 'c2rph', - ... 'c1amp', - ... 'c2amp', - ... 'rawtemp', )), - ... ('optode_176', ('o2concentration', - ... 'airsaturation', - ... 'temperature', - ... 'calphase', - ... 'tcphase', - ... 'c1rph', - ... 'c2rph', - ... 'c1amp', - ... 'c2amp', - ... 'rawtemp', )), - ... ('optode_178', ('o2concentration', - ... 'airsaturation', - ... 'temperature', - ... 'calphase', - ... 'tcphase', - ... 'c1rph', - ... 'c2rph', - ... 'c1amp', - ... 'c2amp', - ... 'rawtemp', )), - ... ('optode_179', ('o2concentration', - ... 'airsaturation', - ... 'temperature', - ... 'calphase', - ... 'tcphase', - ... 'c1rph', - ... 'c2rph', - ... 'c1amp', - ... 'c2amp', - ... 'rawtemp', )), - ... ('conductivity', ('conductivity', - ... 'temperature', )), - ... ('pressure', ('pressure', - ... 'temperature', )), - ... ('current', ('abs_speed', + ... 'interval', ]), + ... ('turbidity', ['turbidity', ]), + ... ('optode_127', ['o2concentration', + ... 'airsaturation', + ... 'temperature', + ... 'calphase', + ... 'tcphase', + ... 'c1rph', + ... 'c2rph', + ... 'c1amp', + ... 'c2amp', + ... 'rawtemp', ]), + ... ('optode_167', ['o2concentration', + ... 'airsaturation', + ... 'temperature', + ... 'calphase', + ... 'tcphase', + ... 'c1rph', + ... 'c2rph', + ... 'c1amp', + ... 'c2amp', + ... 'rawtemp', ]), + ... ('optode_169', ['o2concentration', + ... 'airsaturation', + ... 'temperature', + ... 'calphase', + ... 'tcphase', + ... 'c1rph', + ... 'c2rph', + ... 'c1amp', + ... 'c2amp', + ... 'rawtemp', ]), + ... ('optode_170', ['o2concentration', + ... 'airsaturation', + ... 'temperature', + ... 'calphase', + ... 'tcphase', + ... 'c1rph', + ... 'c2rph', + ... 'c1amp', + ... 'c2amp', + ... 'rawtemp', ]), + ... ('optode_171', ['o2concentration', + ... 'airsaturation', + ... 'temperature', + ... 'calphase', + ... 'tcphase', + ... 'c1rph', + ... 'c2rph', + ... 'c1amp', + ... 'c2amp', + ... 'rawtemp', ]), + ... ('optode_172', ['o2concentration', + ... 'airsaturation', + ... 'temperature', + ... 'calphase', + ... 'tcphase', + ... 'c1rph', + ... 'c2rph', + ... 'c1amp', + ... 'c2amp', + ... 'rawtemp', ]), + ... ('optode_173', ['o2concentration', + ... 'airsaturation', + ... 'temperature', + ... 'calphase', + ... 'tcphase', + ... 'c1rph', + ... 'c2rph', + ... 'c1amp', + ... 'c2amp', + ... 'rawtemp', ]), + ... ('optode_175', ['o2concentration', + ... 'airsaturation', + ... 'temperature', + ... 'calphase', + ... 'tcphase', + ... 'c1rph', + ... 'c2rph', + ... 'c1amp', + ... 'c2amp', + ... 'rawtemp', ]), + ... ('optode_176', ['o2concentration', + ... 'airsaturation', + ... 'temperature', + ... 'calphase', + ... 'tcphase', + ... 'c1rph', + ... 'c2rph', + ... 'c1amp', + ... 'c2amp', + ... 'rawtemp', ]), + ... ('optode_178', ['o2concentration', + ... 'airsaturation', + ... 'temperature', + ... 'calphase', + ... 'tcphase', + ... 'c1rph', + ... 'c2rph', + ... 'c1amp', + ... 'c2amp', + ... 'rawtemp', ]), + ... ('optode_179', ['o2concentration', + ... 'airsaturation', + ... 'temperature', + ... 'calphase', + ... 'tcphase', + ... 'c1rph', + ... 'c2rph', + ... 'c1amp', + ... 'c2amp', + ... 'rawtemp', ]), + ... ('conductivity', ['conductivity', + ... 'temperature', ]), + ... ('pressure', ['pressure', + ... 'temperature', ]), + ... ('current', ['abs_speed', ... 'direction', ... 'v', @@ -359,5 +363,5 @@ ... 'std_speed', ... 'strength', - ... 'pings', )), + ... 'pings', ]), ... ) True @@ -395,8 +399,7 @@ columns = ["time", ] - columns.extend([".".join([package, variable]) - for package, variables in packages - for variable in variables]) - + varmap = [("time", None), ] + + # Get all the column headings and variable arrays for package, variables in packages: ncpath = ncfile_pattern % {"location": location, @@ -406,17 +409,40 @@ ncpath = os.path.join(ncdir, ncpath) try: + variables.append("time") cdf = CDF(ncpath) for variable in variables: - cdf.var(variable).get() + var = ".".join([package, variable]) + try: + varmap.append((var, cdf.var(variable).get())) + if variable != "time": + columns.append(var) + except CDFError: + print "Bypassing variable", var except CDFError: - pass - + print "Bypassing package", package + varmap = dict(varmap) + + # Sanity check the time arrays + time_list = [var for var in varmap.keys() if var.endswith(".time")] + time_list = zip(time_list[:-1],time_list[1:]) + for time0,time1 in time_list: + if any(varmap[time0] != varmap[time1]): + raise CDFError("Time arrays are not equivalent.") + varmap["time"] = varmap[time0] + + # Create the rows. + rows = [[] for time in varmap["time"]] + for column in columns: + for pos in range(len(rows)): + rows[pos].append(varmap[column][pos]) + + # Output the CSV file. handle = open(csvpath, "wb") - writer = csv.DictWriter(handle, columns) - headers = dict([(column,column) for column in columns]) - writer.writerow(headers) + writer = csv.writer(handle) + writer.writerow(columns) + writer.writerows(rows) handle.close() - return columns + return @@ -461,6 +487,11 @@ path = os.path.join(csvdir, path) if not os.path.exists(path): - create(path, ncdir, ncfile_pattern, - location, platform, month, packages) + print "=" * 40 + print "Creating", path + try: + create(path, ncdir, ncfile_pattern, + location, platform, month, packages) + except CDFError: + print "Time array mismatch amongst", month, "packages." elif not os.path.isfile(path): raise IOError("CSV file name " + \ @@ -469,4 +500,6 @@ elif month == months[-1]: # Always create last month + print "=" * 40 + print "Creating", path create(path + "new", ncdir, ncfile_pattern, location, platform, month, packages) Index: spongenet/trunk/spongenet/tests/query/bad_config.py =================================================================== --- spongenet/trunk/spongenet/tests/query/bad_config.py (revision 380) +++ spongenet/trunk/spongenet/tests/query/bad_config.py (revision 381) @@ -17,9 +17,9 @@ LOCATION = "location" PLATFORM = "platform" -PACKAGES = (('system', ('voltage', +PACKAGES = (('system', ['voltage', 'memory', - 'interval', )), - ('turbidity', ('turbidity', )), - ('optode_127', ('o2concentration', + 'interval', ]), + ('turbidity', ['turbidity', ]), + ('optode_127', ['o2concentration', 'airsaturation', 'temperature', @@ -30,6 +30,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_167', ('o2concentration', + 'rawtemp', ]), + ('optode_167', ['o2concentration', 'airsaturation', 'temperature', @@ -40,6 +40,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_169', ('o2concentration', + 'rawtemp', ]), + ('optode_169', ['o2concentration', 'airsaturation', 'temperature', @@ -50,6 +50,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_170', ('o2concentration', + 'rawtemp', ]), + ('optode_170', ['o2concentration', 'airsaturation', 'temperature', @@ -60,6 +60,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_171', ('o2concentration', + 'rawtemp', ]), + ('optode_171', ['o2concentration', 'airsaturation', 'temperature', @@ -70,6 +70,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_172', ('o2concentration', + 'rawtemp', ]), + ('optode_172', ['o2concentration', 'airsaturation', 'temperature', @@ -80,6 +80,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_173', ('o2concentration', + 'rawtemp', ]), + ('optode_173', ['o2concentration', 'airsaturation', 'temperature', @@ -90,6 +90,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_175', ('o2concentration', + 'rawtemp', ]), + ('optode_175', ['o2concentration', 'airsaturation', 'temperature', @@ -100,6 +100,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_176', ('o2concentration', + 'rawtemp', ]), + ('optode_176', ['o2concentration', 'airsaturation', 'temperature', @@ -110,6 +110,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_178', ('o2concentration', + 'rawtemp', ]), + ('optode_178', ['o2concentration', 'airsaturation', 'temperature', @@ -120,6 +120,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_179', ('o2concentration', + 'rawtemp', ]), + ('optode_179', ['o2concentration', 'airsaturation', 'temperature', @@ -130,10 +130,10 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('conductivity', ('conductivity', - 'temperature', )), - ('pressure', ('pressure', - 'temperature', )), - ('current', ('abs_speed', + 'rawtemp', ]), + ('conductivity', ['conductivity', + 'temperature', ]), + ('pressure', ['pressure', + 'temperature', ]), + ('current', ['abs_speed', 'direction', 'v', @@ -144,4 +144,4 @@ 'std_speed', 'strength', - 'pings', )), + 'pings', ]), ) Index: spongenet/trunk/spongenet/tests/query/config.py =================================================================== --- spongenet/trunk/spongenet/tests/query/config.py (revision 380) +++ spongenet/trunk/spongenet/tests/query/config.py (revision 381) @@ -16,9 +16,9 @@ LOCATION = "location" PLATFORM = "platform" -PACKAGES = (('system', ('voltage', +PACKAGES = (('system', ['voltage', 'memory', - 'interval', )), - ('turbidity', ('turbidity', )), - ('optode_127', ('o2concentration', + 'interval', ]), + ('turbidity', ['turbidity', ]), + ('optode_127', ['o2concentration', 'airsaturation', 'temperature', @@ -29,6 +29,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_167', ('o2concentration', + 'rawtemp', ]), + ('optode_167', ['o2concentration', 'airsaturation', 'temperature', @@ -39,6 +39,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_169', ('o2concentration', + 'rawtemp', ]), + ('optode_169', ['o2concentration', 'airsaturation', 'temperature', @@ -49,6 +49,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_170', ('o2concentration', + 'rawtemp', ]), + ('optode_170', ['o2concentration', 'airsaturation', 'temperature', @@ -59,6 +59,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_171', ('o2concentration', + 'rawtemp', ]), + ('optode_171', ['o2concentration', 'airsaturation', 'temperature', @@ -69,6 +69,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_172', ('o2concentration', + 'rawtemp', ]), + ('optode_172', ['o2concentration', 'airsaturation', 'temperature', @@ -79,6 +79,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_173', ('o2concentration', + 'rawtemp', ]), + ('optode_173', ['o2concentration', 'airsaturation', 'temperature', @@ -89,6 +89,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_175', ('o2concentration', + 'rawtemp', ]), + ('optode_175', ['o2concentration', 'airsaturation', 'temperature', @@ -99,6 +99,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_176', ('o2concentration', + 'rawtemp', ]), + ('optode_176', ['o2concentration', 'airsaturation', 'temperature', @@ -109,6 +109,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_178', ('o2concentration', + 'rawtemp', ]), + ('optode_178', ['o2concentration', 'airsaturation', 'temperature', @@ -119,6 +119,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_179', ('o2concentration', + 'rawtemp', ]), + ('optode_179', ['o2concentration', 'airsaturation', 'temperature', @@ -129,10 +129,10 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('conductivity', ('conductivity', - 'temperature', )), - ('pressure', ('pressure', - 'temperature', )), - ('current', ('abs_speed', + 'rawtemp', ]), + ('conductivity', ['conductivity', + 'temperature', ]), + ('pressure', ['pressure', + 'temperature', ]), + ('current', ['abs_speed', 'direction', 'v', @@ -143,4 +143,4 @@ 'std_speed', 'strength', - 'pings', )), + 'pings', ]), ) Index: spongenet/trunk/spongenet/tests/query/incomplete_config.py =================================================================== --- spongenet/trunk/spongenet/tests/query/incomplete_config.py (revision 380) +++ spongenet/trunk/spongenet/tests/query/incomplete_config.py (revision 381) @@ -16,9 +16,9 @@ LOCATION = "location" PLATFORM = "platform" -PACKAGES = (('system', ('voltage', +PACKAGES = (('system', ['voltage', 'memory', - 'interval', )), - ('turbidity', ('turbidity', )), - ('optode_127', ('o2concentration', + 'interval', ]), + ('turbidity', ['turbidity', ]), + ('optode_127', ['o2concentration', 'airsaturation', 'temperature', @@ -29,6 +29,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_167', ('o2concentration', + 'rawtemp', ]), + ('optode_167', ['o2concentration', 'airsaturation', 'temperature', @@ -39,6 +39,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_169', ('o2concentration', + 'rawtemp', ]), + ('optode_169', ['o2concentration', 'airsaturation', 'temperature', @@ -49,6 +49,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_170', ('o2concentration', + 'rawtemp', ]), + ('optode_170', ['o2concentration', 'airsaturation', 'temperature', @@ -59,6 +59,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_171', ('o2concentration', + 'rawtemp', ]), + ('optode_171', ['o2concentration', 'airsaturation', 'temperature', @@ -69,6 +69,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_172', ('o2concentration', + 'rawtemp', ]), + ('optode_172', ['o2concentration', 'airsaturation', 'temperature', @@ -79,6 +79,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_173', ('o2concentration', + 'rawtemp', ]), + ('optode_173', ['o2concentration', 'airsaturation', 'temperature', @@ -89,6 +89,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_175', ('o2concentration', + 'rawtemp', ]), + ('optode_175', ['o2concentration', 'airsaturation', 'temperature', @@ -99,6 +99,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_176', ('o2concentration', + 'rawtemp', ]), + ('optode_176', ['o2concentration', 'airsaturation', 'temperature', @@ -109,6 +109,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_178', ('o2concentration', + 'rawtemp', ]), + ('optode_178', ['o2concentration', 'airsaturation', 'temperature', @@ -119,6 +119,6 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('optode_179', ('o2concentration', + 'rawtemp', ]), + ('optode_179', ['o2concentration', 'airsaturation', 'temperature', @@ -129,10 +129,10 @@ 'c1amp', 'c2amp', - 'rawtemp', )), - ('conductivity', ('conductivity', - 'temperature', )), - ('pressure', ('pressure', - 'temperature', )), - ('current', ('abs_speed', + 'rawtemp', ]), + ('conductivity', ['conductivity', + 'temperature', ]), + ('pressure', ['pressure', + 'temperature', ]), + ('current', ['abs_speed', 'direction', 'v', @@ -143,4 +143,4 @@ 'std_speed', 'strength', - 'pings', )), + 'pings', ]), )