Changeset 381
- Timestamp:
- 10/20/10 18:04:50
- Files:
-
- spongenet/trunk/spongenet/query.py (modified) (7 diffs)
- spongenet/trunk/spongenet/tests/query/bad_config.py (modified) (13 diffs)
- spongenet/trunk/spongenet/tests/query/config.py (modified) (13 diffs)
- spongenet/trunk/spongenet/tests/query/incomplete_config.py (modified) (13 diffs)
Legend:
- Unmodified
- Added
- Removed
- Modified
- Copied
- Moved
spongenet/trunk/spongenet/query.py
r380 r381 20 20 __license__ = "GPL2" 21 21 22 import warnings 23 warnings.simplefilter("ignore", DeprecationWarning) 24 22 25 import sys 23 26 import os 24 27 import glob 25 28 import csv 29 import warnings 26 30 from pycdf import CDF, CDFError 27 31 import doctest … … 232 236 >>> platform == "platform" 233 237 True 234 >>> packages == (('system', ('voltage',238 >>> packages == (('system', ['voltage', 235 239 ... 'memory', 236 ... 'interval', )),237 ... ('turbidity', ('turbidity', )),238 ... ('optode_127', ('o2concentration',239 ... 'airsaturation', 240 ... 'temperature', 241 ... 'calphase', 242 ... 'tcphase', 243 ... 'c1rph', 244 ... 'c2rph', 245 ... 'c1amp', 246 ... 'c2amp', 247 ... 'rawtemp', )),248 ... ('optode_167', ('o2concentration',249 ... 'airsaturation', 250 ... 'temperature', 251 ... 'calphase', 252 ... 'tcphase', 253 ... 'c1rph', 254 ... 'c2rph', 255 ... 'c1amp', 256 ... 'c2amp', 257 ... 'rawtemp', )),258 ... ('optode_169', ('o2concentration',259 ... 'airsaturation', 260 ... 'temperature', 261 ... 'calphase', 262 ... 'tcphase', 263 ... 'c1rph', 264 ... 'c2rph', 265 ... 'c1amp', 266 ... 'c2amp', 267 ... 'rawtemp', )),268 ... ('optode_170', ('o2concentration',269 ... 'airsaturation', 270 ... 'temperature', 271 ... 'calphase', 272 ... 'tcphase', 273 ... 'c1rph', 274 ... 'c2rph', 275 ... 'c1amp', 276 ... 'c2amp', 277 ... 'rawtemp', )),278 ... ('optode_171', ('o2concentration',279 ... 'airsaturation', 280 ... 'temperature', 281 ... 'calphase', 282 ... 'tcphase', 283 ... 'c1rph', 284 ... 'c2rph', 285 ... 'c1amp', 286 ... 'c2amp', 287 ... 'rawtemp', )),288 ... ('optode_172', ('o2concentration',289 ... 'airsaturation', 290 ... 'temperature', 291 ... 'calphase', 292 ... 'tcphase', 293 ... 'c1rph', 294 ... 'c2rph', 295 ... 'c1amp', 296 ... 'c2amp', 297 ... 'rawtemp', )),298 ... ('optode_173', ('o2concentration',299 ... 'airsaturation', 300 ... 'temperature', 301 ... 'calphase', 302 ... 'tcphase', 303 ... 'c1rph', 304 ... 'c2rph', 305 ... 'c1amp', 306 ... 'c2amp', 307 ... 'rawtemp', )),308 ... ('optode_175', ('o2concentration',309 ... 'airsaturation', 310 ... 'temperature', 311 ... 'calphase', 312 ... 'tcphase', 313 ... 'c1rph', 314 ... 'c2rph', 315 ... 'c1amp', 316 ... 'c2amp', 317 ... 'rawtemp', )),318 ... ('optode_176', ('o2concentration',319 ... 'airsaturation', 320 ... 'temperature', 321 ... 'calphase', 322 ... 'tcphase', 323 ... 'c1rph', 324 ... 'c2rph', 325 ... 'c1amp', 326 ... 'c2amp', 327 ... 'rawtemp', )),328 ... ('optode_178', ('o2concentration',329 ... 'airsaturation', 330 ... 'temperature', 331 ... 'calphase', 332 ... 'tcphase', 333 ... 'c1rph', 334 ... 'c2rph', 335 ... 'c1amp', 336 ... 'c2amp', 337 ... 'rawtemp', )),338 ... ('optode_179', ('o2concentration',339 ... 'airsaturation', 340 ... 'temperature', 341 ... 'calphase', 342 ... 'tcphase', 343 ... 'c1rph', 344 ... 'c2rph', 345 ... 'c1amp', 346 ... 'c2amp', 347 ... 'rawtemp', )),348 ... ('conductivity', ('conductivity',349 ... 'temperature', )),350 ... ('pressure', ('pressure',351 ... 'temperature', )),352 ... ('current', ('abs_speed',240 ... 'interval', ]), 241 ... ('turbidity', ['turbidity', ]), 242 ... ('optode_127', ['o2concentration', 243 ... 'airsaturation', 244 ... 'temperature', 245 ... 'calphase', 246 ... 'tcphase', 247 ... 'c1rph', 248 ... 'c2rph', 249 ... 'c1amp', 250 ... 'c2amp', 251 ... 'rawtemp', ]), 252 ... ('optode_167', ['o2concentration', 253 ... 'airsaturation', 254 ... 'temperature', 255 ... 'calphase', 256 ... 'tcphase', 257 ... 'c1rph', 258 ... 'c2rph', 259 ... 'c1amp', 260 ... 'c2amp', 261 ... 'rawtemp', ]), 262 ... ('optode_169', ['o2concentration', 263 ... 'airsaturation', 264 ... 'temperature', 265 ... 'calphase', 266 ... 'tcphase', 267 ... 'c1rph', 268 ... 'c2rph', 269 ... 'c1amp', 270 ... 'c2amp', 271 ... 'rawtemp', ]), 272 ... ('optode_170', ['o2concentration', 273 ... 'airsaturation', 274 ... 'temperature', 275 ... 'calphase', 276 ... 'tcphase', 277 ... 'c1rph', 278 ... 'c2rph', 279 ... 'c1amp', 280 ... 'c2amp', 281 ... 'rawtemp', ]), 282 ... ('optode_171', ['o2concentration', 283 ... 'airsaturation', 284 ... 'temperature', 285 ... 'calphase', 286 ... 'tcphase', 287 ... 'c1rph', 288 ... 'c2rph', 289 ... 'c1amp', 290 ... 'c2amp', 291 ... 'rawtemp', ]), 292 ... ('optode_172', ['o2concentration', 293 ... 'airsaturation', 294 ... 'temperature', 295 ... 'calphase', 296 ... 'tcphase', 297 ... 'c1rph', 298 ... 'c2rph', 299 ... 'c1amp', 300 ... 'c2amp', 301 ... 'rawtemp', ]), 302 ... ('optode_173', ['o2concentration', 303 ... 'airsaturation', 304 ... 'temperature', 305 ... 'calphase', 306 ... 'tcphase', 307 ... 'c1rph', 308 ... 'c2rph', 309 ... 'c1amp', 310 ... 'c2amp', 311 ... 'rawtemp', ]), 312 ... ('optode_175', ['o2concentration', 313 ... 'airsaturation', 314 ... 'temperature', 315 ... 'calphase', 316 ... 'tcphase', 317 ... 'c1rph', 318 ... 'c2rph', 319 ... 'c1amp', 320 ... 'c2amp', 321 ... 'rawtemp', ]), 322 ... ('optode_176', ['o2concentration', 323 ... 'airsaturation', 324 ... 'temperature', 325 ... 'calphase', 326 ... 'tcphase', 327 ... 'c1rph', 328 ... 'c2rph', 329 ... 'c1amp', 330 ... 'c2amp', 331 ... 'rawtemp', ]), 332 ... ('optode_178', ['o2concentration', 333 ... 'airsaturation', 334 ... 'temperature', 335 ... 'calphase', 336 ... 'tcphase', 337 ... 'c1rph', 338 ... 'c2rph', 339 ... 'c1amp', 340 ... 'c2amp', 341 ... 'rawtemp', ]), 342 ... ('optode_179', ['o2concentration', 343 ... 'airsaturation', 344 ... 'temperature', 345 ... 'calphase', 346 ... 'tcphase', 347 ... 'c1rph', 348 ... 'c2rph', 349 ... 'c1amp', 350 ... 'c2amp', 351 ... 'rawtemp', ]), 352 ... ('conductivity', ['conductivity', 353 ... 'temperature', ]), 354 ... ('pressure', ['pressure', 355 ... 'temperature', ]), 356 ... ('current', ['abs_speed', 353 357 ... 'direction', 354 358 ... 'v', … … 359 363 ... 'std_speed', 360 364 ... 'strength', 361 ... 'pings', )),365 ... 'pings', ]), 362 366 ... ) 363 367 True … … 395 399 396 400 columns = ["time", ] 397 columns.extend([".".join([package, variable]) 398 for package, variables in packages 399 for variable in variables]) 400 401 varmap = [("time", None), ] 402 403 # Get all the column headings and variable arrays 401 404 for package, variables in packages: 402 405 ncpath = ncfile_pattern % {"location": location, … … 406 409 ncpath = os.path.join(ncdir, ncpath) 407 410 try: 411 variables.append("time") 408 412 cdf = CDF(ncpath) 409 413 for variable in variables: 410 cdf.var(variable).get() 414 var = ".".join([package, variable]) 415 try: 416 varmap.append((var, cdf.var(variable).get())) 417 if variable != "time": 418 columns.append(var) 419 except CDFError: 420 print "Bypassing variable", var 411 421 except CDFError: 412 pass 413 422 print "Bypassing package", package 423 varmap = dict(varmap) 424 425 # Sanity check the time arrays 426 time_list = [var for var in varmap.keys() if var.endswith(".time")] 427 time_list = zip(time_list[:-1],time_list[1:]) 428 for time0,time1 in time_list: 429 if any(varmap[time0] != varmap[time1]): 430 raise CDFError("Time arrays are not equivalent.") 431 varmap["time"] = varmap[time0] 432 433 # Create the rows. 434 rows = [[] for time in varmap["time"]] 435 for column in columns: 436 for pos in range(len(rows)): 437 rows[pos].append(varmap[column][pos]) 438 439 # Output the CSV file. 414 440 handle = open(csvpath, "wb") 415 writer = csv. DictWriter(handle, columns)416 headers = dict([(column,column) for column in columns])417 writer.writerow (headers)441 writer = csv.writer(handle) 442 writer.writerow(columns) 443 writer.writerows(rows) 418 444 handle.close() 419 445 420 return columns446 return 421 447 422 448 … … 461 487 path = os.path.join(csvdir, path) 462 488 if not os.path.exists(path): 463 create(path, ncdir, ncfile_pattern, 464 location, platform, month, packages) 489 print "=" * 40 490 print "Creating", path 491 try: 492 create(path, ncdir, ncfile_pattern, 493 location, platform, month, packages) 494 except CDFError: 495 print "Time array mismatch amongst", month, "packages." 465 496 elif not os.path.isfile(path): 466 497 raise IOError("CSV file name " + \ … … 469 500 elif month == months[-1]: 470 501 # Always create last month 502 print "=" * 40 503 print "Creating", path 471 504 create(path + "new", ncdir, ncfile_pattern, 472 505 location, platform, month, packages) spongenet/trunk/spongenet/tests/query/bad_config.py
r380 r381 17 17 LOCATION = "location" 18 18 PLATFORM = "platform" 19 PACKAGES = (('system', ('voltage',19 PACKAGES = (('system', ['voltage', 20 20 'memory', 21 'interval', )),22 ('turbidity', ('turbidity', )),23 ('optode_127', ('o2concentration',21 'interval', ]), 22 ('turbidity', ['turbidity', ]), 23 ('optode_127', ['o2concentration', 24 24 'airsaturation', 25 25 'temperature', … … 30 30 'c1amp', 31 31 'c2amp', 32 'rawtemp', )),33 ('optode_167', ('o2concentration',32 'rawtemp', ]), 33 ('optode_167', ['o2concentration', 34 34 'airsaturation', 35 35 'temperature', … … 40 40 'c1amp', 41 41 'c2amp', 42 'rawtemp', )),43 ('optode_169', ('o2concentration',42 'rawtemp', ]), 43 ('optode_169', ['o2concentration', 44 44 'airsaturation', 45 45 'temperature', … … 50 50 'c1amp', 51 51 'c2amp', 52 'rawtemp', )),53 ('optode_170', ('o2concentration',52 'rawtemp', ]), 53 ('optode_170', ['o2concentration', 54 54 'airsaturation', 55 55 'temperature', … … 60 60 'c1amp', 61 61 'c2amp', 62 'rawtemp', )),63 ('optode_171', ('o2concentration',62 'rawtemp', ]), 63 ('optode_171', ['o2concentration', 64 64 'airsaturation', 65 65 'temperature', … … 70 70 'c1amp', 71 71 'c2amp', 72 'rawtemp', )),73 ('optode_172', ('o2concentration',72 'rawtemp', ]), 73 ('optode_172', ['o2concentration', 74 74 'airsaturation', 75 75 'temperature', … … 80 80 'c1amp', 81 81 'c2amp', 82 'rawtemp', )),83 ('optode_173', ('o2concentration',82 'rawtemp', ]), 83 ('optode_173', ['o2concentration', 84 84 'airsaturation', 85 85 'temperature', … … 90 90 'c1amp', 91 91 'c2amp', 92 'rawtemp', )),93 ('optode_175', ('o2concentration',92 'rawtemp', ]), 93 ('optode_175', ['o2concentration', 94 94 'airsaturation', 95 95 'temperature', … … 100 100 'c1amp', 101 101 'c2amp', 102 'rawtemp', )),103 ('optode_176', ('o2concentration',102 'rawtemp', ]), 103 ('optode_176', ['o2concentration', 104 104 'airsaturation', 105 105 'temperature', … … 110 110 'c1amp', 111 111 'c2amp', 112 'rawtemp', )),113 ('optode_178', ('o2concentration',112 'rawtemp', ]), 113 ('optode_178', ['o2concentration', 114 114 'airsaturation', 115 115 'temperature', … … 120 120 'c1amp', 121 121 'c2amp', 122 'rawtemp', )),123 ('optode_179', ('o2concentration',122 'rawtemp', ]), 123 ('optode_179', ['o2concentration', 124 124 'airsaturation', 125 125 'temperature', … … 130 130 'c1amp', 131 131 'c2amp', 132 'rawtemp', )),133 ('conductivity', ('conductivity',134 'temperature', )),135 ('pressure', ('pressure',136 'temperature', )),137 ('current', ('abs_speed',132 'rawtemp', ]), 133 ('conductivity', ['conductivity', 134 'temperature', ]), 135 ('pressure', ['pressure', 136 'temperature', ]), 137 ('current', ['abs_speed', 138 138 'direction', 139 139 'v', … … 144 144 'std_speed', 145 145 'strength', 146 'pings', )),146 'pings', ]), 147 147 ) spongenet/trunk/spongenet/tests/query/config.py
r380 r381 16 16 LOCATION = "location" 17 17 PLATFORM = "platform" 18 PACKAGES = (('system', ('voltage',18 PACKAGES = (('system', ['voltage', 19 19 'memory', 20 'interval', )),21 ('turbidity', ('turbidity', )),22 ('optode_127', ('o2concentration',20 'interval', ]), 21 ('turbidity', ['turbidity', ]), 22 ('optode_127', ['o2concentration', 23 23 'airsaturation', 24 24 'temperature', … … 29 29 'c1amp', 30 30 'c2amp', 31 'rawtemp', )),32 ('optode_167', ('o2concentration',31 'rawtemp', ]), 32 ('optode_167', ['o2concentration', 33 33 'airsaturation', 34 34 'temperature', … … 39 39 'c1amp', 40 40 'c2amp', 41 'rawtemp', )),42 ('optode_169', ('o2concentration',41 'rawtemp', ]), 42 ('optode_169', ['o2concentration', 43 43 'airsaturation', 44 44 'temperature', … … 49 49 'c1amp', 50 50 'c2amp', 51 'rawtemp', )),52 ('optode_170', ('o2concentration',51 'rawtemp', ]), 52 ('optode_170', ['o2concentration', 53 53 'airsaturation', 54 54 'temperature', … … 59 59 'c1amp', 60 60 'c2amp', 61 'rawtemp', )),62 ('optode_171', ('o2concentration',61 'rawtemp', ]), 62 ('optode_171', ['o2concentration', 63 63 'airsaturation', 64 64 'temperature', … … 69 69 'c1amp', 70 70 'c2amp', 71 'rawtemp', )),72 ('optode_172', ('o2concentration',71 'rawtemp', ]), 72 ('optode_172', ['o2concentration', 73 73 'airsaturation', 74 74 'temperature', … … 79 79 'c1amp', 80 80 'c2amp', 81 'rawtemp', )),82 ('optode_173', ('o2concentration',81 'rawtemp', ]), 82 ('optode_173', ['o2concentration', 83 83 'airsaturation', 84 84 'temperature', … … 89 89 'c1amp', 90 90 'c2amp', 91 'rawtemp', )),92 ('optode_175', ('o2concentration',91 'rawtemp', ]), 92 ('optode_175', ['o2concentration', 93 93 'airsaturation', 94 94 'temperature', … … 99 99 'c1amp', 100 100 'c2amp', 101 'rawtemp', )),102 ('optode_176', ('o2concentration',101 'rawtemp', ]), 102 ('optode_176', ['o2concentration', 103 103 'airsaturation', 104 104 'temperature', … … 109 109 'c1amp', 110 110 'c2amp', 111 'rawtemp', )),112 ('optode_178', ('o2concentration',111 'rawtemp', ]), 112 ('optode_178', ['o2concentration', 113 113 'airsaturation', 114 114 'temperature', … … 119 119 'c1amp', 120 120 'c2amp', 121 'rawtemp', )),122 ('optode_179', ('o2concentration',121 'rawtemp', ]), 122 ('optode_179', ['o2concentration', 123 123 'airsaturation', 124 124 'temperature', … … 129 129 'c1amp', 130 130 'c2amp', 131 'rawtemp', )),132 ('conductivity', ('conductivity',133 'temperature', )),134 ('pressure', ('pressure',135 'temperature', )),136 ('current', ('abs_speed',131 'rawtemp', ]), 132 ('conductivity', ['conductivity', 133 'temperature', ]), 134 ('pressure', ['pressure', 135 'temperature', ]), 136 ('current', ['abs_speed', 137 137 'direction', 138 138 'v', … … 143 143 'std_speed', 144 144 'strength', 145 'pings', )),145 'pings', ]), 146 146 ) spongenet/trunk/spongenet/tests/query/incomplete_config.py
r380 r381 16 16 LOCATION = "location" 17 17 PLATFORM = "platform" 18 PACKAGES = (('system', ('voltage',18 PACKAGES = (('system', ['voltage', 19 19 'memory', 20 'interval', )),21 ('turbidity', ('turbidity', )),22 ('optode_127', ('o2concentration',20 'interval', ]), 21 ('turbidity', ['turbidity', ]), 22 ('optode_127', ['o2concentration', 23 23 'airsaturation', 24 24 'temperature', … … 29 29 'c1amp', 30 30 'c2amp', 31 'rawtemp', )),32 ('optode_167', ('o2concentration',31 'rawtemp', ]), 32 ('optode_167', ['o2concentration', 33 33 'airsaturation', 34 34 'temperature', … … 39 39 'c1amp', 40 40 'c2amp', 41 'rawtemp', )),42 ('optode_169', ('o2concentration',41 'rawtemp', ]), 42 ('optode_169', ['o2concentration', 43 43 'airsaturation', 44 44 'temperature', … … 49 49 'c1amp', 50 50 'c2amp', 51 'rawtemp', )),52 ('optode_170', ('o2concentration',51 'rawtemp', ]), 52 ('optode_170', ['o2concentration', 53 53 'airsaturation', 54 54 'temperature', … … 59 59 'c1amp', 60 60 'c2amp', 61 'rawtemp', )),62 ('optode_171', ('o2concentration',61 'rawtemp', ]), 62 ('optode_171', ['o2concentration', 63 63 'airsaturation', 64 64 'temperature', … … 69 69 'c1amp', 70 70 'c2amp', 71 'rawtemp', )),72 ('optode_172', ('o2concentration',71 'rawtemp', ]), 72 ('optode_172', ['o2concentration', 73 73 'airsaturation', 74 74 'temperature', … … 79 79 'c1amp', 80 80 'c2amp', 81 'rawtemp', )),82 ('optode_173', ('o2concentration',81 'rawtemp', ]), 82 ('optode_173', ['o2concentration', 83 83 'airsaturation', 84 84 'temperature', … … 89 89 'c1amp', 90 90 'c2amp', 91 'rawtemp', )),92 ('optode_175', ('o2concentration',91 'rawtemp', ]), 92 ('optode_175', ['o2concentration', 93 93 'airsaturation', 94 94 'temperature', … … 99 99 'c1amp', 100 100 'c2amp', 101 'rawtemp', )),102 ('optode_176', ('o2concentration',101 'rawtemp', ]), 102 ('optode_176', ['o2concentration', 103 103 'airsaturation', 104 104 'temperature', … … 109 109 'c1amp', 110 110 'c2amp', 111 'rawtemp', )),112 ('optode_178', ('o2concentration',111 'rawtemp', ]), 112 ('optode_178', ['o2concentration', 113 113 'airsaturation', 114 114 'temperature', … … 119 119 'c1amp', 120 120 'c2amp', 121 'rawtemp', )),122 ('optode_179', ('o2concentration',121 'rawtemp', ]), 122 ('optode_179', ['o2concentration', 123 123 'airsaturation', 124 124 'temperature', … … 129 129 'c1amp', 130 130 'c2amp', 131 'rawtemp', )),132 ('conductivity', ('conductivity',133 'temperature', )),134 ('pressure', ('pressure',135 'temperature', )),136 ('current', ('abs_speed',131 'rawtemp', ]), 132 ('conductivity', ['conductivity', 133 'temperature', ]), 134 ('pressure', ['pressure', 135 'temperature', ]), 136 ('current', ['abs_speed', 137 137 'direction', 138 138 'v', … … 143 143 'std_speed', 144 144 'strength', 145 'pings', )),145 'pings', ]), 146 146 )