root/spongenet/trunk/spongenet/parse.py

#!/usr/bin/env python

"""
Parse combined sponge data XML files.

Usage:

   > python parse.py path/to/xml/file
   > python parse.py -t
   > python parse.py --test

Test silent import.

>>> import parse
"""

__author__ = "Chris Calloway"
__email__ = "cbc@chriscalloway.org"
__copyright__ = "Copyright 2010 UNC-CH Department of Marine Science"
__license__ = "GPL2"

import sys
import os
import re
import glob
import hashlib
import doctest
import unittest
from StringIO import StringIO
import xml.etree.cElementTree as ET

USAGE = "\n".join(__doc__.splitlines()[3:8])
TEST_PATH = os.path.join("tests", "parse")
XMLNS_PATTERN = re.compile(r"(\{.*\})(.*)")
MISSING_VALUE = -9999


def _test():
    """
    Run doctests as unittest suite.

    Test silent import

    >>> from parse import _test
    """

    suite = []
    suite.append(doctest.DocTestSuite())
    suite = unittest.TestSuite(suite)
    unittest.TextTestRunner().run(suite)

    return


def xmldoc_path():
    """
    Return the XML document file path from the command line.

    Supply too few arguments on command line.

    >>> save_stdout = sys.stdout
    >>> temp_stdout = StringIO()
    >>> sys.stdout = temp_stdout
    >>> sys.argv = []
    >>> _xmldoc_path = xmldoc_path()
    >>> sys.stdout = save_stdout
    >>> USAGE == temp_stdout.getvalue()[:-1]
    True

    Supply too many arguments on the command line.

    >>> save_stdout = sys.stdout
    >>> temp_stdout = StringIO()
    >>> sys.stdout = temp_stdout
    >>> sys.argv = ["", "", "",]
    >>> _xmldoc_path = xmldoc_path()
    >>> sys.stdout = save_stdout
    >>> USAGE == temp_stdout.getvalue()[:-1]
    True

    Supply non-file argument.

    >>> save_stdout = sys.stdout
    >>> temp_stdout = StringIO()
    >>> sys.stdout = temp_stdout
    >>> _xmldoc_path = os.path.join(
    ...                    os.path.dirname(
    ...                        os.path.abspath(__file__)),
    ...                    TEST_PATH)
    >>> sys.argv = ["", _xmldoc_path]
    >>> _xmldoc_path = xmldoc_path()
    >>> sys.stdout = save_stdout
    >>> USAGE == temp_stdout.getvalue()[:-1]
    True

    Supply nonexistent file argument.

    >>> save_stdout = sys.stdout
    >>> temp_stdout = StringIO()
    >>> sys.stdout = temp_stdout
    >>> _xmldoc_path = os.path.join(
    ...                    os.path.dirname(
    ...                        os.path.abspath(__file__)),
    ...                    TEST_PATH, "xxxxx")
    >>> sys.argv = ["", _xmldoc_path]
    >>> _xmldoc_path = xmldoc_path()
    >>> sys.stdout = save_stdout
    >>> USAGE == temp_stdout.getvalue()[:-1]
    True

    Supply valid XML document path argument.

    >>> _xmldoc_path = os.path.join(
    ...                    os.path.dirname(
    ...                        os.path.abspath(__file__)),
    ...                    TEST_PATH, "xml","*","*.xml")
    >>> _xmldoc_path = glob.glob(_xmldoc_path)[0]
    >>> sys.argv = ["", _xmldoc_path]
    >>> _xmldoc_path == xmldoc_path()
    True
    """

    path = None
    try:
        if len(sys.argv) == 2:
            if sys.argv[1] == "-t" or sys.argv[1] == "--test":
                _test()
            else:
                path = sys.argv[1]
                if not os.path.exists(path):
                    raise IOError(path + \
                                  " does not exist.")
                elif not os.path.isfile(path):
                    raise IOError(path + \
                                  " is not a file.")
        else:
            raise IOError("Incorrect number of arguments supplied.")
    except IOError:
        print USAGE
    return path


def xmldoc(path):
    """
    Return the XML document as a string from a file at path.

    Get the test reference data.

    >>> xmlref = os.path.join(
    ...              os.path.dirname(
    ...                  os.path.abspath(__file__)),
    ...               TEST_PATH, "xmlref.py")
    >>> namespace = {}
    >>> execfile(xmlref, globals(), namespace)
    >>> xml_doc_lens = namespace["XML_DOC_LENS"]
    >>> xml_doc_md5s = namespace["XML_DOC_MD5S"]

    Pick a test document.

    >>> xmldoc_glob = os.path.join(
    ...                   os.path.dirname(
    ...                       os.path.abspath(__file__)),
    ...                   TEST_PATH, "xml","*","*.xml")
    >>> _xmldoc_path = glob.glob(xmldoc_glob)[0]
    >>> _xmldoc = xmldoc(_xmldoc_path)

    Verify the test document matches the reference data.

    >>> xmldoc_lines = _xmldoc.splitlines()
    >>> xmldoc_lines[0] == '<?xml version="1.0" encoding="utf-8"?>'
    True
    >>> xmldoc_lines[-1] == '</root>'
    True
    >>> len(xmldoc_lines) == xml_doc_lens[os.path.basename(_xmldoc_path)]
    True
    >>> doc_hash = hashlib.md5()
    >>> doc_hash.update(_xmldoc)
    >>> doc_hash.hexdigest() == xml_doc_md5s[os.path.basename(_xmldoc_path)]
    True
    """

    _xmldoc = None
    with open(path) as handle:
        _xmldoc = handle.readlines()

    return "".join(_xmldoc)


class Point(dict):
    """
    A data point for a sponge sensor sample."

    Instantiate a valid Point object.

    >>> xmldoc_glob = os.path.join(
    ...                   os.path.dirname(
    ...                       os.path.abspath(__file__)),
    ...                   TEST_PATH, "xml","*","*.xml")
    >>> _xmldoc_path = glob.glob(xmldoc_glob)[0]
    >>> _xmldoc = xmldoc(_xmldoc_path)
    >>> _data = Data(_xmldoc)
    >>> point = _data.devices[0].sensors["SN100-0"].points["Battery Voltage"]
    >>> len(point.keys()) == 8
    True
    >>> point["id"] != None
    True
    >>> point["descr"] != None
    True
    >>> point["type"] != None
    True
    >>> point["format"] != None
    True
    >>> point["unit"] != None
    True
    >>> point["rangemin"] != None
    True
    >>> point["rangemax"] != None
    True
    >>> point["value"] != None
    True
    """

    def __init__(self, point):
        super(Point, self).__init__({})
        for key, value in point.attrib.items():
            key = key.lower()
            self[key] = value
        for elem in point.getchildren():
            tag = XMLNS_PATTERN.search(elem.tag).groups()[1].lower()
            self[tag] = elem.text


class Sensor(dict):
    """
    A collection of data points for a sponge sensor sample.

    Instantiate a valid Sensor object.

    >>> xmldoc_glob = os.path.join(
    ...                   os.path.dirname(
    ...                       os.path.abspath(__file__)),
    ...                   TEST_PATH, "xml","*","*.xml")
    >>> _xmldoc_path = glob.glob(xmldoc_glob)[0]
    >>> _xmldoc = xmldoc(_xmldoc_path)
    >>> _data = Data(_xmldoc)
    >>> sensor = _data.devices[0].sensors["SN100-0"]
    >>> len(sensor.points.items())
    3
    >>> len(sensor.keys()) == 9
    True
    >>> sensor["id"] != None
    True
    >>> sensor["serialno"] != None
    True
    >>> sensor["prodno"] != None
    True
    >>> sensor["prodname"] != None
    True
    >>> sensor["descr"] != None
    True
    >>> sensor["adr"] != None
    True
    >>> sensor["protocolver"] != None
    True
    >>> sensor["verticalposition"] != None
    True
    >>> sensor["status"] != None
    True
    """

    def __init__(self, sensor, xmlns):
        super(Sensor, self).__init__({})
        for key, value in sensor.attrib.items():
            key = key.lower()
            self[key] = value
        for elem in sensor.getchildren():
            tag = XMLNS_PATTERN.search(elem.tag).groups()[1].lower()
            if tag == "parameters":
                self.points = [Point(point) for point
                                in elem.findall(xmlns + "Point")]
                self.points = dict([(point["descr"], point)
                                   for point in self.points])
            else:
                self[tag] = elem.text


class Device(dict):
    """
    Data from a collection of sponge sensors for a single time sample.

    Instantiate a valid Device object.

    >>> xmldoc_glob = os.path.join(
    ...                   os.path.dirname(
    ...                       os.path.abspath(__file__)),
    ...                   TEST_PATH, "xml","*","*.xml")
    >>> _xmldoc_path = glob.glob(xmldoc_glob)[0]
    >>> _xmldoc = xmldoc(_xmldoc_path)
    >>> _data = Data(_xmldoc)
    >>> device = _data.devices[0]
    >>> len(device.sensors.items())
    15
    >>> len(device.keys())
    16
    >>> device["id"] != None
    True
    >>> device["sessionid"] != None
    True
    >>> device["descr"] != None
    True
    >>> device["serialno"] != None
    True
    >>> device["prodno"] != None
    True
    >>> device["prodname"] != None
    True
    >>> device["devicetype"] != None
    True
    >>> device["protocolver"] != None
    True
    >>> device["time"] != None
    True
    >>> device["status"] != None
    True
    >>> device["location"] != None
    True
    >>> device["verticalposition"] != None
    True
    >>> device["owner"] != None
    True
    >>> device["recordnumber"] != None
    True
    >>> device["data_time"] != None
    True
    >>> device["data_sessionid"] != None
    True
    """

    def __init__(self, device, xmlns):
        super(Device, self).__init__({})
        for key, value in device.attrib.items():
            key = key.lower()
            self[key] = value
        for elem in device.getchildren():
            tag = XMLNS_PATTERN.search(elem.tag).groups()[1].lower()
            if tag == "siteinfo":
                for subelem in elem.getchildren():
                    tag = XMLNS_PATTERN.search(subelem.tag).groups()[1].lower()
                    self[tag] = subelem.text
            elif tag == "data":
                for key, value in elem.attrib.items():
                    key = key.lower()
                    if key == "time":
                        key = "data_time"
                    elif key == "sessionid":
                        key = "data_sessionid"
                    self[key] = value
                self.sensors = [Sensor(sensor, xmlns) for sensor
                                in elem.findall(xmlns + "SensorData")]
                self.sensors = dict([(sensor["id"], sensor)
                                    for sensor in self.sensors])
            else:
                self[tag] = elem.text


class Data(object):
    """
    A collection of sponge data samples from a collection of sensors.

    Instantiate a valid Data object.

    >>> xmldoc_glob = os.path.join(
    ...                   os.path.dirname(
    ...                       os.path.abspath(__file__)),
    ...                   TEST_PATH, "xml","*","*.xml")
    >>> _xmldoc_path = glob.glob(xmldoc_glob)[0]
    >>> _xmldoc = xmldoc(_xmldoc_path)
    >>> _data = Data(_xmldoc)
    >>> _data.xmlns
    '{http://www.aadi.no/RTOutSchema}'
    >>> len(_data.devices)
    50
    """

    def __init__(self, _xmldoc):
        """
        Initialize a new sponge data tree.
        """

        tree = ET.XML(_xmldoc)
        self.xmlns = XMLNS_PATTERN.search(
                         tree.getchildren()[0].tag).groups()[0]
        self.devices = [Device(device, self.xmlns) for device
                        in tree.findall(self.xmlns + "Device")]


def _main():
    """
    Run module as script.

    Test silent import.

    >>> from parse import _main
    """

    data = None

    _xmldoc_path = xmldoc_path()
    if _xmldoc_path:
        _xmldoc = xmldoc(_xmldoc_path)
        data = Data(_xmldoc)

    return data

if __name__ == "__main__":
    DATA = _main()
    print "DATA =", DATA