Index: sodar/trunk/sodar/processedData.py =================================================================== --- sodar/trunk/sodar/processedData.py (revision 72) +++ sodar/trunk/sodar/processedData.py (revision 73) @@ -3,4 +3,11 @@ Classes to handle processed sodar data samples. +A list of sample data. + +A list of dictionaries, each with an item for header and body data. + +A list of dictionaries of sample data, each with a two items: + a header dictionary, and + a body list of alitude dictionaries. """ @@ -16,15 +23,17 @@ class ProcessedData(list): - """Processed daily sodar file data.""" + """Processed daily sodar file data. + + A list of rawData.Sample deep copies. + """ def __init__(self, data): """Prepare raw sodar daily data for analysis.""" super(ProcessedData, self).__init__() - self.data = data + self.extend([sample.data() for sample in data]) self._normalize() def _normalize(self): """Clean up data for analysis.""" - self._copy() self._convert() self._stamp() @@ -43,12 +52,17 @@ # compute plotting parameters - def _copy(self): - """Create a deep copy as a list of Sample copies.""" - self.extend([sample._copy() for sample in self.data]) - def _convert(self): - """Convert to numbers and correct for invalid values.""" + """Convert data to numbers and correct for invalid values.""" INVALID = "-9999" for sample in self: + # convert header data to integers + for key,value in sample['header'].items(): + try: + if value == INVALID: + raise ValueError + sample['header'][key] = int(value) + except (ValueError, TypeError): + sample['header'][key] = n.NaN + # convert body data to floats for altitude in sample['body']: for key,value in altitude.items(): @@ -57,16 +71,9 @@ raise ValueError altitude[key] = float(value) - except (ValueError, TypeError, KeyError): + except (ValueError, TypeError): altitude[key] = n.NaN - for key,value in sample['header'].items(): - try: - if value == INVALID: - raise ValueError - sample['header'][key] = int(value) - except (ValueError, TypeError, KeyError): - sample['header'][key] = n.NaN def _stamp(self): - """Add a datetime stamp to each sample.""" + """Add a datetime stamp item to each sample.""" for sample in self: try: @@ -85,18 +92,11 @@ [sample['stamp'] for sample in self[1:]]) intervals = [interval[1] - interval[0] for interval in intervals] - accumulator = {} - for interval in intervals: - if interval in accumulator: - accumulator[interval] += 1 - else: - accumulator[interval] = 1 - maxVotes = max(accumulator.values()) - for key,value in accumulator.items(): - if value == maxVotes: - self.sampleInterval = key - break - self.sampleInterval = getattr(self, - 'sampleInterval', - datetime.timedelta.resolution) + intervals = [interval for interval in intervals if interval] + intervals = [interval for interval in intervals + if interval != datetime.datetime.min] + try: + self.interval = min(intervals) + except ValueError: + self.interval = datetime.datetime.min def _minimumAltitude(self): @@ -142,5 +142,14 @@ def _main(): """Process as script from command line.""" - pass + import urllib2 + try: + rawDataHandle = urllib2.urlopen('http://nemo.isis.unc.edu/'\ + 'data/nccoos/level0/dukeforest/sodar/'\ + 'store/2007-06/20070601.dat') + rawDataString = rawDataHandle.read() + except: + raise IOError("Failure to read raw test data") + rawDataObject = rawData.RawData(rawDataString) + processedDataObject = ProcessedData(rawDataObject) if __name__ == "__main__": Index: sodar/trunk/sodar/rawData.py =================================================================== --- sodar/trunk/sodar/rawData.py (revision 72) +++ sodar/trunk/sodar/rawData.py (revision 73) @@ -15,7 +15,7 @@ Each Sample object has attributes for a Header and Body object. The Samples -within a RawData object may also be accessed by time using a string of the format -YYYY-MM-DD-HH-MM as in index on the RawData object to return the first matching -Sample in the RawData object: +within a RawData object may also be accessed by time using a string of the +format YYYY-MM-DD-HH-MM as in index on the RawData object to return the first +matching Sample in the RawData object: rawDataObject[0] # the first Sample object of the day @@ -94,7 +94,9 @@ year,month,day,hour,minute = index.split('-') except ValueError: - raise ValueError('RawData index by date must be "YYYY-MM-DD-HH-MM"') + raise ValueError('RawData index by date must be '\ + '"YYYY-MM-DD-HH-MM"') except AttributeError: - raise AttributeError('RawData index by date must be "YYYY-MM-DD-HH-MM"') + raise AttributeError('RawData index by date must be '\ + '"YYYY-MM-DD-HH-MM"') for sample in self: try: @@ -140,8 +142,8 @@ raise IndexError('Sample index out of range') - def _copy(self): - """Create a deep copy as a dictionary of header and body copies.""" - return {'header':self.header._copy(), - 'body':self.body._copy()} + def data(self): + """Create a deep copy as a dictionary of header and body data.""" + return {'header':self.header.data(), + 'body':self.body.data()} @@ -169,6 +171,6 @@ " ".join(headerLines[1::2]).split()))) - def _copy(self): - """Create a shallow copy as a dictionary.""" + def data(self): + """Create a shallow copy of the data as a dictionary.""" return self.copy() @@ -217,6 +219,6 @@ raise IndexError('Body index, out of range') - def _copy(self): - """Create a deep copy as a list of dictionaries.""" + def data(self): + """Create a deep copy of the data as a list of dictionaries.""" return [altitude.copy() for altitude in self] @@ -226,5 +228,7 @@ import urllib2 try: - rawDataHandle = urllib2.urlopen('http://nemo.isis.unc.edu/data/nccoos/level0/dukeforest/sodar/store/2007-06/20070601.dat') + rawDataHandle = urllib2.urlopen('http://nemo.isis.unc.edu/'\ + 'data/nccoos/level0/dukeforest/sodar/'\ + 'store/2007-06/20070601.dat') rawDataString = rawDataHandle.read() except: