root/sodar/trunk/sodar/utils/findMissing.py

"""
Find raw sodar data files in need of processing.

Files are presumed organized according to NCCOOS conventions.
"""

import os

def computeSourceWalk(source):
    """
    Compute source file tree.
    
    computeSourceWalk(source) -> sourceWalk
    
    source - path to raw data directory in NCCOOS format.
    sourceWalk - list of all raw data files with full paths.
    """
    
    sourceWalk = os.walk(source)
    sourceWalk = [(fullPath, subDirs, files)
                 for fullPath, subDirs, files
                 in sourceWalk]
    return sourceWalk[1:]

def computeDestinationWalk(destination,force):
    """
    Compute destination file tree.
    
    computeDestinationWalk(destination, force) -> destinationWalk
    
    destination - path to plot images directory in NCCOOS format.
    force - update all destination plots for which raw data sources exist.
    destinationWalk - sorted list of all pre-existing destination paths.
    """
    
    if force:
        destinationWalk = []
    else:
        destinationWalk = os.walk(destination)
        destinationWalk = [(fullPath, subDirs, files)
                           for fullPath, subDirs, files
                           in destinationWalk]
        destinationWalk = destinationWalk[1:]
        destinationWalk = [fullPath
                           for fullPath, subDirs, files
                           in destinationWalk
                           if not subDirs]
        return sorted(destinationWalk)

def findMissing(source, destination, force):
    """
    Find raw sodar data files in need of processing.
    
    findMissing(source, destination, force) -> differenceWalk
    
    source - path to raw data directory in NCCOOS format.
    destination - path to plot images directory in NCCOOS format.
    force - update all destination plots for which raw data sources exist.
    differenceWalk - sorted list of source,destination full path tuples.
    
    Compute source filetree.
    Compute destination file tree.
    Compute ideal destination file tree from source
    Compare ideal destination file tree to destination file tree
    Compute difference file tree as list of tuples (source, destination)
    """
    
    sourceWalk = computeSourceWalk(source)
    
    destinationWalk = computeDestinationWalk(destination, force)
    # always mark most recent destination as missing
    # to keep it updated as source is updated during the day
    if destinationWalk:
        destinationWalk = destinationWalk[:-1]
    
    idealWalk = [os.path.join(destination,
                              fullPath.replace(source + os.path.sep, ''),
                              fileName.replace('.dat', ''))
                          
                 for fullPath, subDirs, files in sourceWalk
                 for fileName in files]
    
    differenceWalk = [path
                      for path
                      in idealWalk
                      if path not in destinationWalk]
    
    differenceWalk = sorted(differenceWalk)
    
    differenceWalk = [(os.path.join(source,
                                    path.replace(destination + os.path.sep,
                                                 '') + '.dat'),
                       path)
                       for path
                       in differenceWalk]
    
    return differenceWalk

if __name__ == '__main__':
    import optparse
    from pprint import pprint
    
    parser = optparse.OptionParser()
    (values, args) = parser.parse_args()
    (source, destination) = tuple(args)
    if source[-1] == os.path.sep:
        source = source[:-1]
    if destination[-1] == os.path.sep:
        destination = destination[:-1]
    
    pprint(findMissing(source, destination))