"""pyglider.py A module of utilities to run on dockserver for glider operations
parse_glider_mail
generate_track_kml
parse_glider_goto_ma
generate_waypoint_kml
"""
REAL_RE_STR = '\\s*(-?\\d(\\.\\d+|)[Ee][+\\-]\\d\\d?|-?(\\d+\\.\\d*|\\d*\\.\\d+)|-?\\d+)\\s*'
import sys
import os
import re
import time
import datetime
def load_data(inFile):
lines=None
if os.path.exists(inFile):
f = open(inFile, 'r')
lines = f.readlines()
f.close()
if len(lines)<=0:
print 'Empty file: '+ inFile
else:
print 'File does not exist: '+ inFile
return lines
def display_time_diff(diff):
"""Display time difference in HH:MM and days (D) if necessary"""
days = diff.days
minutes, seconds = divmod(diff.seconds, 60)
hours, minutes = divmod(minutes, 60)
if (days<=1):
if minutes<1:
str = "%02:%02d:%02d" % (hours, minutes,seconds)
else:
str = "%02d:%02d" % (hours, minutes)
elif (days>1):
str = "%d Days %02d:%02d" % (days, hours, minutes)
else:
str = "%02d:%02d" % (hours, minutes)
return str
def dt2es(dt):
"""Convert datetime object to epoch seconds (es) as seconds since Jan-01-1970 """
# microseconds of timedelta object not used
delta = dt - datetime.datetime(1970,1,1,0,0,0)
es = delta.days*24*60*60 + delta.seconds
return es
def es2dt(es):
""" Convert epoch seconds (es) to datetime object"""
dt = datetime.datetime(*time.gmtime(es)[0:6])
return dt
# -------------------------------------------------------------------
# playground
# fn = '/var/spool/mail/localuser'
# lines = load_data(fn)
# glider = 'ramses'
# data = parse_glider_mail(lines, glider)
#
# fn = '/home/localuser/realtime/gccs/goto-list/goto_l10_1329750438.ma'
# lines = load_data(fn)
# glider = 'ramses'
# data = parse_glider_goto_ma(lines, glider)
# -------------------------------------------------------------------
def parse_glider_mail(lines, glider):
msg_split_patt = r'From root\@dockserver'
msgs = re.split(msg_split_patt, ''.join(lines))
gms = []
# select messages for specific glider based on "Subject:" line
for msg in msgs:
m = re.search(r'^Subject: Glider: (\w*)', msg, re.MULTILINE)
if m:
if m.group(1) == glider:
gms.append(msg)
data = []
for msg in gms:
m = re.search(r'^Subject: Glider: (\w*).*$', msg, re.MULTILINE)
if m:
subject_string = m.group(0)
subject_glider = m.group(1)
# print subject_string
else:
continue
m = re.search(r'\s*(Event)*: (.*?)', subject_string, re.MULTILINE)
if m:
subject_event = m.group(2)
else:
subject_event = None
m = re.search(r'\s*(Reason)*:\s*(.*)$', subject_string, re.MULTILINE)
if m:
subject_reason = m.group(2)
else:
subject_reason = None
m = re.search(r'^(Vehicle Name:)\s+(\w*)', msg, re.MULTILINE)
if m: glidername = m.group(2)
else: glidername = None
m = re.search(r'^(Curr Time:)\s+(.*)\s+MT:', msg, re.MULTILINE)
if m:
try:
t = time.strptime(m.group(2), "%a %b %d %H:%M:%S %Y")
# the '*' operator unpacks the tuple, producing the argument list.
dt = datetime.datetime(*t[0:6])
diff = datetime.datetime.utcnow() - dt
hours_ago = display_time_diff(diff)
dt_str = datetime.date.strftime(dt, "%Y-%m-%d %H:%M:%S UTC")
if (diff.days)>0:
# dt_str_short = datetime.date.strftime(dt, "%b-%d")
dt_str_short = ""
if (diff.days) <= 0:
dt_str_short = datetime.date.strftime(dt, "%H:%M")
except ValueError, e:
dt_str = None
dt_str_short = None
hours_ago = None
m = re.search(r'^(GPS Location:)\s+(-?\d{2})(\d{2}\.\d+)\s+([NnSs])'+ \
r'\s+(-?\d{2})(\d{2}\.\d+)\s+([EeWw]).*$', msg, re.MULTILINE)
if m:
#
lat_deg = float(m.group(2))
lat_min = float(m.group(3))
lat_hem = m.group(4).upper()
#
lon_deg = float(m.group(5))
lon_min = float(m.group(6))
lon_hem = m.group(7).upper()
if lat_deg<0:
lat = lat_deg - lat_min/60.
else:
lat = lat_deg + lat_min/60.
if lon_deg<0:
lon = lon_deg - lon_min/60.
else:
lon = lon_deg + lon_min/60.
gps_str = m.group(0)
m = re.search(r'GPS Location: (.*) measured', gps_str)
if m: gps_posn_str = m.group(1)
else: gps_posn_str = None
m = re.search(r'measured\s*(\d*)\.\d* secs ago', gps_str)
if m:
gps_secs_ago = int(m.group(1))
gps_dt = dt - datetime.timedelta(0,gps_secs_ago,0)
gps_dt_str = datetime.date.strftime(gps_dt, "%Y-%m-%d %H:%M:%S UTC")
else: gps_dt_str = None
else:
lat = None
lon = None
m = re.search(r'(MT:)\s+(.*)$', msg, re.MULTILINE)
if m: mt = m.group(2)
else: mt = None
m = re.search(r'(sensor:m_battery.*?=)\s*(-?\d+\.\d*)', msg, re.MULTILINE)
if m: batt = float(m.group(2))
else: batt = None
m = re.search(r'(sensor:m_leakdetect.*?=)\s*(-?\d+\.\d*)', msg, re.MULTILINE)
if m: leak = float(m.group(2))
else: leak = None
m = re.search(r'(sensor:m_vacuum.*?=)\s*(-?\d+\.\d*)', msg, re.MULTILINE)
if m: vacuum = float(m.group(2))
else: vacuum = None
m = re.search(r'^(Because:)\s*(.*)', msg, re.MULTILINE)
if m: because = m.group(2)
else: because = 'Unknown'
m = re.search(r'^(MissionName:)\s*(.*?)\s+', msg, re.MULTILINE)
if m: mission_name = m.group(2)
else: mission_name = 'Unknown'
m = re.search(r'\s+(MissionNum:)(.*)', msg, re.MULTILINE)
if m: mission_num = m.group(2)
else: mission_num = 'Unknown'
m = re.search(r'^(Waypoint:)\s+(\(.*\)).*$', msg, re.MULTILINE)
if m:
wp_str = m.group(0)
waypoint_posn = m.group(2)
else:
wp_str = None
waypoint_posn = 'Unknown'
waypoint_range = 'Unknown'
waypoint_bearing = 'Unknown'
waypoint_age = 'Unknown'
wlat=None
wlon=None
if wp_str:
# parse out the next waypoint for a place mark
m = re.search(r'\((-?\d{2})(\d{2}\.\d+),(-?\d{2})(\d{2}\.\d+)\)', waypoint_posn)
if m:
#
lat_deg = float(m.group(1))
lat_min = float(m.group(2))
#
lon_deg = float(m.group(3))
lon_min = float(m.group(4))
if lat_deg<0:
wlat = lat_deg - lat_min/60.
else:
wlat = lat_deg + lat_min/60.
if lon_deg<0:
wlon = lon_deg - lon_min/60.
else:
wlon = lon_deg + lon_min/60.
else:
wlat = None
wlon = None
m = re.search(r'(Range:)\s+(.*?),', wp_str)
if m: waypoint_range = m.group(2)
else: waypoint_range = 'Unknown'
m = re.search(r'(Bearing:)\s+(.*?),', wp_str)
if m: waypoint_bearing = m.group(2)
else: waypoint_bearing = 'Unknown'
m = re.search(r'(Age:)\s+(.*?)$', wp_str)
if m: waypoint_age = m.group(2)
else: waypoint_age = 'Unknown'
if lat and lon:
# generate report table for google earth
# using ![CDATA[ {html} ]] inside of KML description tag
html_report = [
' ',
'
',
'
Surface Report
',
'
',
'',
'Glider: %s | %s |
' % (glider, dt_str,),
'',
'']
html_report.extend(
[
'GPS Location: | %s |
' % (gps_posn_str,),
'GPS Time: | %s |
' % (gps_dt_str,)
])
html_report.extend(
[
'Surface Event: | %s |
' % (subject_event,),
'Surface Reason: | %s |
' % (subject_reason,),
'Surface Because: | %s |
' % (because,),
])
html_report.extend(
[
'Mission Name: | %s |
' % (mission_name,),
'Mission Number: | %s |
' % (mission_num,),
'Mission Time: | %s |
' % (mt,)
])
if batt:
if batt < 10.:
html_report.append('Battery (volts): | %g |
' % (batt,))
elif (batt>=10) and (batt<11):
html_report.append('Battery (volts): | %g |
' % (batt,))
else:
html_report.append('Battery (volts): | %f |
' % (batt,))
else:
html_report.append('Battery (volts): | Unknown |
')
if vacuum:
if (vacuum>=10) or (vacuum<6):
html_report.append('Vacuum (inHg): | %g |
' % (vacuum,))
else:
html_report.append('Vacuum (inHg): | %g |
' % (vacuum,))
else:
html_report.append('Vacuum (inHg): | Unknown |
')
# need tolerances for flagging leak detect in report
if leak:
html_report.append('Leak Detect (volts): | %f |
' % (leak,))
else:
html_report.append('Leak Detect (Volts): | Unknown |
')
#
html_report.extend(
[
'Waypoint: | %s |
' % (waypoint_posn,),
'Waypoint Range: | %s |
' % (waypoint_range,),
'Waypoint Bearing: | %s |
' % (waypoint_bearing,),
'Waypoint Age: | %s |
' % (waypoint_age,)
])
# close the report tbody table div and CDATA
html_report.extend(
['',
'
',
'
',
' '
])
# make a readable CDATA string
html_report_str = '\n'.join(html_report)
# can create a KML object since extracted lat and lon
data.append({'glider': glider,
'dt': dt,
'datetime': dt_str,
'name': dt_str_short,
'hours_ago': hours_ago,
'lat': lat,
'lon': lon,
'description': html_report_str,
'wlat': wlat,
'wlon': wlon,
})
else:
# if no lat lon, can't create a KML object without lat and lon
# but want to append info to last known surface report
html_report = [
'',
'',
'',
'Glider: %s | ABORT Surfacing |
' % (subject_glider,),
'',
'']
html_report.extend(
[
'Surface Event: | %s |
' % (subject_event,),
'Surface Reason: | %s |
' % (subject_reason,),
])
# close the report tbody table div and CDATA
html_report.extend(
['',
'
',
' '
])
html_report_str = data[-1]['description']
html_report_str = html_report_str + '\n'.join(html_report)
# print html_report_str
data[-1]['description'] = html_report_str
# close for-loop of msg in gms:
return data
def generate_track_kml(data, glider):
""" Use pykml to generate kml file for each glider from parsed mail data
A glider track consists of a line and placemarks for each surfacing
Usage: kml_doc_str = generate_track_kml(data, glider)
"""
from pykml.factory import KML_ElementMaker as KML
import lxml.etree
# ***** append LookAt after checking that lat, lon exist in data[-1]
d = data[-1]
# print '(%f, %f)' % (d['lon'], d['lat'])
# start a KML file skeleton with track styles
doc = KML.kml(
KML.Document(
KML.Name(glider + "_track"),
KML.LookAt(
KML.longitude('%f' % d['lon']),
KML.latitude('%f' % d['lat']),
KML.heading('0'),
KML.tilt('0'),
KML.range('60000'),
KML.altitudeMode('absolute')
),
KML.Style(
KML.IconStyle(
KML.scale(0.7),
KML.color("ff00ff00"),
KML.Icon(KML.href("icons/square.png"))
),
KML.LabelStyle(
KML.color("ff00ff00"),
KML.scale(0.8)),
id="lastPosnIcon")
)
)
doc.Document.append(
KML.Style(
KML.IconStyle(
KML.scale(0.8),
KML.color("ff0000ff"),
KML.Icon(KML.href("icons/cross-hairs.png"))),
KML.LabelStyle(
KML.scale(0.8)),
id="lastWayPosnIcon")
)
doc.Document.append(
KML.Style(
KML.IconStyle(
KML.scale(0.7),
KML.color("7d00ffff"),
KML.Icon(KML.href("icons/donut.png"))),
KML.LabelStyle(
KML.color("7d00ffff"),
KML.scale(0.8)),
id="prevPosnIcon")
)
doc.Document.append(
KML.Style(
KML.IconStyle(
KML.scale(0.9),
KML.color("ff00ff00"),
KML.Icon(KML.href("icons/donut.png"))),
KML.LabelStyle(
KML.color("ff00ff00"),
KML.scale(0.7)),
id="histPosnIcon")
)
doc.Document.append(
KML.Style(
KML.LineStyle(
KML.color("7dff0000"),
KML.width(2)
),
id="transBlueLine")
)
doc.Document.append(
KML.Style(
KML.LineStyle(
KML.color("7d00ff00"),
KML.width(2)
),
id="transGreenLine")
)
if glider == 'ramses':
linestyle = "#transBlueLine"
elif glider == 'pelagia':
linestyle = "#transGreenLine"
else:
linestyle = ""
coord_str = ""
for d in data:
if d['lat'] and d['lon']:
coord_str = coord_str + "%f,%f,%f\n" % (d['lon'], d['lat'], 0.)
track_line = KML.Placemark(
KML.name(glider),
KML.description(glider),
KML.styleUrl(linestyle),
KML.LineString(
KML.altitudeMode("absolute"),
KML.coordinates(coord_str)
)
)
# glider placemarks (pms)
pms = []
for d in data[:-1]:
pms.append(
KML.Placemark(
# short time stamp
KML.name(d['name']),
# surface report data
KML.description(d['description']),
KML.styleUrl("#prevPosnIcon"),
KML.Point(
KML.altitudeMode("absolute"),
KML.coordinates("%f,%f,%f" % (d['lon'], d['lat'], 0.))
)
)
)
# glider history placemarks (hpms) using timestamp tag
histpms = []
for idx, d in enumerate(data[:-1]):
dt=d['dt']
# YYYY-MM-DDTHH:MM:SS + Z for kml
dt_str_begin = dt.isoformat()+'Z'
dt_str_end = (data[idx+1]['dt']).isoformat()+'Z'
histpms.append(
KML.Placemark(
# ISO time stamp to be displayed with marker
KML.name(dt_str_begin),
# surface report data (not for history placemarks
# KML.description(d['description']),
KML.styleUrl("#histPosnIcon"),
KML.Point(
KML.altitudeMode("absolute"),
KML.coordinates("%f,%f,%f" % (d['lon'], d['lat'], 0.))
),
KML.TimeSpan(
KML.begin(dt_str_begin),
KML.end(dt_str_end)
)
)
)
d=data[-1]
last_pm = KML.Placemark(
# short time stamp
KML.name(d['name']),
# surface report data
KML.description(d['description']),
KML.styleUrl("#lastPosnIcon"),
KML.Point(
KML.altitudeMode("absolute"),
KML.coordinates("%f,%f,%f" % (d['lon'], d['lat'], 0.))
)
)
if d['wlon'] and d['wlat']:
wp_pm = KML.Placemark(
# KML.name('Surface Report Waypoint'),
# surface report data
KML.description('Surface Report Waypoint'),
KML.styleUrl("#lastWayPosnIcon"),
KML.Point(
KML.altitudeMode("absolute"),
KML.coordinates("%f,%f,%f" % (d['wlon'], d['wlat'], 0.))
)
)
track_folder = KML.Folder(
KML.name(d['glider']),
track_line,
)
for pm in pms:
track_folder.append(pm)
for hpm in histpms:
track_folder.append(hpm)
track_folder.append(last_pm)
if d['wlon'] and d['wlat']:
track_folder.append(wp_pm)
doc.Document.append(
track_folder
)
track_kml = lxml.etree.tostring(doc, pretty_print=True)
return track_kml
def parse_glider_goto_ma(lines, glider):
data = []
m = re.search(r'^(# File creation time:)\s*(.*)', ''.join(lines), re.MULTILINE)
if m:
try:
t = time.strptime(m.group(2), "%d-%b-%Y %H:%M:%S")
# the '*' operator unpacks the tuple, producing the argument list.
# add 5 hours for GMT
dt = datetime.datetime(*t[0:6]) + datetime.timedelta(hours=5)
dt_str = datetime.date.strftime(dt, "%Y-%m-%d %H:%M:%S UTC")
except: dt_str = None
m = re.search(r'\(.*)\', ''.join(lines), re.MULTILINE|re.S)
latlonstr = m.group(1)
latlons = re.split(r'\n', latlonstr)
for ll in latlons:
m = re.search(r'^\s*(-?\d{2})(\d{2}\.\d+)\s*(-?\d{2})(\d{2}\.\d+).*$', ll, re.MULTILINE)
if m:
#
lat_deg = float(m.group(3))
lat_min = float(m.group(4))
# lat_hem = m.group(4).upper()
#
lon_deg = float(m.group(1))
lon_min = float(m.group(2))
# lon_hem = m.group(7).upper()
if lat_deg<0:
lat = lat_deg - lat_min/60.
else:
lat = lat_deg + lat_min/60.
if lon_deg<0:
lon = lon_deg - lon_min/60.
else:
lon = lon_deg + lon_min/60.
else:
lat = None
lon = None
html_str = ''+glider+' goto position at '+ dt_str + '' + \
ll+'
'
if lat and lon:
data.append({'glider': glider,
'name': '',
'description': html_str,
'lon': lon,
'lat': lat})
# close for-loop
return data
def generate_waypoint_kml(data, glider):
"""
Use pykml
Usage: kml_doc_str = generate_waypoint_kml(data, glider)
"""
from pykml.factory import KML_ElementMaker as KML
import lxml.etree
doc = KML.kml(
KML.Document(
KML.Name(glider + "_waypoint"),
KML.Style(
KML.IconStyle(
KML.scale(0.7),
KML.color("ff00ff00"),
KML.Icon(KML.href("icons/square.png"))
),
KML.LabelStyle(
KML.color("ff00ff00"),
KML.scale(0.8)),
id="lastPosnIcon")
)
)
doc.Document.append(
KML.Style(
KML.IconStyle(
KML.scale(0.7),
KML.color("7dff00ff"),
KML.Icon(KML.href("icons/square.png"))
),
KML.LabelStyle(
KML.scale(0.7)
),
id="gotoPosnIcon")
)
doc.Document.append(
KML.Style(
KML.LineStyle(
KML.color("7dff0000"),
KML.width(2)
),
id="transBlueLine")
)
doc.Document.append(
KML.Style(
KML.LineStyle(
KML.color("7d00ff00"),
KML.width(2)
),
id="transGreenLine")
)
if glider == 'ramses':
linestyle = "#transBlueLine"
elif glider == 'pelagia':
linestyle = "#transGreenLine"
else:
linestyle = ""
coord_str = ""
for d in data:
if d['lat'] and d['lon']:
coord_str = coord_str + "%f,%f,%f\n" % (d['lon'], d['lat'], 0.)
waypoint_line = KML.Placemark(
KML.name(glider),
KML.description(glider+' latest goto waypoints'),
KML.styleUrl(linestyle),
KML.LineString(
KML.altitudeMode("absolute"),
KML.coordinates(coord_str)
)
)
# glider placemarks (pms)
pms = []
for d in data[:-1]:
pms.append(
KML.Placemark(
# short time stamp
KML.description(d['description']),
KML.styleUrl("#gotoPosnIcon"),
KML.Point(
KML.altitudeMode("absolute"),
KML.coordinates("%f,%f,%f" % (d['lon'], d['lat'], 0.))
)
)
)
d = data[-1]
wp_folder = KML.Folder(
KML.name(d['glider']),
waypoint_line,
)
for pm in pms:
wp_folder.append(pm)
doc.Document.append(
wp_folder
)
track_kml = lxml.etree.tostring(doc, pretty_print=True)
return track_kml