root/proc2plot/trunk/proc2plot/scratch/test_crow_wq_plot.py

#!/usr/bin/env /opt/env/haines/dataproc/bin/python
# Last modified:  Time-stamp: <2009-09-16 17:27:43 haines>
"""crow_wq_plot"""

import os, sys, glob, re
import datetime, time, dateutil, dateutil.tz
import pycdf
import numpy

sys.path.append('/opt/env/haines/dataproc/raw2proc')
del(sys)

os.environ["MPLCONFIGDIR"]="/home/haines/.matplotlib/"

from pylab import figure, twinx, savefig, setp, getp, cm, colorbar
from matplotlib.dates import DayLocator, HourLocator, MinuteLocator, DateFormatter, date2num, num2date
import procutil

print 'crow_wq_plot ...'

data_dir = '/seacoos/data/nccoos/level1'
platform = 'crow'
package = 'wq'
# yyyy_mm = None
# can be a past month
yyyy_mm = '2009_08'

if not yyyy_mm:
    yyyy_mm = procutil.this_month()
    plot_latest = True
else:
    plot_latest = False
    
prev_month, this_month, next_month = procutil.find_months(yyyy_mm)

package1 = package
#ncFile1='/seacoos/data/nccoos/level1/meet/wq/meet_wq_2009_08.nc'
#ncFile2='/seacoos/data/nccoos/level1/meet/wq/meet_wq_2008_09.nc'
ncFile1 = os.path.join(data_dir, platform, package1, platform+'_'+package1+'_'+prev_month.strftime('%Y_%m')+'.nc') 
ncFile2 = os.path.join(data_dir, platform, package1, platform+'_'+package1+'_'+this_month.strftime('%Y_%m')+'.nc') 

# load wq data

have_ncFile1 = os.path.exists(ncFile1)
have_ncFile2 = os.path.exists(ncFile2)

print ' ... loading data for graph from ...'
print ' ... ... ' + ncFile1 + ' ... ' + str(have_ncFile1)
print ' ... ... ' + ncFile2 + ' ... ' + str(have_ncFile2)

if have_ncFile1 and have_ncFile2:
    nc = pycdf.CDFMF((ncFile1, ncFile2))
elif not have_ncFile1 and have_ncFile2:
    nc = pycdf.CDFMF((ncFile2,))
elif have_ncFile1 and not have_ncFile2:
    nc = pycdf.CDFMF((ncFile1,))
else:
    print ' ... both files do not exist -- NO DATA LOADED'
    print ' ... NO PLOTS CREATED'
    exit()

# get variable names 
ncvars = nc.variables()
# print ncvars
es = nc.var('time')[:]
units = nc.var('time').units
dt = [procutil.es2dt(e) for e in es]
# set timezone info to UTC (since data from level1 should be in UTC!!)
dt = [e.replace(tzinfo=dateutil.tz.tzutc()) for e in dt]
# return new datetime based on computer local
dt_local = [e.astimezone(dateutil.tz.tzlocal()) for e in dt]
dn = date2num(dt)

z = nc.var('z')[:]
wtemp = nc.var('wtemp')[:]
cond = nc.var('cond')[:]
turb = nc.var('turb')[:]
ph = nc.var('ph')[:]
do_mg = nc.var('do_mg')[:]
do_sat = nc.var('do_sat')[:]
batt = nc.var('battvolts')[:]
nc.close()

package2 = 'flow'
ncFile1 = os.path.join(data_dir, platform, package2, platform+'_'+package2+'_'+prev_month.strftime('%Y_%m')+'.nc') 
ncFile2 = os.path.join(data_dir, platform, package2, platform+'_'+package2+'_'+this_month.strftime('%Y_%m')+'.nc') 

# load flow data

have_ncFile1 = os.path.exists(ncFile1)
have_ncFile2 = os.path.exists(ncFile2)

print ' ... loading data for graph from ...'
print ' ... ... ' + ncFile1 + ' ... ' + str(have_ncFile1)
print ' ... ... ' + ncFile2 + ' ... ' + str(have_ncFile2)

if have_ncFile1 and have_ncFile2:
    nc = pycdf.CDFMF((ncFile1, ncFile2))
elif not have_ncFile1 and have_ncFile2:
    nc = pycdf.CDFMF((ncFile2,))
elif have_ncFile1 and not have_ncFile2:
    nc = pycdf.CDFMF((ncFile1,))
else:
    print ' ... both files do not exist -- NO DATA LOADED'

# get variable names
ncvars = nc.variables()
# print ncvars
es2 = nc.var('time')[:]
units = nc.var('time').units
dt2 = [procutil.es2dt(e) for e in es2]
# set timezone info to UTC (since data from level1 should be in UTC!!)
dt2 = [e.replace(tzinfo=dateutil.tz.tzutc()) for e in dt2]
dn2 = date2num(dt2)

r2 = nc.var('rain')[:] # inches of rain in past 15 min
have_sontek = 'sontek_wl' in ncvars.keys() or 'sontek_flow' in ncvars.keys()
have_press = 'press_wl' in ncvars.keys() or 'press_flow' in ncvars.keys()

if have_press:
    pwl2 = nc.var('press_wl')[:] # feet
    pfl2 = nc.var('press_flow')[:] # cfs

if have_sontek:
    swl2 = nc.var('sontek_wl')[:] # feet
    sfl2 = nc.var('sontek_flow')[:] # cfs

nc.close()

# last dt in data for labels
dtu = dt[-1]
dtl = dt_local[-1]

diff = abs(dtu - dtl)
if diff.days>0:
    last_dt_str = dtu.strftime("%H:%M %Z on %b %d, %Y") + ' (' + dtl.strftime("%H:%M %Z, %b %d") + ')'
else:
    last_dt_str = dtu.strftime("%H:%M %Z") + ' (' + dtl.strftime("%H:%M %Z") + ')' \
                  + dtl.strftime(" on %b %d, %Y")
        
#######################################
# Plot month all wq (plus flow) the timeseries data
#######################################

fig = figure(figsize=(10, 9))
fig.subplots_adjust(left=0.1, bottom=0.1, right=0.9, top=0.9, wspace=0.1, hspace=0.1)

ax = fig.add_subplot(6,1,1)
axs = [ax]

# ax.plot returns a list of lines, so unpack tuple
(x, y) = procutil.addnan(dt, wtemp)
ibad = y <= -6999.
y[ibad] = numpy.nan
# if all are NaN's then pyplot throws an error for autoscale
# for some reason setting one value in the timeseries to a real value
# fixes this.
if numpy.isnan(y).all():
    y[-1]=0.

l1, = ax.plot_date(x, y, fmt='b-')
l1.set_label('Water Temperature')

ax.set_ylabel('TEMP\n (deg C)')
ax.set_xlim(date2num(this_month), date2num(next_month-datetime.timedelta(seconds=1)))
ax.xaxis.set_major_locator( DayLocator(range(2,32,2)) )
ax.xaxis.set_minor_locator( HourLocator(range(0,25,12)) )
ax.xaxis.set_major_formatter( DateFormatter('%m/%d') )
ax.set_xticklabels([])

# right-hand side scale
ax2 = twinx(ax)
ax2.yaxis.tick_right()
# convert (lhs) deg C to (rhs) deg F
f = [procutil.celsius2fahrenheit(val) for val in ax.get_ylim()]
ax2.set_ylim(f)
ax2.set_ylabel('(deg F)')

ax2.set_xlim(date2num(this_month), date2num(next_month-datetime.timedelta(seconds=1)))
ax2.xaxis.set_major_locator( DayLocator(range(2,32,2)) )
ax2.xaxis.set_minor_locator( HourLocator(range(0,25,12)) )
ax2.set_xticklabels([])

# this only moves the label not the tick labels
ax.xaxis.set_label_position('top')
ax.set_xlabel('CROW Water Quality -- ' + yyyy_mm)

# legend
ls1 = l1.get_label()
leg = ax.legend((l1,), (ls1,), loc='upper left')
ltext  = leg.get_texts()  # all the text.Text instance in the legend
llines = leg.get_lines()  # all the lines.Line2D instance in the legend
frame  = leg.get_frame()  # the patch.Rectangle instance surrounding the legend
frame.set_facecolor('0.80')      # set the frame face color to light gray
frame.set_alpha(0.5)             # set alpha low to see through
setp(ltext, fontsize='small')    # the legend text fontsize
setp(llines, linewidth=1.5)      # the legend linewidth
# leg.draw_frame(False)           # don't draw the legend frame

#######################################
#
ax = fig.add_subplot(6,1,2)
axs.append(ax)

# ax.plot returns a list of lines, so unpack tuple
(x, y) = procutil.addnan(dt, cond)
l1, = ax.plot_date(x, y, fmt='b-')
l1.set_label('Specific Conductivity')

ax.set_ylabel('Cond\n (mS/cm)')
# ax.set_ylim(0,10)
ax.set_xlim(date2num(this_month), date2num(next_month-datetime.timedelta(seconds=1)))
ax.xaxis.set_major_locator( DayLocator(range(2,32,2)) )
ax.xaxis.set_minor_locator( HourLocator(range(0,25,12)) )
ax.xaxis.set_major_formatter( DateFormatter('%m/%d') )
ax.set_xticklabels([])

# legend
ls1 = l1.get_label()
leg = ax.legend((l1,), (ls1,), loc='upper left')
ltext  = leg.get_texts()  # all the text.Text instance in the legend
llines = leg.get_lines()  # all the lines.Line2D instance in the legend
frame  = leg.get_frame()  # the patch.Rectangle instance surrounding the legend
frame.set_facecolor('0.80')      # set the frame face color to light gray
frame.set_alpha(0.5)             # set alpha low to see through
setp(ltext, fontsize='small')    # the legend text fontsize
setp(llines, linewidth=1.5)      # the legend linewidth
# leg.draw_frame(False)           # don't draw the legend frame

#######################################
#
ax = fig.add_subplot(6,1,3)
axs.append(ax)

# ax.plot returns a list of lines, so unpack tuple
(x, y) = procutil.addnan(dt, do_mg)
l1, = ax.plot_date(x, y, fmt='b-')
l1.set_label('Dissolved Oxygen (DO)')

ax.set_ylabel('DO \n(mg/l)')
# ax.set_ylim(20,120)
ax.set_xlim(date2num(this_month), date2num(next_month-datetime.timedelta(seconds=1)))
ax.xaxis.set_major_locator( DayLocator(range(2,32,2)) )
ax.xaxis.set_minor_locator( HourLocator(range(0,25,12)) )
ax.xaxis.set_major_formatter( DateFormatter('%m/%d') )
ax.set_xticklabels([])

# right-hand side scale
ax2 = twinx(ax)
ax2.yaxis.tick_right()
(x, y) = procutil.addnan(dt, do_sat)
l2, = ax2.plot_date(x, y, fmt='c-')
l2.set_label('Percent Air Saturation')    
# convert (lhs) deg C to (rhs) deg F
# ax2.set_ylim(0,200)
ax2.set_ylabel('DO (%)')
ax2.set_xlim(date2num(this_month), date2num(next_month-datetime.timedelta(seconds=1)))
ax2.xaxis.set_major_locator( DayLocator(range(2,32,2)) )
ax2.xaxis.set_minor_locator( HourLocator(range(0,25,12)) )
ax2.set_xticklabels([])

# legend
ls1 = l1.get_label()
ls2 = l2.get_label()
leg = ax.legend((l1,l2), (ls1,ls2), loc='upper left')
ltext  = leg.get_texts()  # all the text.Text instance in the legend
llines = leg.get_lines()  # all the lines.Line2D instance in the legend
frame  = leg.get_frame()  # the patch.Rectangle instance surrounding the legend
frame.set_facecolor('0.80')      # set the frame face color to light gray
frame.set_alpha(0.5)             # set alpha low to see through
setp(ltext, fontsize='small')    # the legend text fontsize
setp(llines, linewidth=1.5)      # the legend linewidth
# leg.draw_frame(False)           # don't draw the legend frame
#######################################
#
ax = fig.add_subplot(6,1,4)
axs.append(ax)

# ax.plot returns a list of lines, so unpack tuple
(x, y) = procutil.addnan(dt, ph)
l1, = ax.plot_date(x, y, fmt='b-')
l1.set_label('pH')

ax.set_ylabel('pH')
# ax.set_ylim(6, 12)
ax.set_xlim(date2num(this_month), date2num(next_month-datetime.timedelta(seconds=1)))
ax.xaxis.set_major_locator( DayLocator(range(2,32,2)) )
ax.xaxis.set_minor_locator( HourLocator(range(0,25,12)) )
ax.xaxis.set_major_formatter( DateFormatter('%m/%d') )
ax.set_xticklabels([])

# legend
ls1 = l1.get_label()
leg = ax.legend((l1,), (ls1,), loc='upper left')
ltext  = leg.get_texts()  # all the text.Text instance in the legend
llines = leg.get_lines()  # all the lines.Line2D instance in the legend
frame  = leg.get_frame()  # the patch.Rectangle instance surrounding the legend
frame.set_facecolor('0.80')      # set the frame face color to light gray
frame.set_alpha(0.5)             # set alpha low to see through
setp(ltext, fontsize='small')    # the legend text fontsize
setp(llines, linewidth=1.5)      # the legend linewidth
# leg.draw_frame(False)           # don't draw the legend frame

#######################################
#
ax = fig.add_subplot(6,1,5)
axs.append(ax)

# ax.plot returns a list of lines, so unpack tuple
(x, y) = procutil.addnan(dt, turb)
l1, = ax.plot_date(x, y, fmt='b-')
l1.set_label('Turbidity')

ax.set_ylabel('Turb \n(NTU)')
# ax.set_ylim(0,100)
ax.set_xlim(date2num(this_month), date2num(next_month-datetime.timedelta(seconds=1)))
ax.xaxis.set_major_locator( DayLocator(range(2,32,2)) )
ax.xaxis.set_minor_locator( HourLocator(range(0,25,12)) )
ax.xaxis.set_major_formatter( DateFormatter('%m/%d') )
ax.set_xticklabels([])

# legend
ls1 = l1.get_label()
leg = ax.legend((l1,), (ls1,), loc='upper left')
ltext  = leg.get_texts()  # all the text.Text instance in the legend
llines = leg.get_lines()  # all the lines.Line2D instance in the legend
frame  = leg.get_frame()  # the patch.Rectangle instance surrounding the legend
frame.set_facecolor('0.80')      # set the frame face color to light gray
frame.set_alpha(0.5)             # set alpha low to see through
setp(ltext, fontsize='small')    # the legend text fontsize
setp(llines, linewidth=1.5)      # the legend linewidth
# leg.draw_frame(False)           # don't draw the legend frame

#######################################
#
ax = fig.add_subplot(6,1,6)
axs.append(ax)

# ax.plot returns a list of lines, so unpack tuple
if have_press:
    (x, y) = procutil.addnan(dt2, pfl2)
    l1, = ax.plot_date(x, y, fmt='b-')
    l1.set_label('Derived from pressure')
    ls1 = l1.get_label()
if have_sontek:
    (x, y) = procutil.addnan(dt2, sfl2)
    l2, = ax.plot_date(x, y, fmt='c-')
    l2.set_label('Depth-avg Acoustic Profile')    
    ls2 = l2.get_label()

ax.set_ylabel('Flow\n (cfs)')
# ax.set_ylim(0,15)
ax.set_xlim(date2num(this_month), date2num(next_month-datetime.timedelta(seconds=1)))
ax.xaxis.set_major_locator( DayLocator(range(2,32,2)) )
ax.xaxis.set_minor_locator( HourLocator(range(0,25,12)) )
ax.xaxis.set_major_formatter( DateFormatter('%m/%d') )

ax.set_xlabel('CROW Water Quality -- ' + yyyy_mm)

# legend
if have_press and have_sontek:
    leg = ax.legend((l1,l2), (ls1,ls2), loc='upper left')
elif have_press:
    leg = ax.legend((l1,), (ls1,), loc='upper left')        
if have_sontek:
    leg = ax.legend((l2,), (ls2,), loc='upper left')        

ltext  = leg.get_texts()  # all the text.Text instance in the legend
llines = leg.get_lines()  # all the lines.Line2D instance in the legend
frame  = leg.get_frame()  # the patch.Rectangle instance surrounding the legend
frame.set_facecolor('0.80')      # set the frame face color to light gray
frame.set_alpha(0.5)             # set alpha low to see through
setp(ltext, fontsize='small')    # the legend text fontsize
setp(llines, linewidth=1.5)      # the legend linewidth
# leg.draw_frame(False)           # don't draw the legend frame

# save figure    
ofn = '/home/haines/rayleigh/img/'+platform+'/'+platform+'_'+package+'_'+yyyy_mm+'.png'
print '... ... write: %s' % (ofn,)
savefig(ofn)

if plot_latest:
    #######################################
    # Last 30 days
    #######################################
    
    print ' ... Last 30 days'
    ax = axs[0]
    ax.set_xlim(date2num(dt[-1])-30, date2num(dt[-1]))
    ax.xaxis.set_major_locator( DayLocator(range(0,32,2)) )
    ax.xaxis.set_minor_locator( HourLocator(range(0,25,12)) )
    ax.set_xticklabels([])
    ax.set_xlabel('CROW Water Quality -- Last 30 days from ' + last_dt_str)

    ax = axs[1]
    ax.set_xlim(date2num(dt[-1])-30, date2num(dt[-1]))
    ax.xaxis.set_major_locator( DayLocator(range(0,32,2)) )
    ax.xaxis.set_minor_locator( HourLocator(range(0,25,12)) )
    ax.set_xticklabels([])

    ax = axs[2]
    ax.set_xlim(date2num(dt[-1])-30, date2num(dt[-1]))
    ax.xaxis.set_major_locator( DayLocator(range(0,32,2)) )
    ax.xaxis.set_minor_locator( HourLocator(range(0,25,12)) )
    ax.set_xticklabels([])

    ax = axs[3]
    ax.set_xlim(date2num(dt[-1])-30, date2num(dt[-1]))
    ax.xaxis.set_major_locator( DayLocator(range(0,32,2)) )
    ax.xaxis.set_minor_locator( HourLocator(range(0,25,12)) )
    ax.set_xticklabels([])

    ax = axs[4]
    ax.set_xlim(date2num(dt[-1])-30, date2num(dt[-1]))
    ax.xaxis.set_major_locator( DayLocator(range(0,32,2)) )
    ax.xaxis.set_minor_locator( HourLocator(range(0,25,12)) )
    ax.set_xticklabels([])

    ax = axs[5]
    ax.set_xlim(date2num(dt[-1])-30, date2num(dt[-1]))
    ax.xaxis.set_major_locator( DayLocator(range(0,32,2)) )
    ax.xaxis.set_minor_locator( HourLocator(range(0,25,12)) )
    ax.xaxis.set_major_formatter( DateFormatter('%m/%d') )
    ax.set_xlabel('CROW Water Quality -- Last 7 days from ' + last_dt_str)

    # save figure
    savefig('/home/haines/rayleigh/img/crow_wq_last30days.png')

    #######################################
    # Last 7 days
    #######################################

    print ' ... Last 7 days'
    ax = axs[0]
    ax.set_xlim(date2num(dt[-1])-7, date2num(dt[-1]))
    ax.xaxis.set_major_locator( DayLocator(range(0,32,1)) )
    ax.xaxis.set_minor_locator( HourLocator(range(0,25,6)) )
    ax.set_xticklabels([])
    ax.set_xlabel('CROW Water Quality -- Last 7 days from ' + last_dt_str)

    ax = axs[1]
    ax.set_xlim(date2num(dt[-1])-7, date2num(dt[-1]))
    ax.xaxis.set_major_locator( DayLocator(range(2,32,1)) )
    ax.xaxis.set_minor_locator( HourLocator(range(0,25,6)) )
    ax.set_xticklabels([])

    ax = axs[2]
    ax.set_xlim(date2num(dt[-1])-7, date2num(dt[-1]))
    ax.xaxis.set_major_locator( DayLocator(range(2,32,1)) )
    ax.xaxis.set_minor_locator( HourLocator(range(0,25,6)) )
    ax.set_xticklabels([])

    ax = axs[3]
    ax.set_xlim(date2num(dt[-1])-7, date2num(dt[-1]))
    ax.xaxis.set_major_locator( DayLocator(range(2,32,1)) )
    ax.xaxis.set_minor_locator( HourLocator(range(0,25,6)) )
    ax.set_xticklabels([])

    ax = axs[4]
    ax.set_xlim(date2num(dt[-1])-7, date2num(dt[-1]))
    ax.xaxis.set_major_locator( DayLocator(range(2,32,1)) )
    ax.xaxis.set_minor_locator( HourLocator(range(0,25,6)) )
    ax.set_xticklabels([])

    ax = axs[5]
    ax.set_xlim(date2num(dt[-1])-7, date2num(dt[-1]))
    ax.xaxis.set_major_locator( DayLocator(range(2,32,1)) )
    ax.xaxis.set_minor_locator( HourLocator(range(0,25,6)) )
    ax.xaxis.set_major_formatter( DateFormatter('%m/%d') )
    ax.set_xlabel('CROW Water Quality -- Last 7 days from ' + last_dt_str)

    savefig('/home/haines/rayleigh/img/crow_wq_last07days.png')

    #######################################
    # Last 1 day (24hrs)
    #######################################

    print ' ... Last 1 days'

    ax = axs[0]
    ax.set_xlim(date2num(dt[-1])-1, date2num(dt[-1]))
    ax.xaxis.set_major_locator( HourLocator(range(0,25,1)) )
    ax.xaxis.set_minor_locator( MinuteLocator(range(0,61,30)) )
    ax.set_xticklabels([])
    ax.set_xlabel('CROW Water Quality -- Last 24 hours from ' + last_dt_str)

    ax = axs[1]
    ax.set_xlim(date2num(dt[-1])-1, date2num(dt[-1]))
    ax.xaxis.set_major_locator( HourLocator(range(0,25,1)) )
    ax.xaxis.set_minor_locator( MinuteLocator(range(0,61,30)) )
    ax.set_xticklabels([])

    ax = axs[2]
    ax.set_xlim(date2num(dt[-1])-1, date2num(dt[-1]))
    ax.xaxis.set_major_locator( HourLocator(range(0,25,1)) )
    ax.xaxis.set_minor_locator( MinuteLocator(range(0,61,30)) )
    ax.set_xticklabels([])

    ax = axs[3]
    ax.set_xlim(date2num(dt[-1])-1, date2num(dt[-1]))
    ax.xaxis.set_major_locator( HourLocator(range(0,25,1)) )
    ax.xaxis.set_minor_locator( MinuteLocator(range(0,61,30)) )
    ax.set_xticklabels([])

    ax = axs[4]
    ax.set_xlim(date2num(dt[-1])-1, date2num(dt[-1]))
    ax.xaxis.set_major_locator( HourLocator(range(0,25,1)) )
    ax.xaxis.set_minor_locator( MinuteLocator(range(0,61,30)) )
    ax.set_xticklabels([])

    ax = axs[5]
    ax.set_xlim(date2num(dt[-1])-1, date2num(dt[-1]))
    ax.xaxis.set_major_locator( HourLocator(range(0,25,1)) )
    ax.xaxis.set_minor_locator( MinuteLocator(range(0,61,30)) )
    ax.xaxis.set_major_formatter( DateFormatter('%H') )
    ax.set_xlabel('CROW Water Quality -- Last 24 hours from ' + last_dt_str)

    savefig('/home/haines/rayleigh/img/crow_wq_last01days.png')