Index: misc_sodar_scripts/SodarLevel2.py
===================================================================
--- (revision )
+++ misc_sodar_scripts/SodarLevel2.py (revision 520)
@@ -1,0 +1,317 @@
+# -*- coding: utf-8 -*-
+# 3.0
+
+#
+
+import os
+import datetime
+import re
+import shutil
+import numpy as n
+from glob import glob
+from scipy.io import netcdf
+
+#
+
+def create_atts(ncfile, newfile, datecode):
+ now = datetime.datetime.utcnow().replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
+ newfile.title = ncfile.title
+ newfile.institution = ncfile.institution
+ newfile.institution_url = ncfile.institution_url
+ newfile.institution_dods_url = ncfile.institution_dods_url
+ newfile.metadata_url = ncfile.metadata_url
+ newfile.references = ncfile.references
+ newfile.contact = ncfile.contact
+ newfile.source = ncfile.source
+ newfile.history = ncfile.history
+ newfile.comment = ncfile.comment
+ newfile.Conventions = ncfile.Conventions
+ newfile.format_category_code = ncfile.format_category_code
+ newfile.institution_code = ncfile.institution_code
+ newfile.platform_code = ncfile.platform_code
+ newfile.package_code = ncfile.package_code
+ newfile.project = ncfile.project
+ newfile.project_url = ncfile.project_url
+ newfile.start_date = ncfile.start_date
+ newfile.end_date = ncfile.end_date
+ newfile.release_date = now
+ newfile.creation_date = now
+ newfile.modification_date = now
+ newfile.process_level = 'level2'
+ if datecode < '2013_02':
+ newfile.level2_flag_elevation_correction = True
+ if '2011_09' < datecode < '2012_05':
+ newfile.level2_flag_bottom_z_nan_padded = True
+ newfile.level2_flag_top_z_nan_padded = True
+ if '2011_05' < datecode < '2011_11':
+ newfile.level2_flag_upper_z_nan_cropped = True
+ newfile._FillValue = ncfile._FillValue
+
+
+#
+
+def create_dims(ncfile, newfile):
+ newfile.createDimension('ntime', ncfile.dimensions['ntime'])
+ newfile.createDimension('nlat', ncfile.dimensions['nlat'])
+ newfile.createDimension('nlon', ncfile.dimensions['nlon'])
+ newfile.createDimension('nz', 39)
+
+
+#
+
+def create_vars(ncfile, newfile, zs, newz):
+ ncfiles = None
+ if type(ncfile) is list:
+ ncfiles = ncfile
+ ncfile = ncfile[0]
+
+ time = newfile.createVariable('time', 'i', ('ntime',))
+ if ncfiles:
+ time[:] = n.concatenate([ncf.variables['time'].data for ncf in ncfiles])
+ else:
+ time[:] = ncfile.variables['time'].data
+ time.short_name = ncfile.variables['time'].short_name
+ time.long_name = ncfile.variables['time'].long_name
+ time.standard_name = ncfile.variables['time'].standard_name
+ time.units = ncfile.variables['time'].units
+ time.axis = ncfile.variables['time'].axis
+
+ lat = newfile.createVariable('lat', 'f', ('nlat',))
+ lat[:] = ncfile.variables['lat'].data
+ lat.short_name = ncfile.variables['lat'].short_name
+ lat.long_name = ncfile.variables['lat'].long_name
+ lat.standard_name = ncfile.variables['lat'].standard_name
+ lat.reference = ncfile.variables['lat'].reference
+ lat.units = ncfile.variables['lat'].units
+ lat.valid_range = ncfile.variables['lat'].valid_range
+ lat.axis = ncfile.variables['lat'].axis
+
+ lon = newfile.createVariable('lon', 'f', ('nlon',))
+ lon[:] = ncfile.variables['lon'].data
+ lon.short_name = ncfile.variables['lon'].short_name
+ lon.long_name = ncfile.variables['lon'].long_name
+ lon.standard_name = ncfile.variables['lon'].standard_name
+ lon.reference = ncfile.variables['lon'].reference
+ lon.units = ncfile.variables['lon'].units
+ lon.valid_range = ncfile.variables['lon'].valid_range
+ lon.axis = ncfile.variables['lon'].axis
+
+ z = newfile.createVariable('z', 'f', ('nz',))
+ if newz is not None:
+ z[:] = newz
+ else:
+ z[:] = ncfile.variables['z'].data
+ z.short_name = ncfile.variables['z'].short_name
+ z.long_name = ncfile.variables['z'].long_name
+ z.standard_name = ncfile.variables['z'].standard_name
+ z.reference = ncfile.variables['z'].reference
+ z.positive = ncfile.variables['z'].positive
+ z.units = ncfile.variables['z'].units
+ z.axis = ncfile.variables['z'].axis
+
+ u = newfile.createVariable('u', 'f', ('ntime', 'nz'))
+ if ncfiles:
+ u[:,:] = n.concatenate([ncf.variables['u'].data for ncf in ncfiles])
+ else:
+ if zs < 39:
+ u[:,:] = ones(u.shape, dtype=float) * nan
+ u[:,1:38] = ncfile.variables['u'].data
+ elif zs == 39:
+ u[:,:] = ncfile.variables['u'].data
+ else:
+ u[:,:] = ncfile.variables['u'].data[:,0:39]
+ u.short_name = ncfile.variables['u'].short_name
+ u.long_name = ncfile.variables['u'].long_name
+ u.standard_name = ncfile.variables['u'].standard_name
+ u.positive = ncfile.variables['u'].positive
+ u.units = ncfile.variables['u'].units
+
+ v = newfile.createVariable('v', 'f', ('ntime', 'nz'))
+ if ncfiles:
+ v[:,:] = n.concatenate([ncf.variables['v'].data for ncf in ncfiles])
+ else:
+ if zs < 39:
+ v[:,:] = ones(u.shape, dtype=float) * nan
+ v[:,1:38] = ncfile.variables['v'].data
+ elif zs == 39:
+ v[:,:] = ncfile.variables['v'].data
+ else:
+ v[:,:] = ncfile.variables['v'].data[:,0:39]
+ v.short_name = ncfile.variables['v'].short_name
+ v.long_name = ncfile.variables['v'].long_name
+ v.standard_name = ncfile.variables['v'].standard_name
+ v.positive = ncfile.variables['v'].positive
+ v.units = ncfile.variables['v'].units
+
+ w = newfile.createVariable('w', 'f', ('ntime', 'nz'))
+ if ncfiles:
+ w[:,:] = n.concatenate([ncf.variables['w'].data for ncf in ncfiles])
+ else:
+ if zs < 39:
+ w[:,:] = ones(v.shape, dtype=float) * nan
+ w[:,1:38] = ncfile.variables['w'].data
+ elif zs == 39:
+ w[:,:] = ncfile.variables['w'].data
+ else:
+ w[:,:] = ncfile.variables['w'].data[:,0:39]
+ w.short_name = ncfile.variables['w'].short_name
+ w.long_name = ncfile.variables['w'].long_name
+ w.standard_name = ncfile.variables['w'].standard_name
+ w.positive = ncfile.variables['w'].positive
+ w.units = ncfile.variables['w'].units
+
+ sigw = newfile.createVariable('sigw', 'f', ('ntime', 'nz'))
+ if ncfiles:
+ sigw[:,:] = n.concatenate([ncf.variables['sigw'].data for ncf in ncfiles])
+ else:
+ if zs < 39:
+ sigw[:,:] = ones(sigw.shape, dtype=float) * nan
+ sigw[:,1:38] = ncfile.variables['sigw'].data
+ elif zs == 39:
+ sigw[:,:] = ncfile.variables['sigw'].data
+ else:
+ sigw[:,:] = ncfile.variables['sigw'].data[:,0:39]
+ sigw.short_name = ncfile.variables['sigw'].short_name
+ sigw.long_name = ncfile.variables['sigw'].long_name
+ sigw.standard_name = ncfile.variables['sigw'].standard_name
+
+ bck = newfile.createVariable('bck', 'f', ('ntime', 'nz'))
+ if ncfiles:
+ bck[:,:] = n.concatenate([ncf.variables['bck'].data for ncf in ncfiles])
+ else:
+ if zs < 39:
+ bck[:,:] = ones(bck.shape, dtype=float) * nan
+ bck[:,1:38] = ncfile.variables['bck'].data
+ elif zs == 39:
+ bck[:,:] = ncfile.variables['bck'].data
+ else:
+ bck[:,:] = ncfile.variables['bck'].data[:,0:39]
+ bck.short_name = ncfile.variables['bck'].short_name
+ bck.long_name = ncfile.variables['bck'].long_name
+ bck.standard_name = ncfile.variables['bck'].standard_name
+
+ error = newfile.createVariable('error', 'i', ('ntime', 'nz'))
+ if ncfiles:
+ error[:,:] = n.concatenate([ncf.variables['error'].data for ncf in ncfiles])
+ else:
+ if zs < 39:
+ error[:,:] = zeros(error.shape, dtype=int)
+ error[:,1:38] = ncfile.variables['error'].data
+ elif zs == 39:
+ error[:,:] = ncfile.variables['error'].data
+ else:
+ error[:,:] = ncfile.variables['error'].data[:,0:39]
+ error.short_name = ncfile.variables['error'].short_name
+ error.long_name = ncfile.variables['error'].long_name
+ error.standard_name = ncfile.variables['error'].standard_name
+
+#
+
+def combine(ncfiles, dest):
+ print 'Writing', os.path.basename(dest)
+ newfile = netcdf.netcdf_file(dest, 'w')
+ try:
+ pattern = re.compile(r'(\d{4}_\d{2})')
+ create_atts(ncfiles[0], newfile, pattern.search(os.path.basename(dest)).groups()[0])
+ create_dims(ncfiles[0], newfile)
+ create_vars(ncfiles, newfile, None, None)
+ finally:
+ newfile.close()
+
+#
+
+home = os.path.join(os.sep, 'Users', 'cbcoasis', 'Documents', 'sodar')
+src = os.path.join(home, 'ncs')
+dest = os.path.join(home, 'new')
+ncext = 'nc'
+ncs = os.path.join(src, '*' + os.extsep + ncext)
+ncs = glob(ncs)
+ncs = [os.path.basename(nc) for nc in ncs]
+ncs.sort()
+
+#
+
+for nc in ncs:
+ print 'Starting', nc
+ ncfile = netcdf.netcdf_file(os.path.join(src, nc))
+ try:
+ newz = None
+ zs = len(ncfile.variables['z'].data)
+ z0 = ncfile.variables['z'].data[0]
+ if zs < 39:
+ newz = ncfile.variables['z'].data
+ newz = insert(newz, 0, float32(10.0))
+ newz = append(newz, float32(200.0))
+ newz = newz + 1.0
+ elif zs == 39:
+ if z0 != 11.0:
+ newz = ncfile.variables['z'].data + 1.0
+ else:
+ newz = ncfile.variables['z'].data[0:39] + 1.0
+ newfile = netcdf.netcdf_file(os.path.join(dest, nc), 'w')
+ try:
+ pattern = re.compile(r'(\d{4}_\d{2})')
+ create_atts(ncfile, newfile, pattern.search(nc).groups()[0])
+ create_dims(ncfile, newfile)
+ create_vars(ncfile, newfile, zs, newz)
+ finally:
+ newfile.close()
+ finally:
+ ncfile.close()
+ print 'Finishing', nc
+
+#
+
+src = os.path.join(home, 'new')
+ncs = os.path.join(src, '*' + os.extsep + ncext)
+ncs = glob(ncs)
+ncs = [os.path.basename(nc) for nc in ncs]
+ncs.sort()
+pattern = re.compile(r'(\d{4}_\d{2})[A-Za-z]+')
+keys = []
+for nc in ncs:
+ try:
+ key = pattern.search(nc).groups()[0]
+ if key not in keys:
+ keys.append(key)
+ except:
+ pass
+
+
+#
+
+dest = os.path.join(home, 'newer')
+for key in keys:
+ ncs = os.path.join(src, '*' + key + '*' + os.extsep + ncext)
+ ncs = glob(ncs)
+ ncs = [os.path.basename(nc) for nc in ncs]
+ ncs.sort
+ ncfiles = []
+ try:
+ for nc in ncs:
+ print 'Reading', nc
+ ncfiles.append(netcdf.netcdf_file(os.path.join(src, nc)))
+ combine(ncfiles,os.path.join(dest, ncs[0]))
+ finally:
+ for ncfile in ncfiles:
+ ncfile.close()
+ ncfiles = []; ncfile = None # close doesn't work on weak references?
+
+
+#
+
+dest = os.path.join(home, 'newest')
+shutil.rmtree(dest, True)
+shutil.copytree(src, dest)
+for key in keys:
+ ncs = os.path.join(dest, '*' + key + '*' + os.extsep + ncext)
+ ncs = glob(ncs)
+ for nc in ncs:
+ os.remove(nc)
+src = os.path.join(home, 'newer')
+ncs = os.path.join(src, '*' + os.extsep + ncext)
+ncs = glob(ncs)
+for nc in ncs:
+ shutil.copy(nc, dest)
+
+