1 |
# -*- coding: utf-8 -*- |
---|
2 |
# <nbformat>3.0</nbformat> |
---|
3 |
|
---|
4 |
# <codecell> |
---|
5 |
|
---|
6 |
import os |
---|
7 |
import datetime |
---|
8 |
import re |
---|
9 |
import shutil |
---|
10 |
import numpy as n |
---|
11 |
from glob import glob |
---|
12 |
from scipy.io import netcdf |
---|
13 |
|
---|
14 |
# <codecell> |
---|
15 |
|
---|
16 |
def create_atts(ncfile, newfile, datecode): |
---|
17 |
now = datetime.datetime.utcnow().replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S") |
---|
18 |
newfile.title = ncfile.title |
---|
19 |
newfile.institution = ncfile.institution |
---|
20 |
newfile.institution_url = ncfile.institution_url |
---|
21 |
newfile.institution_dods_url = ncfile.institution_dods_url |
---|
22 |
newfile.metadata_url = ncfile.metadata_url |
---|
23 |
newfile.references = ncfile.references |
---|
24 |
newfile.contact = ncfile.contact |
---|
25 |
newfile.source = ncfile.source |
---|
26 |
newfile.history = ncfile.history |
---|
27 |
newfile.comment = ncfile.comment |
---|
28 |
newfile.Conventions = ncfile.Conventions |
---|
29 |
newfile.format_category_code = ncfile.format_category_code |
---|
30 |
newfile.institution_code = ncfile.institution_code |
---|
31 |
newfile.platform_code = ncfile.platform_code |
---|
32 |
newfile.package_code = ncfile.package_code |
---|
33 |
newfile.project = ncfile.project |
---|
34 |
newfile.project_url = ncfile.project_url |
---|
35 |
newfile.start_date = ncfile.start_date |
---|
36 |
newfile.end_date = ncfile.end_date |
---|
37 |
newfile.release_date = now |
---|
38 |
newfile.creation_date = now |
---|
39 |
newfile.modification_date = now |
---|
40 |
newfile.process_level = 'level2' |
---|
41 |
if datecode < '2013_02': |
---|
42 |
newfile.level2_flag_elevation_correction = True |
---|
43 |
if '2011_09' < datecode < '2012_05': |
---|
44 |
newfile.level2_flag_bottom_z_nan_padded = True |
---|
45 |
newfile.level2_flag_top_z_nan_padded = True |
---|
46 |
if '2011_05' < datecode < '2011_11': |
---|
47 |
newfile.level2_flag_upper_z_cropped = True |
---|
48 |
newfile._FillValue = ncfile._FillValue |
---|
49 |
|
---|
50 |
|
---|
51 |
# <codecell> |
---|
52 |
|
---|
53 |
def create_dims(ncfile, newfile): |
---|
54 |
newfile.createDimension('ntime', ncfile.dimensions['ntime']) |
---|
55 |
newfile.createDimension('nlat', ncfile.dimensions['nlat']) |
---|
56 |
newfile.createDimension('nlon', ncfile.dimensions['nlon']) |
---|
57 |
newfile.createDimension('nz', 39) |
---|
58 |
|
---|
59 |
|
---|
60 |
# <codecell> |
---|
61 |
|
---|
62 |
def create_vars(ncfile, newfile, zs, newz): |
---|
63 |
ncfiles = None |
---|
64 |
if type(ncfile) is list: |
---|
65 |
ncfiles = ncfile |
---|
66 |
ncfile = ncfile[0] |
---|
67 |
|
---|
68 |
time = newfile.createVariable('time', 'i', ('ntime',)) |
---|
69 |
if ncfiles: |
---|
70 |
time[:] = n.concatenate([ncf.variables['time'].data for ncf in ncfiles]) |
---|
71 |
else: |
---|
72 |
time[:] = ncfile.variables['time'].data |
---|
73 |
time.short_name = ncfile.variables['time'].short_name |
---|
74 |
time.long_name = ncfile.variables['time'].long_name |
---|
75 |
time.standard_name = ncfile.variables['time'].standard_name |
---|
76 |
time.units = ncfile.variables['time'].units |
---|
77 |
time.axis = ncfile.variables['time'].axis |
---|
78 |
|
---|
79 |
lat = newfile.createVariable('lat', 'f', ('nlat',)) |
---|
80 |
lat[:] = ncfile.variables['lat'].data |
---|
81 |
lat.short_name = ncfile.variables['lat'].short_name |
---|
82 |
lat.long_name = ncfile.variables['lat'].long_name |
---|
83 |
lat.standard_name = ncfile.variables['lat'].standard_name |
---|
84 |
lat.reference = ncfile.variables['lat'].reference |
---|
85 |
lat.units = ncfile.variables['lat'].units |
---|
86 |
lat.valid_range = ncfile.variables['lat'].valid_range |
---|
87 |
lat.axis = ncfile.variables['lat'].axis |
---|
88 |
|
---|
89 |
lon = newfile.createVariable('lon', 'f', ('nlon',)) |
---|
90 |
lon[:] = ncfile.variables['lon'].data |
---|
91 |
lon.short_name = ncfile.variables['lon'].short_name |
---|
92 |
lon.long_name = ncfile.variables['lon'].long_name |
---|
93 |
lon.standard_name = ncfile.variables['lon'].standard_name |
---|
94 |
lon.reference = ncfile.variables['lon'].reference |
---|
95 |
lon.units = ncfile.variables['lon'].units |
---|
96 |
lon.valid_range = ncfile.variables['lon'].valid_range |
---|
97 |
lon.axis = ncfile.variables['lon'].axis |
---|
98 |
|
---|
99 |
z = newfile.createVariable('z', 'f', ('nz',)) |
---|
100 |
if newz is not None: |
---|
101 |
z[:] = newz |
---|
102 |
else: |
---|
103 |
z[:] = ncfile.variables['z'].data |
---|
104 |
z.short_name = ncfile.variables['z'].short_name |
---|
105 |
z.long_name = ncfile.variables['z'].long_name |
---|
106 |
z.standard_name = ncfile.variables['z'].standard_name |
---|
107 |
z.reference = ncfile.variables['z'].reference |
---|
108 |
z.positive = ncfile.variables['z'].positive |
---|
109 |
z.units = ncfile.variables['z'].units |
---|
110 |
z.axis = ncfile.variables['z'].axis |
---|
111 |
|
---|
112 |
u = newfile.createVariable('u', 'f', ('ntime', 'nz')) |
---|
113 |
if ncfiles: |
---|
114 |
u[:,:] = n.concatenate([ncf.variables['u'].data for ncf in ncfiles]) |
---|
115 |
else: |
---|
116 |
if zs < 39: |
---|
117 |
u[:,:] = ones(u.shape, dtype=float) * nan |
---|
118 |
u[:,1:38] = ncfile.variables['u'].data |
---|
119 |
elif zs == 39: |
---|
120 |
u[:,:] = ncfile.variables['u'].data |
---|
121 |
else: |
---|
122 |
u[:,:] = ncfile.variables['u'].data[:,0:39] |
---|
123 |
u.short_name = ncfile.variables['u'].short_name |
---|
124 |
u.long_name = ncfile.variables['u'].long_name |
---|
125 |
u.standard_name = ncfile.variables['u'].standard_name |
---|
126 |
u.positive = ncfile.variables['u'].positive |
---|
127 |
u.units = ncfile.variables['u'].units |
---|
128 |
|
---|
129 |
v = newfile.createVariable('v', 'f', ('ntime', 'nz')) |
---|
130 |
if ncfiles: |
---|
131 |
v[:,:] = n.concatenate([ncf.variables['v'].data for ncf in ncfiles]) |
---|
132 |
else: |
---|
133 |
if zs < 39: |
---|
134 |
v[:,:] = ones(u.shape, dtype=float) * nan |
---|
135 |
v[:,1:38] = ncfile.variables['v'].data |
---|
136 |
elif zs == 39: |
---|
137 |
v[:,:] = ncfile.variables['v'].data |
---|
138 |
else: |
---|
139 |
v[:,:] = ncfile.variables['v'].data[:,0:39] |
---|
140 |
v.short_name = ncfile.variables['v'].short_name |
---|
141 |
v.long_name = ncfile.variables['v'].long_name |
---|
142 |
v.standard_name = ncfile.variables['v'].standard_name |
---|
143 |
v.positive = ncfile.variables['v'].positive |
---|
144 |
v.units = ncfile.variables['v'].units |
---|
145 |
|
---|
146 |
w = newfile.createVariable('w', 'f', ('ntime', 'nz')) |
---|
147 |
if ncfiles: |
---|
148 |
w[:,:] = n.concatenate([ncf.variables['w'].data for ncf in ncfiles]) |
---|
149 |
else: |
---|
150 |
if zs < 39: |
---|
151 |
w[:,:] = ones(v.shape, dtype=float) * nan |
---|
152 |
w[:,1:38] = ncfile.variables['w'].data |
---|
153 |
elif zs == 39: |
---|
154 |
w[:,:] = ncfile.variables['w'].data |
---|
155 |
else: |
---|
156 |
w[:,:] = ncfile.variables['w'].data[:,0:39] |
---|
157 |
w.short_name = ncfile.variables['w'].short_name |
---|
158 |
w.long_name = ncfile.variables['w'].long_name |
---|
159 |
w.standard_name = ncfile.variables['w'].standard_name |
---|
160 |
w.positive = ncfile.variables['w'].positive |
---|
161 |
w.units = ncfile.variables['w'].units |
---|
162 |
|
---|
163 |
sigw = newfile.createVariable('sigw', 'f', ('ntime', 'nz')) |
---|
164 |
if ncfiles: |
---|
165 |
sigw[:,:] = n.concatenate([ncf.variables['sigw'].data for ncf in ncfiles]) |
---|
166 |
else: |
---|
167 |
if zs < 39: |
---|
168 |
sigw[:,:] = ones(sigw.shape, dtype=float) * nan |
---|
169 |
sigw[:,1:38] = ncfile.variables['sigw'].data |
---|
170 |
elif zs == 39: |
---|
171 |
sigw[:,:] = ncfile.variables['sigw'].data |
---|
172 |
else: |
---|
173 |
sigw[:,:] = ncfile.variables['sigw'].data[:,0:39] |
---|
174 |
sigw.short_name = ncfile.variables['sigw'].short_name |
---|
175 |
sigw.long_name = ncfile.variables['sigw'].long_name |
---|
176 |
sigw.standard_name = ncfile.variables['sigw'].standard_name |
---|
177 |
|
---|
178 |
bck = newfile.createVariable('bck', 'f', ('ntime', 'nz')) |
---|
179 |
if ncfiles: |
---|
180 |
bck[:,:] = n.concatenate([ncf.variables['bck'].data for ncf in ncfiles]) |
---|
181 |
else: |
---|
182 |
if zs < 39: |
---|
183 |
bck[:,:] = ones(bck.shape, dtype=float) * nan |
---|
184 |
bck[:,1:38] = ncfile.variables['bck'].data |
---|
185 |
elif zs == 39: |
---|
186 |
bck[:,:] = ncfile.variables['bck'].data |
---|
187 |
else: |
---|
188 |
bck[:,:] = ncfile.variables['bck'].data[:,0:39] |
---|
189 |
bck.short_name = ncfile.variables['bck'].short_name |
---|
190 |
bck.long_name = ncfile.variables['bck'].long_name |
---|
191 |
bck.standard_name = ncfile.variables['bck'].standard_name |
---|
192 |
|
---|
193 |
error = newfile.createVariable('error', 'i', ('ntime', 'nz')) |
---|
194 |
if ncfiles: |
---|
195 |
error[:,:] = n.concatenate([ncf.variables['error'].data for ncf in ncfiles]) |
---|
196 |
else: |
---|
197 |
if zs < 39: |
---|
198 |
error[:,:] = zeros(error.shape, dtype=int) |
---|
199 |
error[:,1:38] = ncfile.variables['error'].data |
---|
200 |
elif zs == 39: |
---|
201 |
error[:,:] = ncfile.variables['error'].data |
---|
202 |
else: |
---|
203 |
error[:,:] = ncfile.variables['error'].data[:,0:39] |
---|
204 |
error.short_name = ncfile.variables['error'].short_name |
---|
205 |
error.long_name = ncfile.variables['error'].long_name |
---|
206 |
error.standard_name = ncfile.variables['error'].standard_name |
---|
207 |
|
---|
208 |
# <codecell> |
---|
209 |
|
---|
210 |
def combine(ncfiles, dest): |
---|
211 |
print 'Writing', os.path.basename(dest) |
---|
212 |
newfile = netcdf.netcdf_file(dest, 'w') |
---|
213 |
try: |
---|
214 |
pattern = re.compile(r'(\d{4}_\d{2})') |
---|
215 |
create_atts(ncfiles[0], newfile, pattern.search(os.path.basename(dest)).groups()[0]) |
---|
216 |
create_dims(ncfiles[0], newfile) |
---|
217 |
create_vars(ncfiles, newfile, None, None) |
---|
218 |
finally: |
---|
219 |
newfile.close() |
---|
220 |
|
---|
221 |
# <codecell> |
---|
222 |
|
---|
223 |
home = os.path.join(os.sep, 'Users', 'cbcoasis', 'Documents', 'sodar') |
---|
224 |
src = os.path.join(home, 'ncs') |
---|
225 |
dest = os.path.join(home, 'new') |
---|
226 |
ncext = 'nc' |
---|
227 |
ncs = os.path.join(src, '*' + os.extsep + ncext) |
---|
228 |
ncs = glob(ncs) |
---|
229 |
ncs = [os.path.basename(nc) for nc in ncs] |
---|
230 |
ncs.sort() |
---|
231 |
|
---|
232 |
# <codecell> |
---|
233 |
|
---|
234 |
for nc in ncs: |
---|
235 |
print 'Starting', nc |
---|
236 |
ncfile = netcdf.netcdf_file(os.path.join(src, nc)) |
---|
237 |
try: |
---|
238 |
newz = None |
---|
239 |
zs = len(ncfile.variables['z'].data) |
---|
240 |
z0 = ncfile.variables['z'].data[0] |
---|
241 |
if zs < 39: |
---|
242 |
newz = ncfile.variables['z'].data |
---|
243 |
newz = insert(newz, 0, float32(10.0)) |
---|
244 |
newz = append(newz, float32(200.0)) |
---|
245 |
newz = newz + 1.0 |
---|
246 |
elif zs == 39: |
---|
247 |
if z0 != 11.0: |
---|
248 |
newz = ncfile.variables['z'].data + 1.0 |
---|
249 |
else: |
---|
250 |
newz = ncfile.variables['z'].data[0:39] + 1.0 |
---|
251 |
newfile = netcdf.netcdf_file(os.path.join(dest, nc), 'w') |
---|
252 |
try: |
---|
253 |
pattern = re.compile(r'(\d{4}_\d{2})') |
---|
254 |
create_atts(ncfile, newfile, pattern.search(nc).groups()[0]) |
---|
255 |
create_dims(ncfile, newfile) |
---|
256 |
create_vars(ncfile, newfile, zs, newz) |
---|
257 |
finally: |
---|
258 |
newfile.close() |
---|
259 |
finally: |
---|
260 |
ncfile.close() |
---|
261 |
print 'Finishing', nc |
---|
262 |
|
---|
263 |
# <codecell> |
---|
264 |
|
---|
265 |
src = os.path.join(home, 'new') |
---|
266 |
ncs = os.path.join(src, '*' + os.extsep + ncext) |
---|
267 |
ncs = glob(ncs) |
---|
268 |
ncs = [os.path.basename(nc) for nc in ncs] |
---|
269 |
ncs.sort() |
---|
270 |
pattern = re.compile(r'(\d{4}_\d{2})[A-Za-z]+') |
---|
271 |
keys = [] |
---|
272 |
for nc in ncs: |
---|
273 |
try: |
---|
274 |
key = pattern.search(nc).groups()[0] |
---|
275 |
if key not in keys: |
---|
276 |
keys.append(key) |
---|
277 |
except: |
---|
278 |
pass |
---|
279 |
|
---|
280 |
|
---|
281 |
# <codecell> |
---|
282 |
|
---|
283 |
dest = os.path.join(home, 'newer') |
---|
284 |
for key in keys: |
---|
285 |
ncs = os.path.join(src, '*' + key + '*' + os.extsep + ncext) |
---|
286 |
ncs = glob(ncs) |
---|
287 |
ncs = [os.path.basename(nc) for nc in ncs] |
---|
288 |
ncs.sort |
---|
289 |
ncfiles = [] |
---|
290 |
try: |
---|
291 |
for nc in ncs: |
---|
292 |
print 'Reading', nc |
---|
293 |
ncfiles.append(netcdf.netcdf_file(os.path.join(src, nc))) |
---|
294 |
combine(ncfiles,os.path.join(dest, ncs[0])) |
---|
295 |
finally: |
---|
296 |
for ncfile in ncfiles: |
---|
297 |
ncfile.close() |
---|
298 |
ncfiles = []; ncfile = None # close doesn't work on weak references? |
---|
299 |
|
---|
300 |
|
---|
301 |
# <codecell> |
---|
302 |
|
---|
303 |
dest = os.path.join(home, 'newest') |
---|
304 |
shutil.rmtree(dest, True) |
---|
305 |
shutil.copytree(src, dest) |
---|
306 |
for key in keys: |
---|
307 |
ncs = os.path.join(dest, '*' + key + '*' + os.extsep + ncext) |
---|
308 |
ncs = glob(ncs) |
---|
309 |
for nc in ncs: |
---|
310 |
os.remove(nc) |
---|
311 |
src = os.path.join(home, 'newer') |
---|
312 |
ncs = os.path.join(src, '*' + os.extsep + ncext) |
---|
313 |
ncs = glob(ncs) |
---|
314 |
for nc in ncs: |
---|
315 |
shutil.copy(nc, dest) |
---|
316 |
|
---|
317 |
|
---|