1 |
#!/usr/bin/env python |
---|
2 |
# Last modified: Time-stamp: <2011-05-05 15:31:37 haines> |
---|
3 |
""" |
---|
4 |
Create, update and load utilities for netcdf files |
---|
5 |
""" |
---|
6 |
|
---|
7 |
from pycdf import * |
---|
8 |
import os |
---|
9 |
import numpy |
---|
10 |
|
---|
11 |
def nc_create(ncFile, (global_atts, var_atts, dim_inits, var_inits, var_data)): |
---|
12 |
""" |
---|
13 |
Create new netcdf file |
---|
14 |
|
---|
15 |
:Parameters: |
---|
16 |
ncFile : string |
---|
17 |
Path and name of file to create |
---|
18 |
(global_atts, var_atts, dim_inits, var_inits, var_data) : tuple |
---|
19 |
Global Attributes, Variable Attributes, Dimensions, Variable Dimensions, and Data |
---|
20 |
Everything you need to create a netCDF file. |
---|
21 |
""" |
---|
22 |
try: |
---|
23 |
# Open new netCDF file, overwrite if it exists, create if does not |
---|
24 |
nc = CDF(ncFile, NC.WRITE|NC.CREATE|NC.TRUNC) |
---|
25 |
# Automatically set define and data modes. |
---|
26 |
nc.automode() |
---|
27 |
# |
---|
28 |
# GLOBALS |
---|
29 |
for attrName in global_atts.keys(): |
---|
30 |
setattr(nc, attrName, global_atts[attrName]) |
---|
31 |
|
---|
32 |
# DIMENSIONS |
---|
33 |
for dim in dim_inits: |
---|
34 |
dimName, dimValue = dim |
---|
35 |
# print '%s = %d' % (dimName, dimValue) |
---|
36 |
ncdim = nc.def_dim(dimName, dimValue) |
---|
37 |
|
---|
38 |
# VARIABLES |
---|
39 |
for var in var_inits: |
---|
40 |
varName, varType, varDim = var |
---|
41 |
ncvar = nc.def_var(varName, varType, varDim) |
---|
42 |
# add attributes |
---|
43 |
for attrName in var_atts[varName].keys(): |
---|
44 |
setattr(ncvar, attrName, var_atts[varName][attrName]) |
---|
45 |
# setattr(ncvar, '_FillValue', numpy.nan) |
---|
46 |
|
---|
47 |
# add data |
---|
48 |
nrecs = nc.inq_unlimlen() |
---|
49 |
for var in var_data: |
---|
50 |
varName, varData = var |
---|
51 |
# print varName |
---|
52 |
# print varData |
---|
53 |
# print varData.shape |
---|
54 |
ncvar = nc.var(varName) |
---|
55 |
# e.g. lat = array(var_data['lat']) |
---|
56 |
# if an array |
---|
57 |
if type(varData) == numpy.ndarray: |
---|
58 |
if ncvar.isrecord(): |
---|
59 |
# time, ens, u, v |
---|
60 |
ncvar[nrecs:nrecs+len(varData)] = varData.tolist() |
---|
61 |
else: |
---|
62 |
ncvar[:] = varData.tolist() # z |
---|
63 |
else: |
---|
64 |
# if tuple, sequence or scalar |
---|
65 |
ncvar[:] = varData |
---|
66 |
|
---|
67 |
nc.close() |
---|
68 |
except CDFError, msg: |
---|
69 |
print "CDFError:", msg |
---|
70 |
# if nc: |
---|
71 |
# nc.close() |
---|
72 |
# del(nc) |
---|
73 |
|
---|
74 |
def nc_update(ncFile, (global_atts, var_atts, var_data)): |
---|
75 |
""" |
---|
76 |
Create new netcdf file |
---|
77 |
|
---|
78 |
:Parameters: |
---|
79 |
ncFile : string |
---|
80 |
Path and name of file to create |
---|
81 |
(global_atts, var_atts, var_data) : tuple |
---|
82 |
Global Attributes, Variable Attributes and Data |
---|
83 |
Everything you need to update a netCDF file. |
---|
84 |
""" |
---|
85 |
try: |
---|
86 |
# Open netCDF in write mode |
---|
87 |
nc = CDF(ncFile, NC.WRITE) |
---|
88 |
# Automatically set define and data modes. |
---|
89 |
nc.automode() |
---|
90 |
# |
---|
91 |
# GLOBALS |
---|
92 |
for attrName in global_atts.keys(): |
---|
93 |
setattr(nc, attrName, global_atts[attrName]) |
---|
94 |
|
---|
95 |
# VARIABLES |
---|
96 |
# update attributes |
---|
97 |
for var in var_atts: |
---|
98 |
varName, atts = var |
---|
99 |
ncvar = nc.var(varName) |
---|
100 |
for attrName in atts.keys(): |
---|
101 |
setattr(ncvar, attrName, atts[attrName]) |
---|
102 |
|
---|
103 |
# update data |
---|
104 |
nrecs = nc.inq_unlimlen() |
---|
105 |
for var in var_data: |
---|
106 |
varName, varData = var |
---|
107 |
ncvar = nc.var(varName) |
---|
108 |
# e.g. lat = array(var_data['lat']) |
---|
109 |
# if an array |
---|
110 |
if type(varData) == numpy.ndarray: |
---|
111 |
if ncvar.isrecord(): |
---|
112 |
# time, ens, u, v (with unlimited dimension) |
---|
113 |
ncvar[nrecs:nrecs+len(varData)] = varData.tolist() |
---|
114 |
else: |
---|
115 |
ncvar[:] = varData.tolist() # z (limited dimension) |
---|
116 |
else: |
---|
117 |
# if tuple, sequence or scalar |
---|
118 |
ncvar[:] = varData |
---|
119 |
|
---|
120 |
nc.close() |
---|
121 |
except CDFError, msg: |
---|
122 |
print "CDFError:", msg |
---|
123 |
# if nc: |
---|
124 |
# nc.close() |
---|
125 |
# del(nc) |
---|
126 |
|
---|
127 |
def nc_get_time(ncFile): |
---|
128 |
"""Get time array from file """ |
---|
129 |
try: |
---|
130 |
nc = CDF(ncFile) |
---|
131 |
ncvars = nc.variables() |
---|
132 |
if 'time' in ncvars.keys(): |
---|
133 |
es = nc.var('time')[:] |
---|
134 |
units = nc.var('time').units |
---|
135 |
else: |
---|
136 |
print "time variable not found in ", ncFile |
---|
137 |
nc.close() |
---|
138 |
return (es, units) |
---|
139 |
except CDFError, msg: |
---|
140 |
print "CDFError:", msg |
---|
141 |
|
---|
142 |
|
---|
143 |
def nc_find_record_vars(ncFile): |
---|
144 |
"""Find which variable are record variables""" |
---|
145 |
try: |
---|
146 |
nc = CDF(ncFile) |
---|
147 |
ncvars = nc.variables() |
---|
148 |
# list which variables is a record variable |
---|
149 |
var_list = [varName for varName in ncvars.keys() if nc.var(varName).isrecord()] |
---|
150 |
nc.close() |
---|
151 |
return var_list |
---|
152 |
except CDFError, msg: |
---|
153 |
print "CDFError:", msg |
---|
154 |
|
---|
155 |
|
---|
156 |
def nc_replace_fillvalue(ncFile, newfillvalue=-99999.0): |
---|
157 |
""" |
---|
158 |
Replaces any occurrence of old _FillValue with new one |
---|
159 |
|
---|
160 |
This function is useful for replacing the _FillValue global |
---|
161 |
attribute and then searching the data for the old value and |
---|
162 |
replacing it with the new one. |
---|
163 |
|
---|
164 |
:Parameters: |
---|
165 |
ncFile : string |
---|
166 |
Path and name of file to create |
---|
167 |
|
---|
168 |
:Other Parameters: |
---|
169 |
newfillvalue : type match to data (generally float) |
---|
170 |
By default is -99999.0 |
---|
171 |
|
---|
172 |
""" |
---|
173 |
try: |
---|
174 |
nc = CDF(ncFile, NC.WRITE) |
---|
175 |
nc.automode() |
---|
176 |
oldfillvalue = nc._FillValue |
---|
177 |
nc._FillValue = newfillvalue |
---|
178 |
for v in nc.variables().keys(): |
---|
179 |
vd = nc.var(v)[:] |
---|
180 |
if numpy.isnan(oldfillvalue): |
---|
181 |
idx = numpy.isnan(vd) |
---|
182 |
else: |
---|
183 |
idx = vd == oldfillvalue |
---|
184 |
if idx.any(): |
---|
185 |
vd[idx] = nc._FillValue |
---|
186 |
nc.var(v)[:] = vd |
---|
187 |
nc.close() |
---|
188 |
except CDFError, msg: |
---|
189 |
print "CDFError:", msg |
---|
190 |
|
---|
191 |
def nc_rename_dimension(ncFile, oldname, newname): |
---|
192 |
""" Rename dimension name """ |
---|
193 |
try: |
---|
194 |
nc = CDF(ncFile, NC.WRITE) |
---|
195 |
nc.definemode() |
---|
196 |
for d in nc.dimensions().keys(): |
---|
197 |
if d==oldname: nc.dim(d).rename(newname) |
---|
198 |
nc.close() |
---|
199 |
except CDFError, msg: |
---|
200 |
print "CDFError:", msg |
---|
201 |
|
---|
202 |
|
---|
203 |
def nc_file_check(fns): |
---|
204 |
"""Check file or list of files to ensure it is a netcdf file |
---|
205 |
If it is not, remove a file or files from the list""" |
---|
206 |
if isinstance(fns, str): |
---|
207 |
try: |
---|
208 |
nc = CDF(fns) |
---|
209 |
nc.close() |
---|
210 |
new_fns = fns |
---|
211 |
except CDFError, msg: |
---|
212 |
print "CDFError:", msg, fns |
---|
213 |
new_fns = None |
---|
214 |
|
---|
215 |
else: |
---|
216 |
new_fns = [] |
---|
217 |
for fn in fns: |
---|
218 |
try: |
---|
219 |
nc = CDF(fn) |
---|
220 |
nc.close() |
---|
221 |
new_fns.append(fn) |
---|
222 |
except CDFError, msg: |
---|
223 |
print "CDFError:", msg, fn |
---|
224 |
|
---|
225 |
return tuple(new_fns) |
---|
226 |
|
---|
227 |
|
---|
228 |
def nc_load(ncFile, varsLoad='all', nameType='variable_name', |
---|
229 |
ga_flag=True, va_flag=True): |
---|
230 |
""" |
---|
231 |
Load netcdf file |
---|
232 |
|
---|
233 |
:Parameters: |
---|
234 |
ncFile : string or list of strings |
---|
235 |
Path and name of file to load |
---|
236 |
If list, then CDFMF |
---|
237 |
|
---|
238 |
:Other Parameters: |
---|
239 |
nameType : string 'variable_name' (default) or 'standard_name' |
---|
240 |
Defines naming convention to use for variable names as data |
---|
241 |
are loaded. Variable name is the name used to store data |
---|
242 |
in file. 'standard_name' means use variable name based on |
---|
243 |
variable attribute called 'standard_name' of netcdf variable. |
---|
244 |
varLoad : string or tuple of strings |
---|
245 |
specific variable names to be loaded into a sequence or scalar |
---|
246 |
in python following specification set in nameType |
---|
247 |
By default, all variables will be loaded. |
---|
248 |
ga_flag : boolean flag |
---|
249 |
By default, load the global file attributes |
---|
250 |
va_flag : boolean flag |
---|
251 |
By default, load the variable file attributes |
---|
252 |
|
---|
253 |
:Returns: |
---|
254 |
(global_atts, var_atts, dim_inits, var_inits, var_data) : tuple |
---|
255 |
Global Attributes, Variable Attributes, Dimensions, Variable Dimensions, and Variable Data |
---|
256 |
Everything you need to create a netCDF file. |
---|
257 |
|
---|
258 |
""" |
---|
259 |
|
---|
260 |
try: |
---|
261 |
if isinstance(ncFile, str): |
---|
262 |
# if only one file and it is a string |
---|
263 |
nc = CDF(ncFile) |
---|
264 |
else: |
---|
265 |
# if multiple filenames |
---|
266 |
nc = CDFMF(tuple(set(ncFile))) |
---|
267 |
|
---|
268 |
ncdims = nc.dimensions(full=1) |
---|
269 |
ncvars = nc.variables() |
---|
270 |
|
---|
271 |
# GLOBAL ATTRIBUTES (global_atts) |
---|
272 |
if ga_flag: |
---|
273 |
global_atts = nc.attributes() |
---|
274 |
else: |
---|
275 |
global_atts = {} |
---|
276 |
|
---|
277 |
# DIMENSIONS (dim_inits) |
---|
278 |
dim_inits = [None for j in range(len(ncdims))] |
---|
279 |
if len(ncdims)>0: |
---|
280 |
for dimName,dimValue in ncdims.items(): |
---|
281 |
val,idx,isUN = dimValue |
---|
282 |
if isUN: |
---|
283 |
dim_inits[idx] = (dimName, NC.UNLIMITED) |
---|
284 |
else: |
---|
285 |
dim_inits[idx] = (dimName, val) |
---|
286 |
|
---|
287 |
if varsLoad == 'all': |
---|
288 |
varNames = ncvars.keys() |
---|
289 |
else: |
---|
290 |
varNames = varsLoad |
---|
291 |
|
---|
292 |
# VARIABLE DIMENSIONS (var_inits) |
---|
293 |
# gets init info for requested variables and original order |
---|
294 |
|
---|
295 |
# initialize with same number of original variables |
---|
296 |
# so order can be preserved by idx |
---|
297 |
var_inits = [None for j in range(len(ncvars))] |
---|
298 |
if len(ncvars)>0: |
---|
299 |
for varName in varNames: |
---|
300 |
val,shape,typ,idx = ncvars[varName] |
---|
301 |
var_inits[idx] = (varName, typ, val) |
---|
302 |
|
---|
303 |
# remove the None values from the list to preserve original order |
---|
304 |
var_inits = [v for v in var_inits if v != None] |
---|
305 |
|
---|
306 |
# VARIABLE ATTRIBUTES (var_atts) |
---|
307 |
# gets attributes of requested variables |
---|
308 |
var_atts = {} |
---|
309 |
if len(ncvars)>0 and va_flag: |
---|
310 |
for var in varNames: |
---|
311 |
varAttrs = nc.var(var).attributes() |
---|
312 |
var_atts[var] = varAttrs |
---|
313 |
|
---|
314 |
# VARIABLE DATA (var_data) |
---|
315 |
# loads requested variables, original order preserved as with var_inits |
---|
316 |
var_data = [None for j in range(len(ncvars))] |
---|
317 |
if len(ncvars)>0: |
---|
318 |
for varName in varNames: |
---|
319 |
val,shape,typ,idx = ncvars[varName] |
---|
320 |
var_data[idx] = (varName, nc.var(varName)[:]) |
---|
321 |
|
---|
322 |
var_data = [v for v in var_data if v != None] |
---|
323 |
|
---|
324 |
# type cast lists into tuples |
---|
325 |
dim_inits = tuple(dim_inits) |
---|
326 |
var_inits = tuple(var_inits) |
---|
327 |
var_data = tuple(var_data) |
---|
328 |
|
---|
329 |
nc.close() |
---|
330 |
return (global_atts, var_atts, dim_inits, var_inits, var_data) |
---|
331 |
|
---|
332 |
except CDFError, msg: |
---|
333 |
print "CDFError:", msg |
---|
334 |
|
---|
335 |
|
---|