1 |
#!/usr/bin/python |
---|
2 |
|
---|
3 |
r""" |
---|
4 |
Fetch and extract Ifremer ASCAT Daily NetCDF files. |
---|
5 |
|
---|
6 |
Usage: |
---|
7 |
python ascat.py start_year start_month start_day \ |
---|
8 |
end_year end_month end_day |
---|
9 |
|
---|
10 |
Where: |
---|
11 |
start_year is YYYY format, |
---|
12 |
start_month is 1-12 for Jan-Dec, |
---|
13 |
start_day is 1-31, |
---|
14 |
end_year is YYYY format, |
---|
15 |
end_month is 1-12 for Jan-Dec, |
---|
16 |
end_day is 1-31. |
---|
17 |
""" |
---|
18 |
|
---|
19 |
__author__ = 'Chris Calloway' |
---|
20 |
__email__ = 'cbc@unc.edu' |
---|
21 |
__copyright__ = 'Copyright 2012 UNC-CH Dept. of Marine Sciences' |
---|
22 |
__license__ = 'GPL2' |
---|
23 |
|
---|
24 |
import os |
---|
25 |
import sys |
---|
26 |
import traceback |
---|
27 |
import calendar |
---|
28 |
import ftplib |
---|
29 |
import bz2 |
---|
30 |
|
---|
31 |
ftp_server = 'ftp.ifremer.fr' |
---|
32 |
ftp_user = 'anonymous' |
---|
33 |
ftp_pass = 'cbc@unc.edu' |
---|
34 |
ftp_dir = 'ifremer/cersat/products/gridded/MWF/L3/ASCAT/Daily/Netcdf/' |
---|
35 |
|
---|
36 |
work_dir = os.path.join(os.path.sep, |
---|
37 |
'afs', |
---|
38 |
'isis.unc.edu', |
---|
39 |
'depts', |
---|
40 |
'marine', |
---|
41 |
'workspace', |
---|
42 |
'hseim', |
---|
43 |
'volumes', |
---|
44 |
'vol1', |
---|
45 |
'nc_wind_2011', |
---|
46 |
'satellite') |
---|
47 |
base_subdir = os.path.join('ascat', 'daily') |
---|
48 |
year_spec = '%(year)4u' |
---|
49 |
month_spec = '%(month)02u' |
---|
50 |
day_spec = '%(day)02u' |
---|
51 |
|
---|
52 |
def _fetch(f): |
---|
53 |
""" |
---|
54 |
Fetch a single bzip2 file. |
---|
55 |
""" |
---|
56 |
|
---|
57 |
try: |
---|
58 |
bz = f.nlst()[0] |
---|
59 |
b = open(bz, 'wb') |
---|
60 |
except KeyboardInterrupt: |
---|
61 |
raise |
---|
62 |
except: |
---|
63 |
traceback.print_exc() |
---|
64 |
return None |
---|
65 |
try: |
---|
66 |
print 'Fetching: %s.' % bz |
---|
67 |
print f.retrbinary('RETR ' + bz, b.write) |
---|
68 |
except KeyboardInterrupt: |
---|
69 |
raise |
---|
70 |
except: |
---|
71 |
traceback.print_exc() |
---|
72 |
b.close() |
---|
73 |
return None |
---|
74 |
else: |
---|
75 |
b.close() |
---|
76 |
return bz |
---|
77 |
|
---|
78 |
def _extract(bz): |
---|
79 |
""" |
---|
80 |
Extract a single bzip2 file. |
---|
81 |
""" |
---|
82 |
|
---|
83 |
b = os.path.split(bz) |
---|
84 |
b = os.path.join(b[0],b[1][:8] + os.extsep + "nc") |
---|
85 |
print 'Extracting: %s.' % b |
---|
86 |
try: |
---|
87 |
dest = open(b, 'wb') |
---|
88 |
except KeyboardInterrupt: |
---|
89 |
raise |
---|
90 |
except: |
---|
91 |
traceback.print_exc() |
---|
92 |
return False |
---|
93 |
try: |
---|
94 |
src = bz2.BZ2File(bz) |
---|
95 |
except KeyboardInterrupt: |
---|
96 |
raise |
---|
97 |
except: |
---|
98 |
dest.close() |
---|
99 |
traceback.print_exc() |
---|
100 |
return False |
---|
101 |
try: |
---|
102 |
content = src.read() |
---|
103 |
except KeyboardInterrupt: |
---|
104 |
raise |
---|
105 |
except: |
---|
106 |
dest.close() |
---|
107 |
src.close() |
---|
108 |
traceback.print_exc() |
---|
109 |
return False |
---|
110 |
else: |
---|
111 |
src.close() |
---|
112 |
try: |
---|
113 |
dest.write(content) |
---|
114 |
except KeyboardInterrupt: |
---|
115 |
raise |
---|
116 |
except: |
---|
117 |
dest.close() |
---|
118 |
traceback.print_exc() |
---|
119 |
return False |
---|
120 |
else: |
---|
121 |
dest.close() |
---|
122 |
try: |
---|
123 |
os.remove(bz) |
---|
124 |
except KeyboardInterrupt: |
---|
125 |
raise |
---|
126 |
except: |
---|
127 |
traceback.print_exc() |
---|
128 |
return False |
---|
129 |
return True |
---|
130 |
|
---|
131 |
|
---|
132 |
def _process(f, date): |
---|
133 |
""" |
---|
134 |
Fetch and extract all the files for a month. |
---|
135 |
""" |
---|
136 |
|
---|
137 |
day_range = date['range'] |
---|
138 |
for day in day_range: |
---|
139 |
date.update({'day': day,}) |
---|
140 |
try: |
---|
141 |
print f.cwd(os.path.join(os.path.sep, |
---|
142 |
ftp_dir, |
---|
143 |
year_spec % date, |
---|
144 |
month_spec % date, |
---|
145 |
day_spec % date, |
---|
146 |
"").replace(os.path.sep, '/')) |
---|
147 |
except KeyboardInterrupt: |
---|
148 |
raise |
---|
149 |
except ftplib.error_perm, e: |
---|
150 |
traceback.print_exc() |
---|
151 |
continue |
---|
152 |
bz = _fetch(f) |
---|
153 |
if not bz: continue |
---|
154 |
if not _extract(bz): continue |
---|
155 |
return True |
---|
156 |
|
---|
157 |
def ascat(start_year, start_month, start_day, |
---|
158 |
end_year, end_month, end_day): |
---|
159 |
""" |
---|
160 |
Fetch and extract a range of files. |
---|
161 |
|
---|
162 |
ascat(start_year, start_month, start_day, |
---|
163 |
end_year, end_month, end_day ) -> None |
---|
164 |
|
---|
165 |
start_year is inclusive, |
---|
166 |
start_month is 1-12 inclusive, |
---|
167 |
start_day is 1-31 inclusive, |
---|
168 |
end_year is exclusive, |
---|
169 |
end_month is 2-13 exclusive, |
---|
170 |
end_day is 2-32 exclusive. |
---|
171 |
""" |
---|
172 |
|
---|
173 |
f = ftplib.FTP(ftp_server) |
---|
174 |
print f.login(ftp_user, ftp_pass) |
---|
175 |
year_range = range(start_year, end_year) |
---|
176 |
for year in year_range: |
---|
177 |
if year == year_range[0]: |
---|
178 |
_start_month = start_month |
---|
179 |
else: |
---|
180 |
_start_month = 1 |
---|
181 |
if year == year_range[-1]: |
---|
182 |
_end_month = end_month |
---|
183 |
else: |
---|
184 |
_end_month = 13 |
---|
185 |
month_range = range(_start_month, _end_month) |
---|
186 |
for month in month_range: |
---|
187 |
if year == year_range[0] and month == month_range[0]: |
---|
188 |
_start_day = start_day |
---|
189 |
else: |
---|
190 |
_start_day = 1 |
---|
191 |
if year == year_range[-1] and month == month_range[-1]: |
---|
192 |
_end_day = end_day |
---|
193 |
else: |
---|
194 |
_end_day = calendar.monthrange(year, month)[1] + 1 |
---|
195 |
day_range = range(_start_day, _end_day) |
---|
196 |
date = {'year': year, |
---|
197 |
'month': month, |
---|
198 |
'range': day_range, |
---|
199 |
} |
---|
200 |
try: |
---|
201 |
os.chdir(work_dir) |
---|
202 |
sub_dir = os.path.join(base_subdir, |
---|
203 |
year_spec % date, |
---|
204 |
month_spec % date) |
---|
205 |
os.makedirs(sub_dir) |
---|
206 |
os.chdir(sub_dir) |
---|
207 |
except OSError, e: |
---|
208 |
if not str(e).startswith("[Errno 17] File exists"): |
---|
209 |
traceback.print_exc() |
---|
210 |
continue |
---|
211 |
if not _process(f, date): continue |
---|
212 |
print f.quit() |
---|
213 |
|
---|
214 |
if __name__ == '__main__': |
---|
215 |
try: |
---|
216 |
if len(sys.argv) != 7: raise |
---|
217 |
start_year = int(sys.argv[1]) |
---|
218 |
start_month = int(sys.argv[2]) |
---|
219 |
start_day = int(sys.argv[3]) |
---|
220 |
end_year = int(sys.argv[4]) + 1 |
---|
221 |
end_month = int(sys.argv[5]) + 1 |
---|
222 |
end_day = int(sys.argv[6]) + 1 |
---|
223 |
except KeyboardInterrupt: |
---|
224 |
raise |
---|
225 |
except: |
---|
226 |
print __doc__ |
---|
227 |
sys.exit() |
---|
228 |
ascat(start_year, start_month, start_day, |
---|
229 |
end_year, end_month, end_day) |
---|