root/codar/trunk/bin/archive_nccoos_codar.pl
Revision 6 (checked in by cbc, 18 years ago) | |
---|---|
|
Line | |
---|---|
1 | #!/afs/isis/pkg/isis/bin/perl |
2 | # copy_codar2nemo.perl |
3 | # Time-stamp: <2006-07-06 16:13:13 haines> |
4 | # |
5 | # |
6 | |
7 | # Modified by Sara Haines, 06 Jul 2006 |
8 | # |
9 | # 1. removed scp and rsync stuff |
10 | # 2. general code cleanup |
11 | # 3. changed where data written to nemo from hws-AFS-space |
12 | # 4. all code (MATLAB and perl scripts) located on nemo |
13 | # |
14 | # original code from Luke Stearns |
15 | # |
16 | |
17 | #Let's start by setting up some basic path information: |
18 | # $hws = "/afs/isis.unc.edu/depts/marine/workspace/hseim/"; |
19 | # $seacoos_dir = "{hws}seacoos/data/codar"; |
20 | $nemo = "/seacoos/data/"; |
21 | $codar_dir = "${nemo}nc-coos/hfradar/"; |
22 | |
23 | # also need specific perl module libraries not in AFS space |
24 | use lib '/afs/isis.unc.edu/depts/marine/workspace/hseim/seacoos/perl/site_perl/'; |
25 | |
26 | # ftp library |
27 | use Net::FTP; |
28 | |
29 | # date/time manipulation library |
30 | use Date::Manip; |
31 | |
32 | # I found a neat set manipulation library (to find the difference |
33 | # between two sets (which files do we need to grab?) |
34 | |
35 | use Set::Scalar; |
36 | |
37 | #Get yesterday's date in yymmdd format |
38 | $date=&UnixDate("today", "%y%m%d"); |
39 | |
40 | ###################################################################### |
41 | # Codar ftp get data ################################################# |
42 | ###################################################################### |
43 | |
44 | # I just took this from a script Sara wrote for www.seacoos.org/wap/ |
45 | # Author: Sara Haines (2003-02-22) |
46 | # |
47 | |
48 | # if debugging requested print messages to STDOUT |
49 | if (grep /[debug|DEBUG|d]/, @ARGV) { |
50 | $debug = 1; |
51 | } |
52 | |
53 | # print Date and Time and version of perl + patchlevel /1000 |
54 | $now=&UnixDate(&ParseDate("today"), "%Y.%m.%d %H:%M:%S"); |
55 | if ($debug) { print "\n==== Starting: $now ==== Perl Version: $]\n"; } |
56 | |
57 | # get current directory |
58 | $cwd = `pwd`; |
59 | |
60 | # start stop-watch for cummulative times |
61 | $starttime = time; |
62 | |
63 | ###################################################################### |
64 | # Grab from codar.marine.unc.edu |
65 | |
66 | # '/Codar/SeaSonde/Data/'; |
67 | |
68 | # for each station in list, get file from ftp and place in local data drive |
69 | %dir_info = ( |
70 | "/Codar/SeaSonde/Data/RadialSites/Site_2/", "${codar_dir}/codar_radials/duck/", |
71 | "/Codar/SeaSonde/Data/RadialSites/Site_1/", "${codar_dir}/codar_radials/haty/", |
72 | "/Codar/SeaSonde/Data/Totals/", "${codar_dir}/codar_totals/ouba/", |
73 | #"/Codar/SeaSonde/Data/Totals/TOTL*","tuv_files/", |
74 | #"Pictures/", "img_files/", |
75 | ); |
76 | |
77 | # login to FTP site |
78 | if ($debug) { print "FTP files from codar.marine.unc.edu to nemo ...\n"; } |
79 | |
80 | &ftp_grab_data("codar.marine.unc.edu","codar",'br@ggsc@tt3r',%dir_info) || |
81 | print "Problems connecting to codar.marine.unc.edu!\n"; |
82 | |
83 | ###################################################################### |
84 | # Grab from gulf.marine.rutgers.edu |
85 | |
86 | if (0) { |
87 | # for each station in list, get file from ftp and place in local data drive |
88 | %dir_info = ( |
89 | '../Radial Files/Site 2/', 'radial_files/wild/', |
90 | ); |
91 | |
92 | # login to FTP site |
93 | if ($debug) { print "Ftp files from gulf.marine.rutgers.edu to AFS ...\n"; } |
94 | |
95 | $start_rutgers_time = time; |
96 | |
97 | &ftp_grab_data("gulf.marine.rutgers.edu","codar","tne",%dir_info) || |
98 | print "Problems connecting to gulf.marine.rutgers.edu!\n"; |
99 | } |
100 | |
101 | ###################################################################### |
102 | # Done with all grabs ################################################ |
103 | ###################################################################### |
104 | # stop stop-watch |
105 | $cummtime = time - $starttime; |
106 | if ($debug) { print "Total data grab time = $cummtime (seconds).\n"; } |
107 | |
108 | |
109 | ###################################################################### |
110 | # Subroutines ######################################################## |
111 | ###################################################################### |
112 | |
113 | #This one is also hacked from Sara's script |
114 | sub read_bypackage_info { |
115 | |
116 | open(BYPACKINFO, "<$packageinfo"); |
117 | $i=0; |
118 | while ($entry = <BYPACKINFO>) { |
119 | chop($entry); |
120 | ($name[$i],@long_name) = split(/\s+/,$entry); |
121 | $long_name[$i]=join(@long_name,/\s/); |
122 | $i++; |
123 | } |
124 | close(BYPACKINFO); |
125 | |
126 | } # sub read_bypackage_info |
127 | |
128 | ###################################################################### |
129 | ### FTP Grab Data #################################################### |
130 | sub ftp_grab_data { |
131 | |
132 | ($rem_site,$user,$pass,%dir_info)=@_; |
133 | $start_time = time; |
134 | |
135 | #print "Host: $rem_site\nUser: $user\n$pass!\n"; |
136 | #$ftp =''; |
137 | #while (time-$start_time<60) { |
138 | $ftp = Net::FTP->new("$rem_site"); |
139 | #} |
140 | $ftp->login("$user","$pass"); |
141 | #$ftp->binary(); |
142 | foreach $remote_dir (keys(%dir_info)) { |
143 | |
144 | $local_dir=$dir_info{$remote_dir}; |
145 | if ($debug) { print "$local_dir:\n"; } |
146 | print "$remote_dir\n"; |
147 | |
148 | @loc_list=(); |
149 | @we_got_these=(); |
150 | $bad=0; $good=0; |
151 | open(LOCALIST,"$local_dir/ftplog"); |
152 | while (<LOCALIST>) { |
153 | chop; |
154 | #Ignore empty lines |
155 | if($_) { |
156 | push(@loc_list,"$_"); |
157 | } |
158 | } |
159 | close(LOCALIST); |
160 | |
161 | @rem_files=(); |
162 | @rem_files=$ftp->ls("${remote_dir}"); |
163 | #$rem_list =~s/join(',',"@rem_list")/$remote_dir/g; |
164 | $rem_list = join(',',@rem_files); |
165 | $rem_list =~s/$remote_dir//g; |
166 | |
167 | #@rem_list = (); |
168 | #foreach $rem_file (@rem_files) { |
169 | # @path = split('/',$rem_file); |
170 | # my $tmp_var = $rem_file; |
171 | # $tmp_var =~s/$path[-1]$//; |
172 | # $remote_dir = $tmp_var; |
173 | # $rem_file = $path[-1]; |
174 | # push(@rem_list,"$rem_file"); |
175 | #} |
176 | #print "$rem_list\n"; |
177 | @rem_list = split(',',$rem_list); |
178 | |
179 | $set_loc=Set::Scalar->new(@loc_list); |
180 | $set_rem=Set::Scalar->new(@rem_list); |
181 | |
182 | $get_these=$set_rem->difference($set_loc); |
183 | $get_these=~s/[()]//g; |
184 | |
185 | @get_these=split('\s+',$get_these); |
186 | if ($debug) {print "L:$#loc_list R:$#rem_list G:$#get_these\n";} |
187 | print "$rem_list[0]\n"; |
188 | $got_these = 0; |
189 | |
190 | foreach $rem_file (@get_these) { |
191 | $_=$rem_file; |
192 | $f_date=$rem_file; |
193 | #print "$f_date\n"; |
194 | #This matching condition may be a bit confusing, so here goes: |
195 | # any three characters (Rad or Tot) keep track of the fourth (s,z,_) then four more |
196 | # characters, an underscore, keep track of yy then mm, ignore the next three but make |
197 | # sure they are followed by an underscore, then whatever: |
198 | # Rad(s)DUCK_(03)_(11)_11_2200 the things in the parens, I'm keeping |
199 | # Tot(_)OUBA_(03)_(10)_30_1600.jpg got it? |
200 | $f_date=~m/^.{3}(.).{4}_(..).(..).{3}_*/; |
201 | $ideal_measured_total = $1; #This string tells me which one it is |
202 | $yy=$2; |
203 | $mm=$3; |
204 | |
205 | unless ($ideal_measured_total) { |
206 | #If that didn't work because the string is slightly different... |
207 | #RDLm_DUCK_2005_12_07_1000.ruv |
208 | $f_date=~m/^.{3}(.)_.{4}_20(..).(..).{3}_*/; |
209 | $ideal_measured_total = $1; #This string tells me which one it is |
210 | $yy=$2; |
211 | $mm=$3; |
212 | } |
213 | |
214 | if ($yy>0 & $yy<10) { |
215 | $month_folder = "20${yy}_$mm"; |
216 | } |
217 | elsif ($yy>2000) { |
218 | $month_folder = "${yy}_$mm"; |
219 | } |
220 | else { |
221 | #print "Warning $rem_file doesn't have a year that we can see!\n"; |
222 | #Not right, skip it! |
223 | $month_folder=""; |
224 | } |
225 | |
226 | #Handle the month folder extensions: |
227 | # 2003_11_i ideal radials |
228 | # 2003_11_m measured radials |
229 | # 2003_11 totals or images... |
230 | if ($ideal_measured_total=~/s/) { |
231 | #If there is an s, it is radials from an ideal beam pattern |
232 | $month_folder = $month_folder."_i"; |
233 | } |
234 | elsif ($ideal_measured_total=~/z/) { |
235 | #If there is an s, it is radials from an ideal beam pattern |
236 | $month_folder = $month_folder."_m"; |
237 | } |
238 | elsif ($ideal_measured_total=~/m/) { |
239 | #If there is an s, it is radials from an ideal beam pattern |
240 | $month_folder = $month_folder."_uvm"; |
241 | } |
242 | elsif ($ideal_measured_total=~/i/) { |
243 | #If there is an s, it is radials from an ideal beam pattern |
244 | $month_folder = $month_folder."_uvi"; |
245 | } |
246 | else { |
247 | #don't do anything to the month folder name!! |
248 | print "$ideal_measured_total -> $rem_file\n"; |
249 | } |
250 | |
251 | #if ($debug) {print "$month_folder\n";} |
252 | unless (/_40_/ || /File/ || /old/ || !$month_folder) { |
253 | if ($debug) {print "$rem_file -> $month_folder\n";} |
254 | mkdir ("$local_dir/$month_folder"); |
255 | $ftp->get("${remote_dir}$rem_file", |
256 | "$local_dir/$month_folder/$rem_file"); |
257 | if ($rem_file) { |
258 | push(@we_got_these,"$rem_file"); |
259 | $got_these++; |
260 | } |
261 | } |
262 | } |
263 | |
264 | $we_got_these = join("\n",@we_got_these); |
265 | open(LOCALIST,">>$local_dir/ftplog"); |
266 | print LOCALIST "$we_got_these\n"; |
267 | close(LOCALIST); |
268 | |
269 | if ($debug) { print "Got $got_these files.\n\n";} |
270 | } |
271 | |
272 | ## logoff FTP site |
273 | $ftp->quit; |
274 | |
275 | #system("ftp codar.marine.unc.edu"); |
276 | #$img_dir="$codar_dir/img_files/"; |
277 | |
278 | } |
279 |
Note: See TracBrowser for help on using the browser.