NCCOOS Trac Projects: Top | Web | Platforms | Processing | Viz | Sprints | Sandbox | (Wind)

root/codar/trunk/bin/archive_nccoos_codar.pl

Revision 6 (checked in by cbc, 18 years ago)

--

  • Property svn:executable set to
Line 
1 #!/afs/isis/pkg/isis/bin/perl
2 # copy_codar2nemo.perl
3 # Time-stamp: <2006-07-06 16:13:13 haines>
4 #
5 #
6
7 # Modified by Sara Haines, 06 Jul 2006
8 #
9 # 1. removed scp and rsync stuff
10 # 2. general code cleanup
11 # 3. changed where data written to nemo from hws-AFS-space
12 # 4. all code (MATLAB and perl scripts) located on nemo
13 #
14 # original code from Luke Stearns
15 #
16
17 #Let's start by setting up some basic path information:
18 # $hws         = "/afs/isis.unc.edu/depts/marine/workspace/hseim/";
19 # $seacoos_dir = "{hws}seacoos/data/codar";
20 $nemo      = "/seacoos/data/";
21 $codar_dir = "${nemo}nc-coos/hfradar/";
22
23 # also need specific perl module libraries not in AFS space
24 use lib '/afs/isis.unc.edu/depts/marine/workspace/hseim/seacoos/perl/site_perl/';
25
26 # ftp library
27 use Net::FTP;
28
29 # date/time manipulation library
30 use Date::Manip;
31
32 # I found a neat set manipulation library (to find the difference
33 # between two sets (which files do we need to grab?)
34
35 use Set::Scalar;
36
37 #Get yesterday's date in yymmdd format
38 $date=&UnixDate("today", "%y%m%d");
39
40 ######################################################################
41 # Codar ftp get data #################################################
42 ######################################################################
43
44 # I just took this from a script Sara wrote for www.seacoos.org/wap/
45 # Author: Sara Haines (2003-02-22)
46 #
47
48 # if debugging requested print messages to STDOUT
49 if (grep /[debug|DEBUG|d]/, @ARGV) {
50   $debug = 1;
51 }
52
53 # print Date and Time and version of perl + patchlevel /1000
54 $now=&UnixDate(&ParseDate("today"), "%Y.%m.%d %H:%M:%S");
55 if ($debug) { print "\n==== Starting: $now ==== Perl Version: $]\n"; }
56
57 # get current directory
58 $cwd = `pwd`;
59
60 # start stop-watch for cummulative times
61 $starttime = time;
62
63 ######################################################################
64 # Grab from codar.marine.unc.edu
65
66 # '/Codar/SeaSonde/Data/';
67
68 # for each station in list, get file from ftp and place in local data drive
69 %dir_info = (
70              "/Codar/SeaSonde/Data/RadialSites/Site_2/", "${codar_dir}/codar_radials/duck/",
71              "/Codar/SeaSonde/Data/RadialSites/Site_1/", "${codar_dir}/codar_radials/haty/",
72              "/Codar/SeaSonde/Data/Totals/", "${codar_dir}/codar_totals/ouba/",
73              #"/Codar/SeaSonde/Data/Totals/TOTL*","tuv_files/",
74              #"Pictures/", "img_files/",
75              );
76
77 # login to FTP site
78 if ($debug) { print "FTP files from codar.marine.unc.edu to nemo ...\n"; }
79
80 &ftp_grab_data("codar.marine.unc.edu","codar",'br@ggsc@tt3r',%dir_info) ||
81     print "Problems connecting to codar.marine.unc.edu!\n";
82
83 ######################################################################
84 # Grab from gulf.marine.rutgers.edu
85
86 if (0) {
87     # for each station in list, get file from ftp and place in local data drive
88     %dir_info = (
89                  '../Radial Files/Site 2/', 'radial_files/wild/',
90                  );
91
92     # login to FTP site
93     if ($debug) { print "Ftp files from gulf.marine.rutgers.edu to AFS ...\n"; }
94
95     $start_rutgers_time = time;
96
97     &ftp_grab_data("gulf.marine.rutgers.edu","codar","tne",%dir_info) ||
98         print "Problems connecting to gulf.marine.rutgers.edu!\n";
99 }
100
101 ######################################################################
102 # Done with all grabs ################################################
103 ######################################################################
104 # stop stop-watch
105 $cummtime = time - $starttime;
106 if ($debug) { print "Total data grab time = $cummtime (seconds).\n"; }
107
108
109 ######################################################################
110 # Subroutines ########################################################
111 ######################################################################
112
113 #This one is also hacked from Sara's script
114 sub read_bypackage_info {
115
116   open(BYPACKINFO, "<$packageinfo");
117   $i=0;
118   while ($entry = <BYPACKINFO>) {
119     chop($entry);
120     ($name[$i],@long_name) = split(/\s+/,$entry); 
121     $long_name[$i]=join(@long_name,/\s/);
122     $i++;
123   }
124   close(BYPACKINFO);
125  
126 } # sub read_bypackage_info
127
128 ######################################################################
129 ### FTP Grab Data ####################################################
130 sub ftp_grab_data {
131
132     ($rem_site,$user,$pass,%dir_info)=@_;
133     $start_time = time;
134
135     #print "Host: $rem_site\nUser: $user\n$pass!\n";
136     #$ftp ='';
137     #while (time-$start_time<60) {
138     $ftp = Net::FTP->new("$rem_site");
139     #}
140     $ftp->login("$user","$pass");
141     #$ftp->binary();
142     foreach $remote_dir (keys(%dir_info)) {
143        
144         $local_dir=$dir_info{$remote_dir};
145         if ($debug) { print "$local_dir:\n"; }
146         print "$remote_dir\n";
147
148         @loc_list=();
149         @we_got_these=();
150         $bad=0; $good=0;
151         open(LOCALIST,"$local_dir/ftplog");
152         while (<LOCALIST>) {
153             chop;
154             #Ignore empty lines
155             if($_) {
156                 push(@loc_list,"$_");
157             }
158         }
159         close(LOCALIST);
160        
161         @rem_files=();
162         @rem_files=$ftp->ls("${remote_dir}");
163         #$rem_list =~s/join(',',"@rem_list")/$remote_dir/g;
164         $rem_list = join(',',@rem_files);
165         $rem_list =~s/$remote_dir//g;
166
167         #@rem_list = ();
168         #foreach $rem_file (@rem_files) {
169         #    @path = split('/',$rem_file);
170         #    my $tmp_var = $rem_file;
171         #    $tmp_var =~s/$path[-1]$//;
172         #    $remote_dir = $tmp_var;
173         #    $rem_file = $path[-1];
174         #    push(@rem_list,"$rem_file");
175         #}
176         #print "$rem_list\n";
177         @rem_list = split(',',$rem_list);
178        
179         $set_loc=Set::Scalar->new(@loc_list);
180         $set_rem=Set::Scalar->new(@rem_list);
181        
182         $get_these=$set_rem->difference($set_loc);
183         $get_these=~s/[()]//g;
184
185         @get_these=split('\s+',$get_these);
186         if ($debug) {print "L:$#loc_list R:$#rem_list G:$#get_these\n";}
187         print "$rem_list[0]\n";
188         $got_these = 0;
189  
190         foreach $rem_file (@get_these) {
191             $_=$rem_file;
192             $f_date=$rem_file;
193             #print "$f_date\n";
194             #This matching condition may be a bit confusing, so here goes:
195             # any three characters (Rad or Tot) keep track of the fourth (s,z,_) then four more
196             # characters, an underscore, keep track of yy then mm, ignore the next three but make
197             # sure they are followed by an underscore, then whatever:
198             # Rad(s)DUCK_(03)_(11)_11_2200  the things in the parens, I'm keeping
199             # Tot(_)OUBA_(03)_(10)_30_1600.jpg   got it?
200             $f_date=~m/^.{3}(.).{4}_(..).(..).{3}_*/;
201             $ideal_measured_total = $1; #This string tells me which one it is
202             $yy=$2;
203             $mm=$3;
204
205             unless ($ideal_measured_total) {
206                 #If that didn't work because the string is slightly different...
207                 #RDLm_DUCK_2005_12_07_1000.ruv
208                 $f_date=~m/^.{3}(.)_.{4}_20(..).(..).{3}_*/;
209                 $ideal_measured_total = $1; #This string tells me which one it is
210                 $yy=$2;
211                 $mm=$3;
212             }
213    
214             if ($yy>0 & $yy<10) {
215                 $month_folder = "20${yy}_$mm";
216             }
217             elsif ($yy>2000) {
218                 $month_folder = "${yy}_$mm";
219             }
220             else {
221                 #print "Warning $rem_file doesn't have a year that we can see!\n";
222                 #Not right, skip it!
223                 $month_folder="";
224             }
225
226             #Handle the month folder extensions:
227             # 2003_11_i ideal radials
228             # 2003_11_m measured radials
229             # 2003_11   totals or images...
230             if ($ideal_measured_total=~/s/) {
231                 #If there is an s, it is radials from an ideal beam pattern
232                 $month_folder = $month_folder."_i";
233             }
234             elsif ($ideal_measured_total=~/z/) {
235                 #If there is an s, it is radials from an ideal beam pattern
236                 $month_folder = $month_folder."_m";
237             }
238             elsif ($ideal_measured_total=~/m/) {
239                 #If there is an s, it is radials from an ideal beam pattern
240                 $month_folder = $month_folder."_uvm";
241             }
242             elsif ($ideal_measured_total=~/i/) {
243                 #If there is an s, it is radials from an ideal beam pattern
244                 $month_folder = $month_folder."_uvi";
245             }
246             else {
247                 #don't do anything to the month folder name!!
248                 print "$ideal_measured_total -> $rem_file\n";
249             }
250            
251             #if ($debug) {print "$month_folder\n";}
252             unless (/_40_/ || /File/ || /old/ || !$month_folder) {
253                 if ($debug) {print "$rem_file -> $month_folder\n";}
254                 mkdir ("$local_dir/$month_folder"); 
255                 $ftp->get("${remote_dir}$rem_file",
256                           "$local_dir/$month_folder/$rem_file");
257                 if ($rem_file) {
258                     push(@we_got_these,"$rem_file");
259                     $got_these++;
260                 }
261             }
262         }
263        
264         $we_got_these = join("\n",@we_got_these);
265         open(LOCALIST,">>$local_dir/ftplog");
266         print LOCALIST "$we_got_these\n";
267         close(LOCALIST);
268        
269         if ($debug) { print "Got $got_these files.\n\n";}
270     }
271    
272     ## logoff FTP site
273     $ftp->quit;
274
275     #system("ftp codar.marine.unc.edu");
276     #$img_dir="$codar_dir/img_files/";
277
278 }
279
Note: See TracBrowser for help on using the browser.