Index: codar/bin/archive_nccoos_codar.pl
===================================================================
---  (revision )
+++ codar/bin/archive_nccoos_codar.pl (revision 6)
@@ -1,0 +1,279 @@
+#!/afs/isis/pkg/isis/bin/perl
+# copy_codar2nemo.perl 
+# Time-stamp: <2006-07-06 16:13:13 haines>
+#
+#
+
+# Modified by Sara Haines, 06 Jul 2006
+# 
+# 1. removed scp and rsync stuff
+# 2. general code cleanup
+# 3. changed where data written to nemo from hws-AFS-space
+# 4. all code (MATLAB and perl scripts) located on nemo
+#
+# original code from Luke Stearns
+#
+
+#Let's start by setting up some basic path information:
+# $hws         = "/afs/isis.unc.edu/depts/marine/workspace/hseim/";
+# $seacoos_dir = "{hws}seacoos/data/codar";
+$nemo      = "/seacoos/data/";
+$codar_dir = "${nemo}nc-coos/hfradar/";
+
+# also need specific perl module libraries not in AFS space
+use lib '/afs/isis.unc.edu/depts/marine/workspace/hseim/seacoos/perl/site_perl/';
+
+# ftp library
+use Net::FTP;
+
+# date/time manipulation library
+use Date::Manip;
+
+# I found a neat set manipulation library (to find the difference 
+# between two sets (which files do we need to grab?)
+
+use Set::Scalar;
+
+#Get yesterday's date in yymmdd format
+$date=&UnixDate("today", "%y%m%d");
+
+######################################################################
+# Codar ftp get data #################################################
+######################################################################
+
+# I just took this from a script Sara wrote for www.seacoos.org/wap/
+# Author: Sara Haines (2003-02-22) 
+#
+
+# if debugging requested print messages to STDOUT
+if (grep /[debug|DEBUG|d]/, @ARGV) {
+  $debug = 1;
+}
+
+# print Date and Time and version of perl + patchlevel /1000
+$now=&UnixDate(&ParseDate("today"), "%Y.%m.%d %H:%M:%S");
+if ($debug) { print "\n==== Starting: $now ==== Perl Version: $]\n"; }
+
+# get current directory
+$cwd = `pwd`;
+
+# start stop-watch for cummulative times
+$starttime = time;
+
+######################################################################
+# Grab from codar.marine.unc.edu
+
+# '/Codar/SeaSonde/Data/';
+
+# for each station in list, get file from ftp and place in local data drive
+%dir_info = (
+	     "/Codar/SeaSonde/Data/RadialSites/Site_2/", "${codar_dir}/codar_radials/duck/",
+	     "/Codar/SeaSonde/Data/RadialSites/Site_1/", "${codar_dir}/codar_radials/haty/",
+	     "/Codar/SeaSonde/Data/Totals/", "${codar_dir}/codar_totals/ouba/",
+	     #"/Codar/SeaSonde/Data/Totals/TOTL*","tuv_files/",
+	     #"Pictures/", "img_files/",
+	     );
+
+# login to FTP site
+if ($debug) { print "FTP files from codar.marine.unc.edu to nemo ...\n"; }
+
+&ftp_grab_data("codar.marine.unc.edu","codar",'br@ggsc@tt3r',%dir_info) ||
+    print "Problems connecting to codar.marine.unc.edu!\n";
+
+######################################################################
+# Grab from gulf.marine.rutgers.edu
+
+if (0) {
+    # for each station in list, get file from ftp and place in local data drive
+    %dir_info = (
+		 '../Radial Files/Site 2/', 'radial_files/wild/',
+		 );
+
+    # login to FTP site
+    if ($debug) { print "Ftp files from gulf.marine.rutgers.edu to AFS ...\n"; }
+
+    $start_rutgers_time = time;
+
+    &ftp_grab_data("gulf.marine.rutgers.edu","codar","tne",%dir_info) ||
+	print "Problems connecting to gulf.marine.rutgers.edu!\n";
+}
+
+######################################################################
+# Done with all grabs ################################################
+######################################################################
+# stop stop-watch
+$cummtime = time - $starttime;
+if ($debug) { print "Total data grab time = $cummtime (seconds).\n"; }
+
+
+######################################################################
+# Subroutines ########################################################
+######################################################################
+
+#This one is also hacked from Sara's script 
+sub read_bypackage_info {
+
+  open(BYPACKINFO, "<$packageinfo");
+  $i=0;
+  while ($entry = <BYPACKINFO>) {
+    chop($entry);
+    ($name[$i],@long_name) = split(/\s+/,$entry);  
+    $long_name[$i]=join(@long_name,/\s/);
+    $i++;
+  }
+  close(BYPACKINFO);
+  
+} # sub read_bypackage_info
+
+######################################################################
+### FTP Grab Data ####################################################
+sub ftp_grab_data {
+
+    ($rem_site,$user,$pass,%dir_info)=@_;
+    $start_time = time;
+
+    #print "Host: $rem_site\nUser: $user\n$pass!\n";
+    #$ftp ='';
+    #while (time-$start_time<60) {
+    $ftp = Net::FTP->new("$rem_site");
+    #}
+    $ftp->login("$user","$pass");
+    #$ftp->binary();
+    foreach $remote_dir (keys(%dir_info)) {
+	
+	$local_dir=$dir_info{$remote_dir};
+	if ($debug) { print "$local_dir:\n"; }
+	print "$remote_dir\n";
+
+	@loc_list=();
+	@we_got_these=();
+	$bad=0; $good=0;
+	open(LOCALIST,"$local_dir/ftplog");
+	while (<LOCALIST>) {
+	    chop;
+	    #Ignore empty lines
+	    if($_) {
+		push(@loc_list,"$_");
+	    }
+	}
+	close(LOCALIST);
+	
+	@rem_files=();
+	@rem_files=$ftp->ls("${remote_dir}");
+	#$rem_list =~s/join(',',"@rem_list")/$remote_dir/g;
+	$rem_list = join(',',@rem_files);
+	$rem_list =~s/$remote_dir//g;
+
+	#@rem_list = ();
+	#foreach $rem_file (@rem_files) {
+	#    @path = split('/',$rem_file);
+	#    my $tmp_var = $rem_file;
+	#    $tmp_var =~s/$path[-1]$//;
+	#    $remote_dir = $tmp_var;
+	#    $rem_file = $path[-1];
+	#    push(@rem_list,"$rem_file");
+	#}
+	#print "$rem_list\n";
+	@rem_list = split(',',$rem_list);
+	
+	$set_loc=Set::Scalar->new(@loc_list);
+	$set_rem=Set::Scalar->new(@rem_list);
+	
+	$get_these=$set_rem->difference($set_loc);
+	$get_these=~s/[()]//g;
+
+	@get_these=split('\s+',$get_these);
+	if ($debug) {print "L:$#loc_list R:$#rem_list G:$#get_these\n";}
+	print "$rem_list[0]\n";
+	$got_these = 0;
+  
+	foreach $rem_file (@get_these) {
+	    $_=$rem_file;
+	    $f_date=$rem_file;
+	    #print "$f_date\n";
+	    #This matching condition may be a bit confusing, so here goes:
+	    # any three characters (Rad or Tot) keep track of the fourth (s,z,_) then four more
+	    # characters, an underscore, keep track of yy then mm, ignore the next three but make 
+	    # sure they are followed by an underscore, then whatever:
+	    # Rad(s)DUCK_(03)_(11)_11_2200  the things in the parens, I'm keeping
+	    # Tot(_)OUBA_(03)_(10)_30_1600.jpg   got it?
+	    $f_date=~m/^.{3}(.).{4}_(..).(..).{3}_*/;
+	    $ideal_measured_total = $1; #This string tells me which one it is
+	    $yy=$2;
+	    $mm=$3;
+
+	    unless ($ideal_measured_total) {
+		#If that didn't work because the string is slightly different...
+		#RDLm_DUCK_2005_12_07_1000.ruv
+		$f_date=~m/^.{3}(.)_.{4}_20(..).(..).{3}_*/;
+		$ideal_measured_total = $1; #This string tells me which one it is
+		$yy=$2;
+		$mm=$3;
+	    }
+    
+	    if ($yy>0 & $yy<10) {
+		$month_folder = "20${yy}_$mm";
+	    }
+	    elsif ($yy>2000) {
+		$month_folder = "${yy}_$mm";
+	    }
+	    else {
+		#print "Warning $rem_file doesn't have a year that we can see!\n";
+		#Not right, skip it!
+		$month_folder="";
+	    }
+
+	    #Handle the month folder extensions: 
+	    # 2003_11_i ideal radials
+	    # 2003_11_m measured radials
+	    # 2003_11   totals or images...
+	    if ($ideal_measured_total=~/s/) {
+		#If there is an s, it is radials from an ideal beam pattern
+		$month_folder = $month_folder."_i";
+	    }
+	    elsif ($ideal_measured_total=~/z/) {
+		#If there is an s, it is radials from an ideal beam pattern
+		$month_folder = $month_folder."_m";
+	    }
+	    elsif ($ideal_measured_total=~/m/) {
+		#If there is an s, it is radials from an ideal beam pattern
+		$month_folder = $month_folder."_uvm";
+	    }
+	    elsif ($ideal_measured_total=~/i/) {
+		#If there is an s, it is radials from an ideal beam pattern
+		$month_folder = $month_folder."_uvi";
+	    }
+	    else {
+		#don't do anything to the month folder name!!
+		print "$ideal_measured_total -> $rem_file\n";
+	    }
+	    
+	    #if ($debug) {print "$month_folder\n";}
+	    unless (/_40_/ || /File/ || /old/ || !$month_folder) {
+		if ($debug) {print "$rem_file -> $month_folder\n";}
+		mkdir ("$local_dir/$month_folder");  
+		$ftp->get("${remote_dir}$rem_file",
+			  "$local_dir/$month_folder/$rem_file");
+		if ($rem_file) {
+		    push(@we_got_these,"$rem_file");
+		    $got_these++;
+		}
+	    }
+	}
+	
+	$we_got_these = join("\n",@we_got_these);
+	open(LOCALIST,">>$local_dir/ftplog");
+	print LOCALIST "$we_got_these\n";
+	close(LOCALIST);
+	
+	if ($debug) { print "Got $got_these files.\n\n";}
+    }
+    
+    ## logoff FTP site
+    $ftp->quit;
+
+    #system("ftp codar.marine.unc.edu");
+    #$img_dir="$codar_dir/img_files/";
+
+}
+