#!/afs/isis/pkg/isis/bin/perl
# copy_codar2nemo.perl 
# Time-stamp: <2006-07-06 16:13:13 haines>
#
#

# Modified by Sara Haines, 06 Jul 2006
# 
# 1. removed scp and rsync stuff
# 2. general code cleanup
# 3. changed where data written to nemo from hws-AFS-space
# 4. all code (MATLAB and perl scripts) located on nemo
#
# original code from Luke Stearns
#

#Let's start by setting up some basic path information:
# $hws         = "/afs/isis.unc.edu/depts/marine/workspace/hseim/";
# $seacoos_dir = "{hws}seacoos/data/codar";
$nemo      = "/seacoos/data/";
$codar_dir = "${nemo}nc-coos/hfradar/";

# also need specific perl module libraries not in AFS space
use lib '/afs/isis.unc.edu/depts/marine/workspace/hseim/seacoos/perl/site_perl/';

# ftp library
use Net::FTP;

# date/time manipulation library
use Date::Manip;

# I found a neat set manipulation library (to find the difference 
# between two sets (which files do we need to grab?)

use Set::Scalar;

#Get yesterday's date in yymmdd format
$date=&UnixDate("today", "%y%m%d");

######################################################################
# Codar ftp get data #################################################
######################################################################

# I just took this from a script Sara wrote for www.seacoos.org/wap/
# Author: Sara Haines (2003-02-22) 
#

# if debugging requested print messages to STDOUT
if (grep /[debug|DEBUG|d]/, @ARGV) {
  $debug = 1;
}

# print Date and Time and version of perl + patchlevel /1000
$now=&UnixDate(&ParseDate("today"), "%Y.%m.%d %H:%M:%S");
if ($debug) { print "\n==== Starting: $now ==== Perl Version: $]\n"; }

# get current directory
$cwd = `pwd`;

# start stop-watch for cummulative times
$starttime = time;

######################################################################
# Grab from codar.marine.unc.edu

# '/Codar/SeaSonde/Data/';

# for each station in list, get file from ftp and place in local data drive
%dir_info = (
	     "/Codar/SeaSonde/Data/RadialSites/Site_2/", "${codar_dir}/codar_radials/duck/",
	     "/Codar/SeaSonde/Data/RadialSites/Site_1/", "${codar_dir}/codar_radials/haty/",
	     "/Codar/SeaSonde/Data/Totals/", "${codar_dir}/codar_totals/ouba/",
	     #"/Codar/SeaSonde/Data/Totals/TOTL*","tuv_files/",
	     #"Pictures/", "img_files/",
	     );

# login to FTP site
if ($debug) { print "FTP files from codar.marine.unc.edu to nemo ...\n"; }

&ftp_grab_data("codar.marine.unc.edu","codar",'br@ggsc@tt3r',%dir_info) ||
    print "Problems connecting to codar.marine.unc.edu!\n";

######################################################################
# Grab from gulf.marine.rutgers.edu

if (0) {
    # for each station in list, get file from ftp and place in local data drive
    %dir_info = (
		 '../Radial Files/Site 2/', 'radial_files/wild/',
		 );

    # login to FTP site
    if ($debug) { print "Ftp files from gulf.marine.rutgers.edu to AFS ...\n"; }

    $start_rutgers_time = time;

    &ftp_grab_data("gulf.marine.rutgers.edu","codar","tne",%dir_info) ||
	print "Problems connecting to gulf.marine.rutgers.edu!\n";
}

######################################################################
# Done with all grabs ################################################
######################################################################
# stop stop-watch
$cummtime = time - $starttime;
if ($debug) { print "Total data grab time = $cummtime (seconds).\n"; }


######################################################################
# Subroutines ########################################################
######################################################################

#This one is also hacked from Sara's script 
sub read_bypackage_info {

  open(BYPACKINFO, "<$packageinfo");
  $i=0;
  while ($entry = <BYPACKINFO>) {
    chop($entry);
    ($name[$i],@long_name) = split(/\s+/,$entry);  
    $long_name[$i]=join(@long_name,/\s/);
    $i++;
  }
  close(BYPACKINFO);
  
} # sub read_bypackage_info

######################################################################
### FTP Grab Data ####################################################
sub ftp_grab_data {

    ($rem_site,$user,$pass,%dir_info)=@_;
    $start_time = time;

    #print "Host: $rem_site\nUser: $user\n$pass!\n";
    #$ftp ='';
    #while (time-$start_time<60) {
    $ftp = Net::FTP->new("$rem_site");
    #}
    $ftp->login("$user","$pass");
    #$ftp->binary();
    foreach $remote_dir (keys(%dir_info)) {
	
	$local_dir=$dir_info{$remote_dir};
	if ($debug) { print "$local_dir:\n"; }
	print "$remote_dir\n";

	@loc_list=();
	@we_got_these=();
	$bad=0; $good=0;
	open(LOCALIST,"$local_dir/ftplog");
	while (<LOCALIST>) {
	    chop;
	    #Ignore empty lines
	    if($_) {
		push(@loc_list,"$_");
	    }
	}
	close(LOCALIST);
	
	@rem_files=();
	@rem_files=$ftp->ls("${remote_dir}");
	#$rem_list =~s/join(',',"@rem_list")/$remote_dir/g;
	$rem_list = join(',',@rem_files);
	$rem_list =~s/$remote_dir//g;

	#@rem_list = ();
	#foreach $rem_file (@rem_files) {
	#    @path = split('/',$rem_file);
	#    my $tmp_var = $rem_file;
	#    $tmp_var =~s/$path[-1]$//;
	#    $remote_dir = $tmp_var;
	#    $rem_file = $path[-1];
	#    push(@rem_list,"$rem_file");
	#}
	#print "$rem_list\n";
	@rem_list = split(',',$rem_list);
	
	$set_loc=Set::Scalar->new(@loc_list);
	$set_rem=Set::Scalar->new(@rem_list);
	
	$get_these=$set_rem->difference($set_loc);
	$get_these=~s/[()]//g;

	@get_these=split('\s+',$get_these);
	if ($debug) {print "L:$#loc_list R:$#rem_list G:$#get_these\n";}
	print "$rem_list[0]\n";
	$got_these = 0;
  
	foreach $rem_file (@get_these) {
	    $_=$rem_file;
	    $f_date=$rem_file;
	    #print "$f_date\n";
	    #This matching condition may be a bit confusing, so here goes:
	    # any three characters (Rad or Tot) keep track of the fourth (s,z,_) then four more
	    # characters, an underscore, keep track of yy then mm, ignore the next three but make 
	    # sure they are followed by an underscore, then whatever:
	    # Rad(s)DUCK_(03)_(11)_11_2200  the things in the parens, I'm keeping
	    # Tot(_)OUBA_(03)_(10)_30_1600.jpg   got it?
	    $f_date=~m/^.{3}(.).{4}_(..).(..).{3}_*/;
	    $ideal_measured_total = $1; #This string tells me which one it is
	    $yy=$2;
	    $mm=$3;

	    unless ($ideal_measured_total) {
		#If that didn't work because the string is slightly different...
		#RDLm_DUCK_2005_12_07_1000.ruv
		$f_date=~m/^.{3}(.)_.{4}_20(..).(..).{3}_*/;
		$ideal_measured_total = $1; #This string tells me which one it is
		$yy=$2;
		$mm=$3;
	    }
    
	    if ($yy>0 & $yy<10) {
		$month_folder = "20${yy}_$mm";
	    }
	    elsif ($yy>2000) {
		$month_folder = "${yy}_$mm";
	    }
	    else {
		#print "Warning $rem_file doesn't have a year that we can see!\n";
		#Not right, skip it!
		$month_folder="";
	    }

	    #Handle the month folder extensions: 
	    # 2003_11_i ideal radials
	    # 2003_11_m measured radials
	    # 2003_11   totals or images...
	    if ($ideal_measured_total=~/s/) {
		#If there is an s, it is radials from an ideal beam pattern
		$month_folder = $month_folder."_i";
	    }
	    elsif ($ideal_measured_total=~/z/) {
		#If there is an s, it is radials from an ideal beam pattern
		$month_folder = $month_folder."_m";
	    }
	    elsif ($ideal_measured_total=~/m/) {
		#If there is an s, it is radials from an ideal beam pattern
		$month_folder = $month_folder."_uvm";
	    }
	    elsif ($ideal_measured_total=~/i/) {
		#If there is an s, it is radials from an ideal beam pattern
		$month_folder = $month_folder."_uvi";
	    }
	    else {
		#don't do anything to the month folder name!!
		print "$ideal_measured_total -> $rem_file\n";
	    }
	    
	    #if ($debug) {print "$month_folder\n";}
	    unless (/_40_/ || /File/ || /old/ || !$month_folder) {
		if ($debug) {print "$rem_file -> $month_folder\n";}
		mkdir ("$local_dir/$month_folder");  
		$ftp->get("${remote_dir}$rem_file",
			  "$local_dir/$month_folder/$rem_file");
		if ($rem_file) {
		    push(@we_got_these,"$rem_file");
		    $got_these++;
		}
	    }
	}
	
	$we_got_these = join("\n",@we_got_these);
	open(LOCALIST,">>$local_dir/ftplog");
	print LOCALIST "$we_got_these\n";
	close(LOCALIST);
	
	if ($debug) { print "Got $got_these files.\n\n";}
    }
    
    ## logoff FTP site
    $ftp->quit;

    #system("ftp codar.marine.unc.edu");
    #$img_dir="$codar_dir/img_files/";

}