root/codar/trunk/bin/archive_nccoos_codar.pl

#!/afs/isis/pkg/isis/bin/perl
# copy_codar2nemo.perl 
# Time-stamp: <2006-07-06 16:13:13 haines>
#
#

# Modified by Sara Haines, 06 Jul 2006
# 
# 1. removed scp and rsync stuff
# 2. general code cleanup
# 3. changed where data written to nemo from hws-AFS-space
# 4. all code (MATLAB and perl scripts) located on nemo
#
# original code from Luke Stearns
#

#Let's start by setting up some basic path information:
# $hws         = "/afs/isis.unc.edu/depts/marine/workspace/hseim/";
# $seacoos_dir = "{hws}seacoos/data/codar";
$nemo      = "/seacoos/data/";
$codar_dir = "${nemo}nc-coos/hfradar/";

# also need specific perl module libraries not in AFS space
use lib '/afs/isis.unc.edu/depts/marine/workspace/hseim/seacoos/perl/site_perl/';

# ftp library
use Net::FTP;

# date/time manipulation library
use Date::Manip;

# I found a neat set manipulation library (to find the difference 
# between two sets (which files do we need to grab?)

use Set::Scalar;

#Get yesterday's date in yymmdd format
$date=&UnixDate("today", "%y%m%d");

######################################################################
# Codar ftp get data #################################################
######################################################################

# I just took this from a script Sara wrote for www.seacoos.org/wap/
# Author: Sara Haines (2003-02-22) 
#

# if debugging requested print messages to STDOUT
if (grep /[debug|DEBUG|d]/, @ARGV) {
  $debug = 1;
}

# print Date and Time and version of perl + patchlevel /1000
$now=&UnixDate(&ParseDate("today"), "%Y.%m.%d %H:%M:%S");
if ($debug) { print "\n==== Starting: $now ==== Perl Version: $]\n"; }

# get current directory
$cwd = `pwd`;

# start stop-watch for cummulative times
$starttime = time;

######################################################################
# Grab from codar.marine.unc.edu

# '/Codar/SeaSonde/Data/';

# for each station in list, get file from ftp and place in local data drive
%dir_info = (
             "/Codar/SeaSonde/Data/RadialSites/Site_2/", "${codar_dir}/codar_radials/duck/",
             "/Codar/SeaSonde/Data/RadialSites/Site_1/", "${codar_dir}/codar_radials/haty/",
             "/Codar/SeaSonde/Data/Totals/", "${codar_dir}/codar_totals/ouba/",
             #"/Codar/SeaSonde/Data/Totals/TOTL*","tuv_files/",
             #"Pictures/", "img_files/",
             );

# login to FTP site
if ($debug) { print "FTP files from codar.marine.unc.edu to nemo ...\n"; }

&ftp_grab_data("codar.marine.unc.edu","codar",'br@ggsc@tt3r',%dir_info) ||
    print "Problems connecting to codar.marine.unc.edu!\n";

######################################################################
# Grab from gulf.marine.rutgers.edu

if (0) {
    # for each station in list, get file from ftp and place in local data drive
    %dir_info = (
                 '../Radial Files/Site 2/', 'radial_files/wild/',
                 );

    # login to FTP site
    if ($debug) { print "Ftp files from gulf.marine.rutgers.edu to AFS ...\n"; }

    $start_rutgers_time = time;

    &ftp_grab_data("gulf.marine.rutgers.edu","codar","tne",%dir_info) ||
        print "Problems connecting to gulf.marine.rutgers.edu!\n";
}

######################################################################
# Done with all grabs ################################################
######################################################################
# stop stop-watch
$cummtime = time - $starttime;
if ($debug) { print "Total data grab time = $cummtime (seconds).\n"; }


######################################################################
# Subroutines ########################################################
######################################################################

#This one is also hacked from Sara's script 
sub read_bypackage_info {

  open(BYPACKINFO, "<$packageinfo");
  $i=0;
  while ($entry = <BYPACKINFO>) {
    chop($entry);
    ($name[$i],@long_name) = split(/\s+/,$entry);  
    $long_name[$i]=join(@long_name,/\s/);
    $i++;
  }
  close(BYPACKINFO);
  
} # sub read_bypackage_info

######################################################################
### FTP Grab Data ####################################################
sub ftp_grab_data {

    ($rem_site,$user,$pass,%dir_info)=@_;
    $start_time = time;

    #print "Host: $rem_site\nUser: $user\n$pass!\n";
    #$ftp ='';
    #while (time-$start_time<60) {
    $ftp = Net::FTP->new("$rem_site");
    #}
    $ftp->login("$user","$pass");
    #$ftp->binary();
    foreach $remote_dir (keys(%dir_info)) {
        
        $local_dir=$dir_info{$remote_dir};
        if ($debug) { print "$local_dir:\n"; }
        print "$remote_dir\n";

        @loc_list=();
        @we_got_these=();
        $bad=0; $good=0;
        open(LOCALIST,"$local_dir/ftplog");
        while (<LOCALIST>) {
            chop;
            #Ignore empty lines
            if($_) {
                push(@loc_list,"$_");
            }
        }
        close(LOCALIST);
        
        @rem_files=();
        @rem_files=$ftp->ls("${remote_dir}");
        #$rem_list =~s/join(',',"@rem_list")/$remote_dir/g;
        $rem_list = join(',',@rem_files);
        $rem_list =~s/$remote_dir//g;

        #@rem_list = ();
        #foreach $rem_file (@rem_files) {
        #    @path = split('/',$rem_file);
        #    my $tmp_var = $rem_file;
        #    $tmp_var =~s/$path[-1]$//;
        #    $remote_dir = $tmp_var;
        #    $rem_file = $path[-1];
        #    push(@rem_list,"$rem_file");
        #}
        #print "$rem_list\n";
        @rem_list = split(',',$rem_list);
        
        $set_loc=Set::Scalar->new(@loc_list);
        $set_rem=Set::Scalar->new(@rem_list);
        
        $get_these=$set_rem->difference($set_loc);
        $get_these=~s/[()]//g;

        @get_these=split('\s+',$get_these);
        if ($debug) {print "L:$#loc_list R:$#rem_list G:$#get_these\n";}
        print "$rem_list[0]\n";
        $got_these = 0;
  
        foreach $rem_file (@get_these) {
            $_=$rem_file;
            $f_date=$rem_file;
            #print "$f_date\n";
            #This matching condition may be a bit confusing, so here goes:
            # any three characters (Rad or Tot) keep track of the fourth (s,z,_) then four more
            # characters, an underscore, keep track of yy then mm, ignore the next three but make 
            # sure they are followed by an underscore, then whatever:
            # Rad(s)DUCK_(03)_(11)_11_2200  the things in the parens, I'm keeping
            # Tot(_)OUBA_(03)_(10)_30_1600.jpg   got it?
            $f_date=~m/^.{3}(.).{4}_(..).(..).{3}_*/;
            $ideal_measured_total = $1; #This string tells me which one it is
            $yy=$2;
            $mm=$3;

            unless ($ideal_measured_total) {
                #If that didn't work because the string is slightly different...
                #RDLm_DUCK_2005_12_07_1000.ruv
                $f_date=~m/^.{3}(.)_.{4}_20(..).(..).{3}_*/;
                $ideal_measured_total = $1; #This string tells me which one it is
                $yy=$2;
                $mm=$3;
            }
    
            if ($yy>0 & $yy<10) {
                $month_folder = "20${yy}_$mm";
            }
            elsif ($yy>2000) {
                $month_folder = "${yy}_$mm";
            }
            else {
                #print "Warning $rem_file doesn't have a year that we can see!\n";
                #Not right, skip it!
                $month_folder="";
            }

            #Handle the month folder extensions: 
            # 2003_11_i ideal radials
            # 2003_11_m measured radials
            # 2003_11   totals or images...
            if ($ideal_measured_total=~/s/) {
                #If there is an s, it is radials from an ideal beam pattern
                $month_folder = $month_folder."_i";
            }
            elsif ($ideal_measured_total=~/z/) {
                #If there is an s, it is radials from an ideal beam pattern
                $month_folder = $month_folder."_m";
            }
            elsif ($ideal_measured_total=~/m/) {
                #If there is an s, it is radials from an ideal beam pattern
                $month_folder = $month_folder."_uvm";
            }
            elsif ($ideal_measured_total=~/i/) {
                #If there is an s, it is radials from an ideal beam pattern
                $month_folder = $month_folder."_uvi";
            }
            else {
                #don't do anything to the month folder name!!
                print "$ideal_measured_total -> $rem_file\n";
            }
            
            #if ($debug) {print "$month_folder\n";}
            unless (/_40_/ || /File/ || /old/ || !$month_folder) {
                if ($debug) {print "$rem_file -> $month_folder\n";}
                mkdir ("$local_dir/$month_folder");  
                $ftp->get("${remote_dir}$rem_file",
                          "$local_dir/$month_folder/$rem_file");
                if ($rem_file) {
                    push(@we_got_these,"$rem_file");
                    $got_these++;
                }
            }
        }
        
        $we_got_these = join("\n",@we_got_these);
        open(LOCALIST,">>$local_dir/ftplog");
        print LOCALIST "$we_got_these\n";
        close(LOCALIST);
        
        if ($debug) { print "Got $got_these files.\n\n";}
    }
    
    ## logoff FTP site
    $ftp->quit;

    #system("ftp codar.marine.unc.edu");
    #$img_dir="$codar_dir/img_files/";

}