#!/bin/bash
# tr_fetch
# Fetch transp files

# Arguments to tr_fetch are project.yy and runid.
# An optional argument is where to copy the fetched files.

# LOGIC
# 1) Look in transp/result/ directory for 'ready' file holding names of
#     copyable files.
#    Ready-file name is <project.yy>_<runid>.FILESREADY
# 2) If read-file not there, then wait X seconds and retry
# 3) Read contents of ready-file. File names are separated by line
#     feeds. Format is <project.yy>/filename 
# 4) Copy files to local area.
# 5) Write a files-copied file back to transp/result directory
#    File name will be <project.yy>_<runid>.FILESCOPIED
#    Contents to be file name, start time, finish time 
# CONDITIONS
#  o GLOBUS must be set up and a valid GLOBUS proxy must exist
#  o If a target directory is given, files will be copied to that area
#     otherwise to the current area
#  o TRANSP_GRIDFTP_HOST if set contains the host name, otherwise
#    transpgrid.pppl.gov is used.
#  o TRANSP_GRIDFTP_RATE if set contains how often to probe for 
#    the filesready file otherwise 60 seconds is used.

# 2002-09-04 ler
# 2002-09-09 ler  update directory name
# 2003-10-06 clf  args to upper case
# 2003-11-17 clf  support nowait
# 2004-01-20 clf  Option -debug = -d plus globus-ftp-copy -dbg
# 2006-07-31 clf  Wait 10 seconds after each copy
# 2011-07-11 clf  handle missing year

PGR="tr_fetch"
GREP=/bin/grep
QSHARE=/p/transpgrid/qshare
os=`uname -s`
if [ "$os" == "SunOS" ]; then
   DATE=/usr/gnu/bin/date
   TAR_CMD=/usr/gnu/bin/tar
else
   DATE=date
   TAR_CMD=tar
fi
ERR="$PGR -E- "
trap `rm -f $tmpfile; exit 1` INT TERM
infomsg () { $silent || echo "$@" ; }
debugmsg () { $debug && echo "$@" ; }

usage="usage: $PGR [ -nowait ] [ -s ] [ -d{ebug} ] [-tar] <runid> <project> [<yy>] [<TARGETDIRECTORY>]"
nofile_prefix="error: the server sent an error response: 550 550"

# Get Options
silent=false
debug=false

# Check Arguments
let n=0
for arg
do
  case "$arg" in
  -nowait) nowait=true; shift;;
  -tar) tar=true; shift;;
  -debug) debug=true; DBG="-dbg -vb"; shift;;
  -s) silent=true; shift;;
  -d) debug=true; shift;;
  -*) shift ;;
   *) 
      e[$n]=$1
      let n=n+1
      shift
      ;;
  esac
done

if [ $n -eq 0 ]; then 
  echo "$usage" 1>&2
  exit 1
fi

# Check if last argument is an optional directory
if [ $n -gt 2 -a -d "${e[$n-1]}" ]; then
   let n=n-1
   tardir=${e[$n]}
   havedir=y
   let i=0
   while [ $i -lt $n ]; do  
      essential[$i]=${e[$i]}  
      let i=i+1      
   done
else
   essential=(${e[@]})
fi 

if [ $n -eq 0 -o $n -gt 3 ]; then 
  echo "$usage" 1>&2
  exit 1
fi

# Get Target Directory and Check
if [ -z "$tardir" ]; then
  tardir="`pwd`"
fi
if [ ! -d "/$tardir" ] ; then
  tardir="`pwd`/$tardir"
  echo "usisng $tardir"
fi
if [ ! -d "$tardir" ] ; then
  echo "$ERR Given directory ($tardir) is not a directory." 1>&2
  exit 1 
elif [ ! -w "$tardir" ] ; then
  echo "$ERR Given directory ($tardir) is not writable." 1>&2
  exit 1
fi
# Call tr_ckargs
tmpfile="/tmp/${PGM}_`$DATE +%H%M%S`_$$"
tr_ckargs $tmpfile ${essential[@]}
if [ $? -eq 1 ]; then
   order=`cat $tmpfile`
   runid=`echo $order |cut -f1 -d:`
   project=`echo $order |cut -f2 -d:`
   year=`echo $order |cut -f3 -d:`
   rm -f $tmpfile
else
   echo "$usage" 1>&2
   exit 1
fi
if [ -z "$year" -a -n "$runid" -a -n "$project" ]; then
   set_yr $runid  $project  > /dev/null 2>&1
   if [ -f ${runid}ID.TMP ]; then
      year=`cat ${runid}ID.TMP | sed 's/.*\.//'`
      rm ${runid}ID.TMP
   fi
fi
if [ -z "$runid" -o -z "$project" -o -z "$year" ]; then
   echo "$usage" 1>&2
   exit 1
fi

projyy=${project}.${year}

# Calculate Ready File Name and Copied file name
if [ -n "$tar" ]; then
  filesready=${projyy}_${runid}.tar.gz
else
  filesready=${projyy}_${runid}.FILESREADY
fi
filescopied=${projyy}_${runid}.FILESCOPIED
# Calculate Source Directory
srcdir=\~/transp/result/$projyy
debugmsg "silent=$silent, debug=$debug, projyy=$projyy, runid=$runid"
debugmsg "filesready=$filesready, filescopied=$filescopied"
debugmsg "srcdir=$srcdir, tardir=$tardir"

# Check Globus proxy
$GLOBUS_LOCATION/bin/grid-proxy-info -exists > /dev/null 2>&1
status=$?
if [ $status -ne 0 ] ; then
  echo "$ERR Globus not initialized and/or no valid grid proxy" 2>&1
  exit 1
fi

# Find name of gridftp host
if [ -n "$TRANSPGRID_SERVER" ]; then
  srvnam="$TRANSPGRID_SERVER"
else
  srvnam="transpgrid.pppl.gov"
fi

host=${TRANSP_GRIDFTP_HOST:-$srvnam}   
rate=${TRANSP_GRIDFTP_RATE:-60}

debugmsg "host=$host, rate=$rate"

# Get names of files when ready
if [ -n "$tar" ]; then
  filesreadynamehere=${tardir}/$filesready
else
  filesreadynamehere=/tmp/tr_fetch_$filesready
fi
filesreadynamethere=$srcdir/$filesready
filescopiednamehere=/tmp/tr_fetch_$filescopied
filescopiednamethere=$srcdir/$filescopied

infomsg "Looking for files on TRANSP host"
i=0
while [ 1 ]
do
  debugmsg "i=$i"
  debugmsg " gsiftp://$host/$filesreadynamethere file://$filesreadynamehere"
  if [ -n "$DBG" ]; then
     a=`globus-url-copy -nodcau $DBG gsiftp://$host/$filesreadynamethere \
                file://$filesreadynamehere`
     status=$?
  else
     a=`globus-url-copy -nodcau gsiftp://$host/$filesreadynamethere \
                file://$filesreadynamehere 2>&1`
     status=$?
  fi
  debugmsg "status=$status"
  debugmsg "$a"
  if [ $status -eq 0 ] ; then
    break
  else 
#    echo "$a" | $GREP "^$nofile_prefix" > /dev/null 2&>1 || echo "GridFTP failure '$a'"
    status=`echo $a | grep "not a plain file"`
    if [ -z "$status" ]; then 
       status=`echo $a | grep "$nofile_prefix" `
       if [ -z "$status" ]; then 
           echo "GridFTP failure '$a'"
       fi
    fi
    if [ $i -eq 0 ]; then
       infomsg "...check if already fetched"
       if [ -n "$DBG" ]; then
         globus-url-copy -nodcau $DBG gsiftp://$host/$filescopiednamethere \
	    file://$filescopiednamehere
         status=$?
       else
         globus-url-copy -nodcau gsiftp://$host/$filescopiednamethere \
	    file://$filescopiednamehere 2> /dev/null
         status=$?
       fi
       if [ $status -eq 0 ] ; then
          infomsg "Run already fetched"
          rm -f $filescopiednamehere fi
          exit 1
       fi
       project=${projyy%.*}
       there=$QSHARE/${runid}_${project}.FETCHED
       if [ -n "$DBG" ]; then
         globus-url-copy -nodcau $DBG gsiftp://$host/$there \
            file://$filescopiednamehere
         status=$?
       else
         globus-url-copy -nodcau gsiftp://$host/$there \
            file://$filescopiednamehere 2> /dev/null
         status=$?
       fi
       if [ $status -eq 0 ] ; then
          infomsg "Run already fetched"
          rm -f $filescopiednamehere
          exit 1
       fi
    fi
    if [ -n "$nowait" ]; then
       infomsg "No files found"
       exit 0
    fi
    i=`expr ${i} + 1`
    infomsg "... waiting `$DATE`" 1>&2

    sleep $rate
  fi
done

if [ -z "$tar" ]; then
# Fetch individual files
  infomsg "Retrieving files from TRANSP host"
  success=1
  touch $filescopiednamehere

while read line ; do
  infomsg " ... retrieving file $line" 
  there="$srcdir/$line"
  here="$tardir/$line"
  date_start=`$DATE -u`
  globus-url-copy -nodcau $DBG gsiftp://$host/$there  file://$here
  status=$?
  date_finish=`$DATE -u`
  if [ $status -eq 0 ] ; then
    echo "$there START=$date_start FINISH=$date_finish" >> $filescopiednamehere
    infomsg "File $there copied"
  else
    success=0
    echo "$there START=$date_start FAIL" >> $filescopiednamehere
    infomsg "File $there copy failure"
  fi
#Temporary
    sleep 10
done < $filesreadynamehere
else
  debugmsg "check $filesreadynamehere"
  if [ -f $filesreadynamehere ]; then
#    Verify tar file
     $TAR_CMD ztf $filesreadynamehere  1>&2 >/dev/null
     if [ $? -eq 0 ]; then
        success=1 
        date_finish=`$DATE -u`
        debugmsg "FINISH=$date_finish > $filescopiednamehere"
        echo "FINISH=$date_finish" > $filescopiednamehere
     else
        success=0
     fi
  else
     success=0
  fi
fi
#
if [ $success -eq 1 ]; then
  infomsg "Files retrieved"
  there=$filescopiednamethere
  here=$filescopiednamehere
  debugmsg "globus-url-copy -nodcau $DBG file://$here gsiftp://$host/$there"
  globus-url-copy -nodcau $DBG file://$here gsiftp://$host/$there
else 
  echo "$ERR Files not retrieved" 1>&2
fi

# Clean up
if [ -z "$tar" ]; then
   /bin/rm -f $filesreadynamehere
fi
/bin/rm $filescopiednamehere

exit $success
