#! /bin/bash
#
#@usage tr_start <runid>
#@arg <runid>  runid for this shot (e.g. 37065Y04)
#
#@! Main entry point for setting up the execution request and mdsplus tree.
#@! This script sets the environment and obtains the mdsplus server and tree names.
#@!
#@! Note: This script is not used by GA, they have their own software for filling
#@! the MDS+ tree and building the request file.
#@! 05/03/02 C.Ludescher
#@!
#@! if the TRANSPGRID_SERVER environment variable is set, this will be used as the
#@! actual name of the transpgrid.pppl.gov server otherwise the existing default
#@! server TRANSPGRID2.PPPL.GOV will be used for the actual name
#

# 
# 'module load nstx/mdsplus'  resets the LANG environment variable to something
# which prevents sed, called from the tr_start.pl perl script, from correctly
# identifying the *_PSERVE variables in the namelist.  Hopefully this won't
# mess up non pppl systems.
#
export LANG=en_US.UTF-8

PGM=tr_start
USAGE="
usage: $PGM <runid> [tshare] [big | infiniband | fast] [walltime] [notriage]

   Argument:
      runid

   Optional Arguments:
   A) code options
      tshare      -  alternate code (default is pshare)

   B) pbs queue options (other than default) 
      infiniband  -  kruskal (for mpi jobs)
      fast        -  stix (for mpi jobs)
      big         -  for serial requesting 8.GB memory
                     for mpi 6.GB per core
                     (default is 2.GB per core)
      walltime  - set approximate  run walltime in hours
                      from1 hour to 80 hours
                      (default is 80 hours)
      notriage  - please use this flag flag if you do not request
                      a triage in case of crash
"



#@! Check arguments
#@! ---------------
if [ $# -lt 1 ] ; then
  echo "$USAGE" 1>&2
  exit 1
fi

case "$1" in
     -help | --help | -h | --h)  
         echo "$USAGE" 2>&1
         exit 1 
         ;;
     *)
         runid=$1
         ;;
esac

if [ $# -gt 1 ]; then
  while [ -n "$2" ]; do
    case "$2" in
     tshare)
          tshare="tshare"
          shift;;
     big)
          BIG="-q big"
          shift;;
     fast)
          BIG="-q fast"
          shift;;
     infiniband)
          BIG="-q infiniband"
          shift;;
     randre)
          tshare="-alt randre"    # for testing
          shift;;
     xyuan2)
          tshare="-alt xyuan2"    # for testing
          shift;;
     fpoli)
         tshare="-alt fpoli"     # for testing
         shift;;
     notriage)
         notriage="notriage"     # do not triage a crash
         shift;;
     walltime)
            _min=1
            _max=80
	    _proclist="[${_min},2,3,...,${_max}]"
	    echo "$PGM: you are asking for run $  " 
	    while [ -z "$walltime" ]
	    do
              echo  -n "          Enter number hours ${_proclist}: "
	      read -e _WTIME
              if [ $_WTIME -eq $_WTIME 2> /dev/null ]; then
		  if [ $_WTIME -lt $_min -o $_WTIME -gt $_max ]; then
		      echo " ? $_WTIME not valid "
		  else
		      walltime="--time= $_WTIME:00:00"
		  fi
              else 
		  echo " ? $_WTIME not an integer "
              fi  
            done
         shift;;
      *)
         echo "?tr_start: unrecognized option = $3"
          exit 1
    esac
  done
fi
#
#@create env .userrc  loads in the standard environment unless developer
#@use    env $TRANSP_DEVELOPER transp developers should define this to avoid using .userrc
#
if [ -z "$TRANSP_DEVELOPER" ]; then
   if test -f /u/pshare/globus/.userrc; then
     . /u/pshare/globus/.userrc
     pppl="TRUE"
#@! Check not running as tr_<user>
     if [ "$zid2" = "tr_" ]; then
        echo "
        Please do not run as `whoami`
        login as yourself and start again
        "
        exit 1
     fi
   fi
fi

#@! Check if Namelist TR.DAT exists
if [ ! -f ${runid}TR.DAT ]; then
    echo "
    ${runid}TR.DAT does not exist
    "
    exit 1
fi
#
#@! Get Server -- use $MDS_TRANSP_SERVER otherwise query user
#@use env $MDS_TRANSP_SERVER default mdsplus server to use
#@! ----------------
#
if test -z "$MDS_TRANSP_SERVER"; then
   echo "$PGM: MDS_TRANSP_SERVER not defined"
   echo -n "     Enter Server: "
   read -e IN
   SERVER=`echo $IN | tr 'a-z' 'A-Z'`
else
   SERVER=`echo $MDS_TRANSP_SERVER | tr 'a-z' 'A-Z'`
fi

until [ $valid ]; do
   base=${SERVER/.*/}
   if [ $base = $SERVER -a $SERVER != "NONE" ]; then
      echo    "?    You must specify full Server name"
      echo -n "     Enter Server: "
      read -e IN
      SERVER=`echo $IN | tr 'a-z' 'A-Z'`
   else
      valid=1 
   fi
done


if [ $SERVER = BIRCH.PPPL.GOV ]; then
#@! Use TRANSPGRID
    SERVER="TRANSPGRID.PPPL.GOV"
    echo "$PGM: All Runs are now stored on TRANSPGRID.PPPL.GOV"
    DTREE=TRANSP_
    DTREE_W="TRANSP_<tok>"
    opt="noxtar"
    pppl_opt="FORCE_MDS"
fi

#07/27/2011 CLF
#05Feb2016 RGA
if [ $SERVER = TRANSPGRID.PPPL.GOV ]; then
    if [ -n "$TRANSPGRID_SERVER" ]; then
      SERVER="$TRANSPGRID_SERVER" 
    else
      SERVER="TRANSPGRID.PPPL.GOV"
    fi
fi

if [ $SERVER = TRANSPGRID.PPPL.GOV -o $SERVER = "$TRANSPGRID_SERVER" ]; then
#    uid=`getent passwd |grep ^\`whoami\` |awk -F: '{print $3}'`
#    if [ -f /tmp/x509up_u${uid} -o -n "$X509_USER_PROXY" ]; then
#       SERVER="_TRANSPGRID.PPPL.GOV"
#    else
#       echo " "
#       echo "--W-- TRANSP now requires a FusionGrid Certificate"
#       echo "      To apply goto https://cert.fusiongrid.org "
#       echo "      To automatically renew at login see:"
#       echo "         http://w3.pppl.gov/transp/pppl.html#proxy "
#       echo " "
#    fi
    DTREE=TRANSP_
    DTREE_W="TRANSP_<tok>"
    opt="noxtar"
    if [ -n "$pppl" ]; then 
       pppl_opt="FORCE_MDS"
    fi
elif [ $SERVER = _TRANSPGRID.PPPL.GOV ]; then
    DTREE=TRANSP_
    DTREE_W="TRANSP_<tok>"
    opt="noxtar"
    if [ -n "$pppl" ]; then 
       SERVER = TRANSPGRID.PPPL.GOV
       pppl_opt="FORCE_MDS"
    else
       SERVER = TRANSPGRID.PPPL.GOV
    fi
elif [ $SERVER = NONE ]; then
    opt="nomdsplus"
    MDS_TRANSP_TREE=NONE
fi

#
#@! Get Tree name based on server or use $MDS_TRANSP_TREE or query user
#@use    env $MDS_TRANSP_TREE  mdsplus tree to use if no default
#@! ------------------
#
if [ -n "$DTREE" ]; then
   if [ -n "$MDS_TRANSP_TREE" ]; then
      base=${MDS_TRANSP_TREE/_*/_}
      if [ $base != $DTREE ]; then
         echo    "?    MDS_TRANSP_TREE defined as $MDS_TRANSP_TREE"
         echo    "?    should be $DTREE_W "
         echo -n "     Verify Tree: "
         read -e IN
         TREE=`echo $IN | tr 'a-z' 'A-Z'`
      else
         TREE=$DTREE
      fi
   else
      echo "$PGM: will use Default Tree = $DTREE_W"
      TREE=$DTREE
   fi
else
   if [ -n "$MDS_TRANSP_TREE" ]; then
      TREE=$MDS_TRANSP_TREE
   else
      echo    "$PGM: MDS_TRANSP_TREE not defined"
      echo -n "      Enter Tree: "
      read -e IN
      TREE=`echo $IN | tr 'a-z' 'A-Z'`
   fi
fi
#check if _transpgrid in PPPL Namelist
if [ -n "$pppl_opt" ]; then
   zmds=`grep -i '^\ *kmdsplus\ *=.*' ${runid}TR.DAT |sed 's/.*= *//' |sed 's/\ *\!.*//'`
   if [ $? = 0 ]; then
     if [ "$zmds" = "1" ]; then
        zserv=`grep -i '^\ *mds_server\ *=\ *._transpgrid' ${runid}TR.DAT`
        if [ $? = 0 ]; then  
           cat ${runid}TR.DAT |sed -e's/_transpgrid/transpgrid/g' -e's/_TRANSPGRID/TRANSPGRID/g' > foo
           if [ $? = 0 ]; then
              mv foo ${runid}TR.DAT
           fi
        fi
     fi
   fi
fi 


# Handle MPI
# 07/03/2013 CLF
#            source tr_ck_mpi
#            was extracted from tr_start to be used by nersc_qsub
if [ -n "$CODESYSDIR" ]; then
   source $CODESYSDIR/qcsh/tr_ck_mpi
elif [ -n "$TRANSP_LOCATION" ]; then
   source $TRANSP_LOCATION/tr_ck_mpi
else
   echo "You must define TRANSP_LOCATION"
   exit 1
fi

LIST="NB TORIC PTR NGEN NCQL"
for i in $LIST; do
   inp=0   np=""
   echo "$i"
   handle_mpi $runid $i 
   if [ $inp -gt 0 ]; then
      case $i in
         NB)
            NBI_NPROCS=$inp
            npb=$np
            ;;
          NGEN)
            GENRAY_NPROCS=$inp
            npg=$np
            ;;
          NCQL)
            NCQL3D_NPROCS=$inp
            npc=$np
            ;;
         TORIC)
            TOR_NPROCS=$inp
            npt=$np
            ;;
         PTR)
            PTR_NPROCS=$inp
            nptr=$np
            ;;
       esac
    fi
done

# check if need to use mpi
tr_ck_user_mpi $runid $tok
if [ $? -ne 0 ]; then
   echo "
 Your Namelist setting requires to use mpi.
 Please change your Namelist: set nbi_pserve=1 

 "
   exit 1
fi
#
#@call tr_start.pl <tr_start_server> <tr_start_tree> $1
#
echo "tr_start.pl $SERVER $TREE $runid $opt $pppl_opt $tshare $npb $npt $nptr $BIG $walltime $notriage"
tr_start.pl $SERVER $TREE $runid $opt $pppl_opt $tshare $npb $npt $nptr $npg $npc $BIG $walltime $notriage

status=$?  
if [ $status -ne 0 ]; then
   exit 1
else
  exit 0
fi


