#!/bin/tcsh -f
#
# DMC -- modified for NxM MPI run -- mpi_nubeam_comp_exec
#        (can also be used for serial run -- $NPROCS = 1).
#
# Assumption: ~/.cshrc sets USER_ENV_RESET_DISABLED to prevent system
# reset of PATH and module loads on shell startup.  Note use of "-f" switch.
#
# Check whether ~/.login needs to be executed.  This is done by seeing 
# if a known command is defined.
#
# mpi_nubeam_comp_exec requirements:  these environment variables must
# be defined:  see $CODESYSDIR/csh/nubeam_qsub script...
#
#    NUBEAM_ACTION  --  operation to be performed by mpi_nubeam_comp_exec
#
#    NUBEAM_WORKPATH  --  directory containing working files 
#       this has to be a relative path to a subdirectory of the directory 
#       from which this script is submitted.
#
#    PREACTDIR -- full directory path to atomic/nuclear physics database
#       used by NUBEAM  (TRANSP developers should have this defined from 
#       login scripts).
#
#    ADASDIR -- full directory path to ADAS atomic/nuclear physics database
#       used by NUBEAM  (TRANSP developers should have this defined from 
#       login scripts).
#
# additional environment variables that may be used:
#
#    NUBEAM_REPEAT_COUNT -- control the number and duration of NUBEAM
#       time steps against the fixed target plasma state;
#
#    NUBEAM_POSTPROC -- "FBM_WRITE" or "FBM_WRITE:<filename>" to request
#       distribution function NetCDF file output
#
#    FRANTIC_INIT
#    FRANTIC_ACTION
#
#============================================================================
# CAUTION -- this job involves a lot of file movement.
# It will not be efficient -- this is for testing, not production use...
#============================================================================
#
# (comments from the PPPL system PBS script template...)
#
# --- this is a sample script for running a program on the beowulf
#     cluster. You modify this script for your needs, then use the
#     'qsub' command to submit it to the cluster's job scheduler.
#
#     The job scheduler is the Portable Batch System (PBS). It
#     uses directives in this file to determine where to run
#     the job, how many hosts to run it upon, where to put output
#     files, etc.
#
# remarks: a line beginning with # is a comment
#          a line beginning with #PBS is a pbs directive
#          assume (upper/lower) case to be sensitive
#
# ------------------------------------------------------------
# Basic Information
#
# --- the name of your job
#PBS -N testjob_nnxmm
 
# --- Mail options
#     mail on execution("b"), termination ("e"), or interruption ("a")
#PBS -m aeb
 
# ------------------------------------------------------------
# Job execution details

###NERSC -A m876
###  John=m681 Wael=m876
###NERSC -S /bin/bash
###NERSC -l mppwidth=N
###NERSC -l mppnppn=M
###NERSC -V
 
# --- run the job on 'n' nodes with 'm' processors per node (ppn)
#     (script to replace value, below, with nodes=<n>:ppn=<m>)
#PBS -l NxM

# --- specify the period of time the job will run in the format
#     hh:mm:ss (ex. 72 hours is 72:00:00)
#PBS -l walltime=24:00:00

# --- submit this job to a special queue. If you do not specify a
#     special queue, the job will run in the default queue
#     There are three main queues at PPPL:
#     kestrel - the default queue used for multi node jobs
#     kite - a special queue of nodes connected with Infiniband
#     sque - nodes used for single CPU jobs (serial jobs)
###PBS -q kestrel
###PBS -q kruskal !if using this made also changes in mpirun -np ... see below

# --- do not rerun this job if it fails
#PBS -r n

# ------------------------------------------------------------
# When a batch job starts execution, a number of environment
# variables are predefined, which include:
#
#      Variables defined on the execution host.
#      Variables exported from the submission host with
#                -v (selected variables) and -V (all variables).
#      Variables defined by PBS.
#
# The following reflect the environment where the user ran qsub:
# PBS_O_HOST    The host where you ran the qsub command.
# PBS_O_LOGNAME Your user ID where you ran qsub.
# PBS_O_HOME    Your home directory where you ran qsub.
# PBS_O_WORKDIR The working directory where you ran qsub.
#
# These reflect the environment where the job is executing:
# PBS_ENVIRONMENT is set by PBS to "PBS_BATCH" if the job is a batch job, or
#                 to "PBS_INTERACTIVE" if the job is an interactive job.
# PBS_O_QUEUE   The original queue you submitted to.
# PBS_QUEUE     The queue the job is executing from.
# PBS_JOBID     The job's PBS identifier.
# PBS_JOBNAME   The job's name.

# --- define standard output and error files for the job
#     default is <jobname>.o<jobid> and <jobname>.e<jobid>
##PBS -o test.out
##PBS -e test.err
# --- or combine both output and error streams into one file with the
#     job name and id as the default file name
#PBS -j oe

# ------------------------------------------------------------
# CHECK ENVIRONMENT

set loc_tshare = FALSE
if ( $?TSHARE_FLAG ) then
  echo "pppl_mpi_nubeam detects TSHARE_FLAG = $TSHARE_FLAG "
  set loc_tshare = $TSHARE_FLAG
else
  echo "pppl_mpi_nubeam detects no TSHARE_FLAG (OK) "
endif

if ( $loc_tshare != TRUE ) then
  if ( $?TSHARE_SIGNAL ) then
    echo "pppl_mpi_nubeam detects TSHARE_SIGNAL = $TSHARE_SIGNAL "
    if ( $TSHARE_SIGNAL == TRUE ) then
      echo " ...run tshare_setup... "
      source ~pshare/tshare_setup.csh
    endif
  endif
endif

echo " +++++++++++++++++++++++++++++++++++ "
set zdate = `date`
echo -n "Starting job at date: $zdate"

# If mounted filesystem, do not create/move directories
set NODE = `uname -n`
# if ( $?NERSC ) then
if ($NODE == stix.pppl.gov || $?NERSC ) then
   set NO_MOVE = 1

   #! define this to prevent portlib MPI code from attempting certain I/O ops
   setenv PARALLEL_FILE_SYSTEM ASSUMED

   #   if ( $NERSC_HOST == "carver" ) then
   #      set NO_PLIST = 1
   #   endif
endif

set runtr_chk = `which mpi_nubeam_comp_exec`
@ ier = 0
if ( $#runtr_chk != 1 ) then
  @ ier++
else
  if ( ! -f $runtr_chk ) then
    @ ier++
  endif
endif

if ( $ier > 0 ) then
  echo " "
  echo " mpi_nubeam_comp_exec: not found -- so, source ~/.login "
  if ( -f ~/.login ) then
     source ~/.login
  endif
endif

echo " "
set runtr_chk = `which mpi_nubeam_comp_exec`
echo " which mpi_nubeam_comp_exec: $runtr_chk"
echo " "

@ ier = 0
if ( $#runtr_chk != 1 ) then
  @ ier++
else
  if ( ! -f $runtr_chk ) then
    @ ier++
  endif
endif

if ( $ier > 0 ) then
  echo " "
  echo " mpi_nubeam_comp_exec: STILL not found -- exiting..."
  exit 1
endif

# ------------------------------------------------------------
# environment tests specific for mpi_nubeam_comp_exec

if ( $?NUBEAM_ACTION ) then
  echo " NUBEAM_ACTION: $NUBEAM_ACTION "
else
  echo " ?? NUBEAM_ACTION undefined, exiting... "
  exit 1
endif

if ( $?LOG_LEVEL) then
  echo " LOG_LEVEL: $LOG_LEVEL "
else
  setenv  LOG_LEVEL 1
  echo " LOG_LEVEL set to default: $LOG_LEVEL "
endif


if ( $?PREACTDIR ) then
  echo " PREACTDIR: $PREACTDIR "
  if ( ! -d $PREACTDIR ) then
    echo " ?? directory not found: $PREACTDIR "
    exit 1
  endif
else
  echo " ?? PREACTDIR undefined, exiting... "
  exit 1
endif
if ( $?ADASDIR ) then
  echo " ADASDIR: $ADASDIR "
  if ( ! -d $ADASDIR ) then
    echo " ?? directory not found: $ADASDIR "
    exit 1
  endif
else
  echo " ?? ADASDIR undefined, exiting... "
  exit 1
endif

if ( $?NUBEAM_WORKPATH ) then
  echo " NUBEAM_WORKPATH: $NUBEAM_WORKPATH "
else
  echo " ?? NUBEAM_WORKPATH undefined, exiting... "
  exit 1
endif

if ( $?INPUT_STATE ) then
  echo " INPUT_STATE: $INPUT_STATE "
else
  echo " ?? INPUT_STATE undefined, exiting... "
  exit 1
endif

if ( $?OUTPUT_STATE ) then
  echo " OUTPUT_STATE: $OUTPUT_STATE "
else
  echo " ?? OUTPUT_STATE undefined, exiting... "
  exit 1
endif
  echo " FRANTIC_ACTION = $FRANTIC_ACTION"

# ------------------------------------------------------------
# Log interesting information
#

echo "-------------------"
echo "This is a $PBS_ENVIRONMENT job"
echo "This job was submitted to the queue: $PBS_QUEUE"
echo "The job's id is: $PBS_JOBID"
echo "-------------------"
echo "The master node of this job is: $PBS_O_HOST"

# --- the nodes allocated to this job are listed in the
#     file PBS_NODEFILE

# --- count the number of processors allocated to this run 
# --- $NPROCS is required by mpirun.
set NPROCS=`wc -l < $PBS_NODEFILE`

set NNODES=`uniq $PBS_NODEFILE | wc -l`

echo "This job is using $NPROCS CPU(s) on the following $NNODES node(s):"
echo "-----------------------"
uniq $PBS_NODEFILE | sort
echo "-----------------------"

set NODE_LIST = `uniq $PBS_NODEFILE | sort`

# ------------------------------------------------------------
# Setup execution variables
#
# Setup working directory & data source directory
#
# --- PBS_O_WORKDIR is the working directory from
#     which this job was submitted using 'qsub'
#
echo "The job origination directory is $PBS_O_WORKDIR"

set suffix = `echo $PBS_O_WORKDIR | sed 's#^/local/##'`
if ( "$suffix" != "$PBS_O_WORKDIR" ) then

  # submitted from directory on /local/ disk

  set orig_host = `echo $PBS_O_HOST | sed 's#\..*##'`
  set source_dir = /l/$orig_host/$suffix

else
  set source_dir = $PBS_O_WORKDIR
  if ( ! $?NO_MOVE) then
    set sctail = $source_dir:t
    set username = `whoami`

    set suffix = $username/nubeam/$sctail
    set wkroot = /local/$suffix

    run_gmkdir $wkroot/$NUBEAM_WORKPATH
  else
    set wkroot = $PBS_O_WORKDIR
  endif
endif

set orig_wkpath = $source_dir/$NUBEAM_WORKPATH
if ( ! -d $orig_wkpath ) then
  mkdir $source_dir/$NUBEAM_WORKPATH
endif

setenv NUBEAM_SUMMARY_PATH $orig_wkpath

if ( $NUBEAM_ACTION == "init" ) then
  echo " "
  set flist = `ls $orig_wkpath`
  foreach file ( $flist ) 
    set ztes1 = `echo $file | sed 's#\.msgs##'`
    set ztes2 = `echo $file | sed 's#_ptcl_##'`
    set ztes3 = `echo $file | sed 's#_birth_##'`
    if ( $ztes1 != $file || $ztes2 != $file || $ztes3 != $file ) then
      echo "NUBEAM_ACTION init: remove file: $file"
      rm -f $orig_wkpath/$file
    endif
  end
endif

echo " "
echo "The mpi job will start here:"
if ( "$suffix" != "$PBS_O_WORKDIR" ) then
  rm -rf /local/$suffix/$NUBEAM_WORKPATH
  run_gmkdir /local/$suffix/$NUBEAM_WORKPATH

  cd /local/$suffix
  pwd

else

  cd $wkroot
  pwd

endif

echo " "
echo " get processor list... "
set plist = $NUBEAM_WORKPATH/proclist.dat
rm -f $plist
if ( $NPROCS > 1 ) then
  if ( $?MPI_CMD) then
    eval 'set mpx = "'"$MPI_CMD"'"'
  else
    set mpx = `which mpirun`
  endif
  echo " $mpx -np $NPROCS `which mpi_proclist` $plist"
  $mpx -np $NPROCS `which mpi_proclist` $plist
else
  set cur_host = `hostname | sed 's#\..*##'`
  echo " 0 $cur_host " > $plist
endif

if ( ! -f $plist ) then
  echo "mpi_proclist failed"
  exit 1
else
  echo "-----------------------------"
  echo "  pwd = `pwd` "
  echo "  plist = $plist"
  echo "  plist file contents:"
  cat $plist
  echo "-----------------------------"
endif

set cpnums = `cat $plist | awk '{print $1}'` 
set cpnames = `cat $plist | awk '{print $2}' | sed 's#\..*##' | uniq`

if ( ! $?NO_MOVE) then
  echo " create work directories on each compute server node..."
  set targlist = ()
# 03/06/09 CLF: $NODE_LIST not usable on kruskal, use $cpnames
  foreach node ( $cpnames )
    set nnode = `echo $node | sed 's#\..*##'`
    set target = "/l/$nnode/$suffix"
    run_gmkdir $target/$NUBEAM_WORKPATH
    set targlist = ( $targlist $target )
  end

# check (due to paranoia about NFS...)

  echo " check work directories on each compute server node..."
  @ ier = 0
    set targlist = ()
  foreach node ( $cpnames )
    set nnode = `echo $node | sed 's#\..*##'`
    set target = "/l/$nnode/$suffix"
    set targlist = ( $targlist $target )
    echo " work directory: $target/$NUBEAM_WORKPATH "
    if ( ! -d $target/$NUBEAM_WORKPATH ) then
      @ ier++
      echo " ?? compute node work directory missing: $target/$NUBEAM_WORKPATH "
    else
      # delete any possibly left over files in the work directories
      set dlist = `ls $target/$NUBEAM_WORKPATH`
      @ j = 0
      while ( $j < $#dlist )
        @ j++
        if ( $dlist[$j] != "proclist.dat" ) then
          rm -f $target/$NUBEAM_WORKPATH/$dlist[$j]
        endif
      end
    endif
  end
endif  # End not NO_MOVE

if ( $ier > 0 ) then
  exit 1
endif

##### 04/08/09 CLF
if (! $?NO_MOVE) then
  if ( $?CODESYSDIR ) then
    echo "running in `pwd`"
    echo "plist = $plist"
    ls -al $plist
    $CODESYSDIR/pbs/trmpi_mvnb $NPROCS $NUBEAM_WORKPATH $source_dir $suffix $plist  
  else
    trmpi_mvnb $NPROCS $NUBEAM_WORKPATH $source_dir $suffix $plist
  endif
endif

# ------------------------------------------------------------

echo " $orig_wkpath contents: "
ls -l $orig_wkpath

echo " "
echo " ------------------------------------------------- "
echo " *** mpirun mpi_nubeam_comp_exec ... *** "
echo " ------------------------------------------------- "
echo " "

if ( $NPROCS > 1 ) then
#  mpirun --mca btl openib -np $NPROCS `which mpi_nubeam_comp_exec` !for kruskal
  $mpx -np $NPROCS `which mpi_nubeam_comp_exec`
  set rstat = $status
else
  nubeam_comp_exec
  set rstat = $status
endif

echo " "
echo " ------------------------------------------------- "
echo " *** mpi_nubeam_comp_exec exit status: $rstat *** "
echo " ------------------------------------------------- "
echo " "

echo " $orig_wkpath contents: "
ls -l $orig_wkpath

if ( ! $?NO_MOVE) then
  echo " "
  echo " pppl_mpi_test script: loop through compute node work directories..."
  echo " mv all results to: $orig_wkpath"

  @ j = 0
  while ( $j < $#targlist )
    @ j++
    set targdir = $targlist[$j]/$NUBEAM_WORKPATH
    set targdir_src = $targlist[$j]
    echo " "
    echo " ------------------- "
    echo " contents of: $targdir (w/changes)"
    echo " "
    set zlist = `ls $targdir`
    if ( $#zlist == 0 ) then
      echo " (empty) "
    else
      foreach file ( $zlist )
        if ( ! -f $orig_wkpath/$file ) then
          echo " moved: $file "
          mv $targdir/$file $orig_wkpath/$file
        else
          cmp -s $targdir/$file $orig_wkpath/$file
	  set cstat = $status
	  if ( $cstat == 0 ) then
	    # if identical: remove compute server copy
            echo " unchanged: $file"
	    rm -f $targdir/$file
          else
            echo " changed: $file"
            mv $targdir/$file $orig_wkpath/$file
          endif
        endif
      end
    endif

    set zlist = `ls $targdir`
    foreach file ( $zlist ) 
      set ztes1 = `echo $file | sed 's#_birth##'`
      if ( $ztes1 != $file ) then
        echo "finish: move file: $file"
        mv  $targdir/$file $orig_wkpath/$file
      endif
    end

  end
endif


if ( $rstat != 0 ) then
  echo " "
  echo " exit now: status = $rstat"
  echo " "
  echo " ------------------------------------------------- "
  echo " *** error exit *** "
  echo " copy back local files after NUBEAM_COMP_EXEC exit:"

  exit $rstat
else

# ----------------------------------

# in case the value has to be restored, someday...

set save_workpath = $NUBEAM_WORKPATH

# move to home of output files

cd $orig_wkpath

# set value so nubeam_cleanup_files can work...

setenv NUBEAM_WORKPATH $cwd

# cleanup if flag is set

nubeam_cleanup_files

# ----------------------------------

if ( $OUTPUT_STATE != "NONE" ) then
  echo " ====================================== "
  echo " post-processing to create $OUTPUT_STATE "

  update_state -input $INPUT_STATE -output $OUTPUT_STATE \
    -updates state_changes.cdf

  set ustat = $status
  if ( $ustat != 0 ) then
    echo " ?update_state exit status: $ustat "
    exit $ustat
  endif

endif
echo " "
echo " ====================================== "
set zdate = `date`
echo -n "Ended job at date: $zdate"
echo " " 
echo "Done (normal exit)." 
echo " " 
exit 0
endif
