#!/bin/tcsh -f
#
# DMC -- modified for NxM MPI run -- fast test program
#        mpi_portlib_env_test -- environment sharing
#
# Assumption: ~/.cshrc sets USER_ENV_RESET_DISABLED to prevent system
# reset of PATH and module loads on shell startup.  Note use of "-f" switch.
#
# Check whether ~/.login needs to be executed.  This is done by seeing 
# if a known command is defined.
#
#============================================================================
# (comments from the PPPL system PBS script template...)
#
# --- this is a sample script for running a program on the beowulf
#     cluster. You modify this script for your needs, then use the
#     'qsub' command to submit it to the cluster's job scheduler.
#
#     The job scheduler is the Portable Batch System (PBS). It
#     uses directives in this file to determine where to run
#     the job, how many hosts to run it upon, where to put output
#     files, etc.
#
# remarks: a line beginning with # is a comment
#          a line beginning with #PBS is a pbs directive
#          assume (upper/lower) case to be sensitive
#
# ------------------------------------------------------------
# Basic Information
#
# --- the name of your job
#PBS -N testjob_nnxmm
 
# --- Mail options
#     mail on execution("b"), termination ("e"), or interruption ("a")
#PBS -m aeb
 
# ------------------------------------------------------------
# Job execution details
 
# --- run the job on 'n' nodes with 'm' processors per node (ppn)
#     (script to replace value, below, with nodes=<n>:ppn=<m>)
#PBS -l NxM

# --- specify the period of time the job will run in the format
#     hh:mm:ss (ex. 72 hours is 72:00:00)
#PBS -l walltime=1:00:00
####PBS -l walltime=50:00

# --- submit this job to a special queue. If you do not specify a
#     special queue, the job will run in the default queue
#     There are three main queues at PPPL:
#     kestrel - the default queue used for multi node jobs
#     kite - a special queue of nodes connected with Infiniband
#     sque - nodes used for single CPU jobs (serial jobs)
####now defined in mpi_test_qsub and passed as argument
####PBS -q kestrel

# --- do not rerun this job if it fails
#PBS -r n

# ------------------------------------------------------------
# When a batch job starts execution, a number of environment
# variables are predefined, which include:
#
#      Variables defined on the execution host.
#      Variables exported from the submission host with
#                -v (selected variables) and -V (all variables).
#      Variables defined by PBS.
#
# The following reflect the environment where the user ran qsub:
# PBS_O_HOST    The host where you ran the qsub command.
# PBS_O_LOGNAME Your user ID where you ran qsub.
# PBS_O_HOME    Your home directory where you ran qsub.
# PBS_O_WORKDIR The working directory where you ran qsub.
#
# These reflect the environment where the job is executing:
# PBS_ENVIRONMENT is set by PBS to "PBS_BATCH" if the job is a batch job, or
#                 to "PBS_INTERACTIVE" if the job is an interactive job.
# PBS_O_QUEUE   The original queue you submitted to.
# PBS_QUEUE     The queue the job is executing from.
# PBS_JOBID     The job's PBS identifier.
# PBS_JOBNAME   The job's name.

# --- define standard output and error files for the job
#     default is <jobname>.o<jobid> and <jobname>.e<jobid>
##PBS -o test.out
##PBS -e test.err
# --- or combine both output and error streams into one file with the
#     job name and id as the default file name
#PBS -j oe

# ------------------------------------------------------------
# CHECK ENVIRONMENT

echo " +++++++++++++++++++++++++++++++++++ "
echo " printenv | sort "
printenv | sort
echo " "
echo " printenv.dat written after cd to work directory..."
echo " "

echo " +++++++++++++++++++++++++++++++++++ "
set zdate = `date`
echo -n "Starting job at date: $zdate"

set runtr_chk = `which run_gmkdir`

@ ier = 0
if ( $#runtr_chk != 1 ) then
  @ ier++
else
  if ( ! -f $runtr_chk ) then
    @ ier++
  endif
endif

if ( $ier > 0 ) then
  echo " "
  echo " run_gmkdir: not found -- so, source ~/.login "

  source ~/.login
endif

echo " "
echo " which run_gmkdir: "
which run_gmkdir
echo " "

# ------------------------------------------------------------
# Log interesting information
#

echo "-------------------"
echo "This is a $PBS_ENVIRONMENT job"
echo "This job was submitted to the queue: $PBS_QUEUE"
echo "The job's id is: $PBS_JOBID"
echo "-------------------"
echo "The master node of this job is: $PBS_O_HOST"

# --- the nodes allocated to this job are listed in the
#     file PBS_NODEFILE

# --- count the number of processors allocated to this run 
# --- $NPROCS is required by mpirun.
set NPROCS=`wc -l < $PBS_NODEFILE`

set NNODES=`uniq $PBS_NODEFILE | wc -l`

echo "This job is using $NPROCS CPU(s) on the following $NNODES node(s):"
echo "-----------------------"
uniq $PBS_NODEFILE | sort
echo "-----------------------"

set NODE_LIST = `uniq $PBS_NODEFILE | sort`

# ------------------------------------------------------------
# Setup execution variables
#
# Setup working directory & data source directory
#
# --- PBS_O_WORKDIR is the working directory from
#     which this job was submitted using 'qsub'
#
echo "The job origination directory is $PBS_O_WORKDIR"

set test = `echo $PBS_O_WORKDIR | sed 's#^/local/##'`
if ( "$test" != "$PBS_O_WORKDIR" ) then

  run_gmkdir $PBS_O_WORKDIR
  cd $PBS_O_WORKDIR

#  determine output directory: if /local, replace with /l/<hostname>

  set master_cwd = `pwd`
  set master_host = `echo $PBS_O_HOST | sed 's#\..*##'`
  if ( $test != $master_cwd ) then
      set master_cwd = "/l/$master_host/$test"
  endif

  echo "The mpi job will start here and all output files will be here:"
  set targlist = ( $master_cwd )
  cd $master_cwd
  pwd

else

  cd $PBS_O_WORKDIR
  echo "The job starts in the origination directory and writes all files here:"
  set targlist = ( $cwd )
  pwd

endif

printenv | sort > printenv.dat

set script = `which run_mpi_portlib_env_test`
if ( $#script > 1 ) then
   echo " script not found: run_mpi_portlib_env_test "
   exit 1
endif

$script $NPROCS $targlist

exit $status

