#! /bin/bash
#PBS -A m876
#PBS -m e
#PBS -j oe
#PBS -l walltime=0:30:00
#PBS -N MPI_ENV_TEST
#PBS -l size=4
###PBS -l nodes=4  
#PBS -q debug
###PBS -N MPI_ENV_TEST_256
###PBS -l size=256  
###PBS -q regular
#PBS -S /bin/bash

NP=4
#NP=256
echo -n "Starting job at date: `date`"
echo "`whoami` running on `hostname`"

echo "-------------------"
echo "This is a $PBS_ENVIRONMENT job"
echo "This job was submitted to the queue: $PBS_QUEUE"
echo "The job's id is: $PBS_JOBID"
echo "-------------------"
echo "The master node of this job is: $PBS_O_HOST"

# $PBS_NODEFILE contains one node, which is NOT one of the mpi nodes!!!
# NPROCS=`wc -l < $PBS_NODEFILE`
# NNODES=`uniq $PBS_NODEFILE | wc -l`

echo " 
  cat PBS_NODEFILE
" 
cat $PBS_NODEFILE
echo " "

echo "The job origination directory is $PBS_O_WORKDIR"
cd $PBS_O_WORKDIR
echo " "
echo PATH = $PATH
echo " "

rm -f proclist.dat
aprun -n $NP `which mpi_proclist` proclist.dat

echo " "
echo " proclist.dat: "
cat proclist.dat
NPROCS=`cat proclist.dat | wc -l`
echo "
NPROCS = $NPROCS
"

NNODES=`cat proclist.dat | awk '{ print $2 }' | uniq | wc -l`
NODES=`cat proclist.dat | awk '{ print $2 }' | uniq | sort`

echo "This job is using $NPROCS CPU(s) on the following $NNODES node(s):"
echo "-----------------------"
#uniq $PBS_NODEFILE | sort
echo $NODES 
echo "-----------------------"

aprun -n $NPROCS `which mpi_portlib_env_test`  this is a test

rstat=$?
 echo " ------------------------------------------------- "
echo " *** exit status: $rstat (1 expected) *** "
echo "     (program tests error handling) "
echo " ------------------------------------------------- "


