#! /bin/csh -f
#
#  Use qsub to start an MPI test job
#
#     $1  -- job prefix -- will look for $CODESYSDIR/pbs/$1_NxM.tcsh
#     $2  -- N = #nodes: 1 <= N <= 64
#     $3  -- M = #processes per node: 1,2,4,8,16,24,32,48,64
#
#     $4 and up -- passed as arguments to "qsub"...
#
#     total processes: N*M

  @ ier = 0
  if ( $#argv < 3 ) then
    @ ier++
  else
    set jpre = $1

    @ ifound = 0
    if ( $?CODESYSDIR ) then
        set PBSPATH = $CODESYSDIR/pbs
    else if ( $?TRANSP_LOCATION ) then
        set PBSPATH = $TRANSP_LOCATION
    endif
    if ( -f $PBSPATH/${jpre}_NxM.tcsh ) then
      @ ifound++
      set ifile = $PBSPATH/${jpre}_NxM.tcsh
    endif

    if ( $ifound == 0 ) then
      @ ier++
      echo " PBS file to go with $jpre -- not found."
    else
      echo " PBS template file: $ifile "
    endif

    set test = `@ Nnodes = $2`
    if ( $status != 0 ) then
      @ ier++
      echo " N is not a number: $2"
    else
      @ Nnodes = $2
      echo " N (#nodes): $Nnodes"
      if ( $Nnodes < 1 || $Nnodes > 3200 ) then
        @ ier++
        echo " ...out of range 1:3200"
      endif
    endif

    set test = `@ Mppn = $3`
    if ( $status != 0 ) then
      @ ier++
      echo " M is not a number: $3"
    else
      @ Mppn = $3
      echo " M (#processors per node): $Mppn"
      if ( $Mppn != 1 && $Mppn != 2 && $Mppn != 4 && $Mppn != 8  && $Mppn != 16 && \
           $Mppn != 32 && $Mppn != 64 && $Mppn != 24 && $Mppn != 48 ) then
        @ ier++
        echo " ...not in set of allowed values {1,2,4,8,16,24,32,48,64}"
      endif
    endif

  endif

  if ( $ier > 0 ) then
    echo " ?syntax: pbs_mpi_qsub <pbs-script-prefix> N M [qsub_args...]"
    echo "  script looks for $CODESYSDIR/pbs/<pbs-script-prefix>_NxM.tcsh "
    echo "  replaces PBS directive NxM with nodes=N:ppn=M and submits job"
    echo "  with arguments [qsub_args...] if specified."
    exit(1)
  endif

  set qargs = ()

  @ iargct = 3
  while ( $iargct < $#argv )
    @ iargct++
    if ( "$argv[$iargct]" == "-runid" ) then
       @ iargct++
       set runid = $argv[$iargct]
    else
       set qargs = ( $qargs $argv[$iargct] )
    endif
  end

  echo " "
  echo "OK:"
  echo "cwd: $cwd"

  set pidi = `get_a_pid`
  set target = ${jpre}_${Nnodes}x${Mppn}_${pidi}.tcsh
  if ( $#qargs == 0 ) then
    echo "(no qsub optional args)"
  endif

  set nnxmm = ${Nnodes}x${Mppn}
  @ Ntot = $Nnodes * $Mppn
  if ($?NERSC) then
    set target=`echo $target | sed s/pppl/${NERSC_HOST}/`
    if ( $NERSC_HOST == "carver" ) then
       if ($Mppn == 1 ) then
         set tostr = "nodes=${Nnodes}"
       else
         set tostr = "nodes=${Nnodes}:ppn=${Mppn}"
       endif
       cat $ifile | sed -e "/-l mppnppn/d" -e "/-l mppwidth/d" -e s"#NxM#$tostr#" -e s"#_nnxmm#$nnxmm#" > foo 
    else
      cat $ifile | sed  -e "/-l NxM/d" -e "s#mppwidth=N#mppwidth=${Ntot}#" \
        -e "s#mppnppn=M#mppnppn=${Mppn}#" > foo 
    endif
# if pppl_mpi_nubeam_NxM set walltime
    set test_nubeam = `echo $jpre | sed 's#mpi_nubeam##'`
    if ( $test_nubeam != $jpre ) then
    cat foo | sed -e 's#walltime=.*#walltime=00:30:00#' -e 's#_nnxmm#$nnxmm#' > foo1
    else
       cat foo | sed -e 's#_nnxmm#$nnxmm#' > foo1
    endif    
    cat foo1 | sed -e 's#$NPROCS#'"${Ntot}"'#' -e s"# -np# -n#g" -e 's/###NERSC/#PBS/' > $target
    rm -f foo foo1
  else
    if ($Mppn == 1 ) then
      set tostr = "nodes=${Nnodes}"
    else
      set tostr = "nodes=${Nnodes}:ppn=${Mppn}"
    endif
    cat $ifile | sed s"#NxM#$tostr#" | sed s"#_nnxmm#$nnxmm#" > $target 
  endif

# check for different number of nbi or toric
  if ( $?runid ) then
    set znbi=`grep -i '^\ *nbi_pserve\ *=.*' ${runid}TR.DAT |sed 's/.*= *//' |cut -d! -f1`
    set ztoric=`grep -i '^\ *ntoric_pserve\ *=.*' ${runid}TR.DAT |sed 's/.*= *//' |cut -d! -f1`
    set zptr=`grep -i '^\ *nptr_pserve\ *=.*' ${runid}TR.DAT |sed 's/.*= *//' |cut -d! -f1`
  else
    set znbi=0
    set ztoric=0
    set zptr=0
  endif

  if ( "$znbi" == 1) then
     if ( $?PMQ_NBI_NPROCS ) then
        set Nnbi = $PMQ_NBI_NPROCS
     else 
        echo " " 
        echo -n "  To use MPI Group Communicator for NUBEAM, enter nbi_nprocs: "
        set Nnbi = $<
     endif
     if ( "$Nnbi" ) then
        cat $target | sed s"/###setenv NBI_NPROCS Nnbi/setenv NBI_NPROCS $Nnbi/" > foo
        rm $target
        mv foo $target
     endif
  endif
  if ( "$ztoric" == 1) then
     if ( $?PMQ_TOR_NPROCS ) then
        set Ntor = $PMQ_TOR_NPROCS 
     else
        echo " "  
        echo -n "  To use MPI Group Communicator for TORIC, enter toric_nprocs: "
        set Ntor = $<
     endif
     if ( "$Ntor" ) then
        cat $target | sed s"/###setenv TOR_NPROCS Ntor/setenv TOR_NPROCS $Ntor/" > foo
        rm $target
        mv foo $target
     endif
  endif
  if ( "$zptr" == 1) then
     if ( $?PMQ_PTR_NPROCS ) then
        set Nptr = $PMQ_PTR_NPROCS
     else
        echo " " 
        echo -n "  To use MPI Group Communicator for PTRANSP, enter ptr_nprocs: "
        set Nptr = $<
     endif
     if ( "$Nptr" ) then
        cat $target | sed s"/###setenv PTR_NPROCS Nptr/setenv PTR_NPROCS $Nptr/" > foo
        rm $target
        mv foo $target
     endif
  endif


# if tshare gfortran, uncomment SWAPGCC
  if ( $?SWAPGCC ) then
     cat $target | sed s"/###SWAPGCC//" > foo
     rm $target
     mv foo $target
  endif 

  if ( $?PMQ_ENV_BASH ) then
     set btarget=$target
     set ctarget=${jpre}_${Nnodes}x${Mppn}_${pidi}_sub.tcsh
     set chome=`pwd`
     mv $target $ctarget
     chmod +x $ctarget

     echo "#! /bin/bash --login" >$btarget
     grep '^#PBS' $ctarget >>$btarget
     echo  ' ' >>$btarget
     echo  'if [ -f $HOME/.transprc_bash ]; then' >>$btarget
     echo  '    if [ -n "$INIT_TRANSP_DONE" ]; then' >>$btarget
     echo  '      unset INIT_TRANSP_DONE' >>$btarget
     echo  '    fi' >>$btarget
     echo  "    if [ -f $HOME/$PMQ_ENV_BASH ]; then" >>$btarget
     echo  "      echo 'loading $HOME/$PMQ_ENV_BASH from $btarget'" >>$btarget
     echo  "      . $HOME/$PMQ_ENV_BASH" >>$btarget
     echo  '    fi' >>$btarget
     echo  "    echo 'loading $HOME/.transprc_bash from $btarget'" >>$btarget
     echo  '    . $HOME/.transprc_bash' >>$btarget
     echo  'fi' >>$btarget
     echo  "echo 'executing $chome/$ctarget'" >>$btarget
     echo  "$chome/$ctarget" >>$btarget
     echo  'exit $?' >>$btarget
  endif

  echo " "
  echo "pbs_mpi_qsub: "
  if ( $?PBS_DEBUG ) then
     echo "PBS_DEBUG"
     echo "  pbs script is in $target"
     echo "  to submit type:"
     echo "  qsub $qargs $target"
     echo " "
     exit
  endif

# 07/03/2013 CLF: memory was already handled in trqsub
#  if ( $?TRQSUB_MEM_LIMIT ) then
#      set mem_limit = $TRQSUB_MEM_LIMIT
#  else   
#      set mem_limit = "1900mb"
#  endif
#  set mem_limit = `calc_memtot $mem_limit $Nnodes $Mppn`
#  qsub $qargs -l mem=$mem_limit $target

  echo "qsub $qargs $target"
  echo " "
  
  qsub $qargs $target

  set qstat = $status

  echo " "
  echo "pbs_mpi_qsub: done, status = $qstat "
  echo " "

  exit $qstat
