#! /bin/csh -f
#
#  MPI NUBEAM test script at PPPL
#
#  Use qsub to start a PBS (or similar) NxM MPI batch job: mpi_nubeam_comp_exec
#     for job size 
#     see $NUBEAM_WORKPATH/Nnodes and $NUBEAM_WORKPATH/Mpernode
#
#     $1  -- input plasma state (in $NUBEAM_WORKPATH)
#     $2  -- output plasma state (in $NUBEAM_WORKPATH), or "NONE"
#
#  Note: once an init step is taken, N and M are defined and cannot be changed
#  except by another init.  These numbers are saved in files in the sub-
#  directory $NUBEAM_WORKPATH and subsequently retrieved for step, etc.

#  Check Env. variables: 

    @ ier = 0

    if ( $?NUBEAM_WORKPATH != 1 ) then
       echo " ?nubeam_qsub: NUBEAM_WORKPATH undefined."
       @ ier++
    else
       if ( ! -d $NUBEAM_WORKPATH ) then
         echo " nubeam_qsub: no such directory: $NUBEAM_WORKPATH"
         @ ier++
       else
         echo " nubeam_qsub: NUBEAM_WORKPATH = $NUBEAM_WORKPATH"
         set workpath = $NUBEAM_WORKPATH
       endif
    endif

    if ( $?LOG_LEVEL != 1 ) then
       setenv LOG_LEVEL 1
       echo " ?nubeam_qsub: LOG_LEVEL set default = $LOG_LEVEL (warn) "
    else
       echo " nubeam_qsub: LOG_LEVEL = $LOG_LEVEL"
    endif

    if ( $?NUBEAM_CLEANUP != 1 ) then
       setenv NUBEAM_CLEANUP FALSE
    endif

    setenv TSHARE_SIGNAL FALSE
    if ( $?TSHARE_FLAG ) then
       setenv TSHARE_SIGNAL $TSHARE_FLAG
    endif

    if ( $?NUBEAM_ACTION != 1 ) then
       echo " ?nubeam_qsub: NUBEAM_ACTION undefined."
       @ ier++
    else
       echo " nubeam_qsub: NUBEAM_ACTION = $NUBEAM_ACTION"
    endif

    if ( $#argv < 2 ||  $#argv > 4) then
       @ ier++
       echo " syntax: nubeam_qsub <input_state> <output_state>|NONE"
    endif

    set input_state = $1
    set output_state = $2
    if ( $#argv > 2 ) then
       @ iarg = 2
       while ( $iarg < $#argv )
          @ iarg++
          if ("${argv[$iarg]}" == "-q" ) then
             @ iarg++
             set PBSQUE = ${argv[$iarg]}
	     echo pbsque = $PBSQUE
          endif
       end
    endif

    if ( ! -f $NUBEAM_WORKPATH/$input_state ) then
       echo " ?nubeam_qsub: input plasma state not found: $input_state "
       @ ier++
    endif

    if ( $ier == 0 ) then
       if ( ! -f $NUBEAM_WORKPATH/Nnodes ) then
          echo " ?nubeam_qsub: not found: $NUBEAM_WORKPATH/Nnodes "
          @ ier++
       endif
       if ( ! -f $NUBEAM_WORKPATH/Mpernode ) then
          echo " ?nubeam_qsub: not found: $NUBEAM_WORKPATH/Mpernode "
          @ ier++
       endif
    endif

    if ( $ier > 0 ) then
       exit 1
    endif

    if ( $?NUBEAM_REPEAT_COUNT != 1 ) then
        setenv NUBEAM_REPEAT_COUNT 1
    endif
    echo " nubeam_qsub: NUBEAM_REPEAT_COUNT = $NUBEAM_REPEAT_COUNT"

    if ( $?NUBEAM_POSTPROC != 1 ) then
        setenv NUBEAM_POSTPROC NONE
    endif
    echo " nubeam_qsub: NUBEAM_POSTPROC = $NUBEAM_POSTPROC"

    if ( $?FRANTIC_ACTION != 1 ) then
        setenv FRANTIC_ACTION NONE
    endif
    echo " nubeam_qsub: FRANTIC_ACTION = $FRANTIC_ACTION"

    set nnodes = `cat $workpath/Nnodes`
    set mnodes = `cat $workpath/Mpernode`

    echo " N -- Number of nodes: $nnodes "
    echo " M -- Number of processes per node: $mnodes "
    echo " "

    set jobname = nubeam_${nnodes}x${mnodes}

    # OK...

    @ ncpu = ( $nnodes * $mnodes ) 

    if ( $?NUBEAM_MEM_LIMIT ) then
      set mem_limit = $NUBEAM_MEM_LIMIT
    else
      set mem_limit = "1900mb"
      echo " %nubeam_qsub memory default, no NUBEAM_MEM_LIMIT, $mem_limit per cpu"
      if ( $ncpu > 1 ) then
        set mem_limit = `calc_memtot $mem_limit $nnodes $mnodes`
        if ( $status ) then
          echo " ?trqsub: error in: calc_memtot $mem_limit $nnodes $mnodes"
          exit 1
        endif
      endif
    endif

    if ( $?NERSC ) then
       set EV = "-l walltime=30:00 -V"
    else
       set EV = " "
    endif
    if ( $ncpu == 1 ) then
      pbs_mpi_qsub pppl_mpi_nubeam $nnodes $mnodes -N $jobname -q sque \
         -l "mem=$mem_limit" $EV \
         -v "NUBEAM_ACTION=$NUBEAM_ACTION,NUBEAM_WORKPATH=$NUBEAM_WORKPATH,NUBEAM_REPEAT_COUNT=$NUBEAM_REPEAT_COUNT,NUBEAM_POSTPROC=$NUBEAM_POSTPROC,FRANTIC_ACTION=$FRANTIC_ACTION,INPUT_STATE=$input_state,OUTPUT_STATE=$output_state,LOG_LEVEL=$LOG_LEVEL,NUBEAM_CLEANUP=$NUBEAM_CLEANUP,TSHARE_SIGNAL=$TSHARE_SIGNAL"

    else
      if ($?PBSQUE) then
         pbs_mpi_qsub pppl_mpi_nubeam $nnodes $mnodes -N $jobname \
          -q $PBSQUE \
          -l "mem=$mem_limit" $EV \
          -v "NUBEAM_ACTION=$NUBEAM_ACTION,NUBEAM_WORKPATH=$NUBEAM_WORKPATH,NUBEAM_REPEAT_COUNT=$NUBEAM_REPEAT_COUNT,NUBEAM_POSTPROC=$NUBEAM_POSTPROC,FRANTIC_ACTION=$FRANTIC_ACTION,INPUT_STATE=$input_state,OUTPUT_STATE=$output_state,LOG_LEVEL=$LOG_LEVEL,NUBEAM_CLEANUP=$NUBEAM_CLEANUP,TSHARE_SIGNAL=$TSHARE_SIGNAL"
       else
         pbs_mpi_qsub pppl_mpi_nubeam $nnodes $mnodes -N $jobname \
          -l "mem=$mem_limit" $EV \
          -v "NUBEAM_ACTION=$NUBEAM_ACTION,NUBEAM_WORKPATH=$NUBEAM_WORKPATH,NUBEAM_REPEAT_COUNT=$NUBEAM_REPEAT_COUNT,NUBEAM_POSTPROC=$NUBEAM_POSTPROC,FRANTIC_ACTION=$FRANTIC_ACTION,INPUT_STATE=$input_state,OUTPUT_STATE=$output_state,LOG_LEVEL=$LOG_LEVEL,NUBEAM_CLEANUP=$NUBEAM_CLEANUP,TSHARE_SIGNAL=$TSHARE_SIGNAL"
       endif
    endif
