#! /bin/csh -f
#
#  csh -f nbx_share_slurm.csh $argv 
#  ...run serial or MPI nubeam_comp_exec
#
#  command line arguments:
#
#     init_only -- stop after INIT (e.g. for debugging)
#     restart   -- start from existing state files
#     sbatch -- run as SLURM job (MPI if #procs > 1)
#             [ -p <slurm queue name> ]
#     no_sbatch -- do NOT run as SLURM job, run under current process
#             (may want this to run an MPI job directly from an interactive
#             parallel machine).
#     hold -- hold RNG seed fixed (instead of setting from sys clock)
#             (this means "nseed" in the init namelist is used).
#     totalview -- invoke totalview debugger when launching nubeam_comp_exec
#     cleanup -- clean up temporary files when done
#             (this prevents restartability and so is not the default).
#     no_fran -- do NOT run FRANTIC
#
#  command line arguments with values:
#     -ncpu <numprocs>  --  if >1 use MPI with indicated number of processors
#     -nptcls <nptcls>  --  use standard namelist with nptcls & nptclf modified
#     -ndep0 <number>   --  use standard namelist with ndep0 modified
#     -init <namelist>  --  use indicated init namelist
#     -step <namelist>  --  use indicated step namelist
#     -input_state <filename>  --  use indicated input state file
#     -input_list <filename>   --  use LIST of input states (time series)
#     -output_state <filename> --  create indicated output state file
#     -plot_script <filename>  --  copy indicated plot script file
#     -wkpath <directory> -- workpath to use
#     -repeat <step-control>  --  specify number & duration of time steps
#     -postproc <postproc-option>  -- post-processing (default: fbm_write)
#     -log  -- set environmental variable to set the logging level in openLog
#              default  LOGFILE_LEVEL=WARN (or 1)   -> warning
#              LOGFILE_LEVEL=INFO (or 0)  -> info
#              LOGFILE_LEVEL=WARN (or 1)  -> warning
#              LOGFILE_LEVEL=ERR  (or 2)  -> error
#
#  the LOG_LEVEL environment variable, if set, can change the default value:
#    LOG_LEVEL = 0 --> default is INFO
#    LOG_LEVEL = 1 --> default is WARN
#    LOG_LEVEL = 2 --> default is ERR
#    
#  example:
#    csh -f nbx_share_slurm.csh -ncpu 4 -input_state tftr_input_state.cdf \
#                                 -output_state tftr_output_state.cdf
#
#  "reasonable" defaults provided for all options
#-------------------------------
#  first check that required programs exist in $PATH...

  date
  echo " "
  echo " ---> nbx_share_slurm.csh: check environment & files... "

  date > ./date.tmp
  set dstat = $status
  if ( $dstat != 0 ) then
    echo " ?? no write access to current working directory: $cwd "
    exit 1
  else
    rm -f ./date.tmp
  endif

  set flist = ( nubeam_init_files.dat nubeam_step_files.dat \
                nubeam_serv_files.dat set_nptcls.csh \
                nubeam_init_example.dat nubeam_step_example.dat )

  @ iexpect = $#flist
  @ ifound = 0
  set SLURMPATH = ' '
  if ( $?CODESYSDIR ) then
    if ( $CODESYSDIR != "" ) then
      set SLURMPATH = $CODESYSDIR/source/nubeam_comp_exec
    endif
  endif 
  if ( $SLURMPATH == "" ) then
    if ( $?TRANSP_LOCATION ) then
      set SLURMPATH = $TRANSP_LOCATION
    endif
  endif
  foreach file ( $flist )
    set cfile = $SLURMPATH/$file
    if ( -f $cfile ) then
      if ( ! -f $file ) then
	echo " nbx_share: copy (from PATH): $SLURMPATH "
        cp $cfile .
        @ ifound++
      else
        cmp -s $file $cfile
        if ( $status == 0 ) then
          echo " nbx_share: $file matches $SLURMPATH copy... "
          @ ifound++
        else
          echo " nbx_share: replace (from $SLURMPATH): $file "
          cp $cfile .
          @ ifound++
        endif
      endif
    else if ( -f $file ) then
      echo " nbx_share: could not check vs. CODESYSDIR or NTCC_ROOT: $file "
      @ ifound++
    else
      echo " nbx_share: *** not found after search: $file "
    endif
  end

  if ( $ifound != $iexpect ) then
    echo " ?? nbx_share_slurm.csh -- failed to locate one or more of: "
    echo "       $flist "
    echo " => conclusion: environment to support this script is not available."
    exit 1
  endif

  echo " "
  echo " ---> nbx_share_slurm.csh: parse arguments "
  echo '      Note: @: Expression Syntax means: integer variable parse fails.'

  set init_only = "FALSE"
  set restart = "FALSE"
  set cleanup = "FALSE"
  @ numprocs = 1
  @ nptcls = 0
  @ ndep0 = 0
  set init_namelist = "DEFAULT"
  set step_namelist = "DEFAULT"
  set input_state = "NONE"
  set input_list = "NONE"
  set output_state = "NONE"
  set plot_script = "NONE"
  set wkpath = "NONE"
  set repeat = "1"
  set sbatch = "FALSE"
  set rng_hold = "FALSE"
  set fran = "TRUE"
  set postproc = "fbm_write"

  set use_totalview = "FALSE"
  set ttv = " "

  set logfile_level = "WARN"
  set logfile_level_num = 1

  if ( $?LOG_LEVEL ) then
    if ( "$LOG_LEVEL" == "0" ) then
      set logfile_level = "INFO"
      set logfile_level_num = 0
    else if ( "$LOG_LEVEL" == "1" ) then
      set logfile_level = "WARN"
      set logfile_level_num = 1
    else if ( "$LOG_LEVEL" == "2" ) then
      set logfile_level = "ERR"
      set logfile_level_num = 2
    endif
  endif

  @ ier = 0
  @ inamel = 0
  @ iarg = 0
  while ( $iarg < $#argv )
     @ iarg++
     if ( "${argv[$iarg]}" == "-ncpu" ) then
	@ iarg++
	@ numprocs = ${argv[$iarg]}
        if ( $numprocs > 1 ) then
          echo "nbx_share_slurm.csh: MPI mode of execution"
        endif
	echo "               numprocs = $numprocs"

     else if ( "${argv[$iarg]}" == "-nptcls" ) then
	@ iarg++
	@ nptcls = ${argv[$iarg]}
	echo "nbx_share_slurm.csh: NPTCLS & NPTCLF = $nptcls "

     else if ( "${argv[$iarg]}" == "-ndep0" ) then
	@ iarg++
	@ ndep0 = ${argv[$iarg]}
	echo "nbx_share_slurm.csh: NDEP0 = $ndep0 "

     else if ( "${argv[$iarg]}" == "-init" ) then
        @ iarg++ 
        set init_namelist = ${argv[$iarg]}
        if ( ! -f $init_namelist ) then
          @ ier++
          echo " init_namelist not found: $init_namelist "
        else
          echo " init_namelist: $init_namelist "
          @ inamel++
        endif

     else if ( "${argv[$iarg]}" == "-step" ) then
        @ iarg++ 
        set step_namelist = ${argv[$iarg]}
        if ( ! -f $step_namelist ) then
          @ ier++
          echo " step_namelist not found: $step_namelist "
        else
          echo " step_namelist: $step_namelist "
          @ inamel++
        endif

     else if ( "${argv[$iarg]}" == "-input_state" ) then
        @ iarg++ 
        set input_state = ${argv[$iarg]}
        if ( ! -f $input_state ) then
          @ ier++
          echo " input_state not found: $input_state "
        else
          echo " input_state: $input_state "
        endif

     else if ( "${argv[$iarg]}" == "-input_list" ) then
        @ iarg++ 
        set input_list = ${argv[$iarg]}
        if ( ! -f $input_list ) then
          @ ier++
          echo " input_list (list of state files) not found: $input_list "
        else
          echo " input_list (list of state files): $input_list "
        endif

     else if ( "${argv[$iarg]}" == "-plot_script" ) then
        @ iarg++ 
        set plot_script = ${argv[$iarg]}
        if ( ! -f $plot_script ) then
          @ ier++
          echo " plot_script not found: $plot_script "
        else
          echo " plot_script: $plot_script "
        endif

     else if ( "${argv[$iarg]}" == "-repeat" ) then
        @ iarg++ 
        set repeat = ${argv[$iarg]}
	echo " repeat (timestep): $repeat "

     else if ( "${argv[$iarg]}" == "-output_state" ) then
        @ iarg++ 
        set output_state = ${argv[$iarg]}
	echo " output_state: $output_state "

     else if ( "${argv[$iarg]}" == "-wkpath" ) then
        @ iarg++ 
        set wkpath = ${argv[$iarg]}
	echo " NUBEAM_WORKPATH: $wkpath "

     else if ( "${argv[$iarg]}" == "-log" ) then
        @ iarg++ 
	set logval = ${argv[$iarg]}
        
        if ( "$logval" == "INFO" || "$logval" == "0" ) then
          set logfile_level = "INFO"
          set logfile_level_num = 0
        endif
        
        if ( "$logval" == "WARN" || "$logval" == "1" ) then
          set logfile_level = "WARN"
          set logfile_level_num = 1
        endif
        
        if ( "$logval" == "ERR" || "$logval" == "2" ) then
          set logfile_level = "ERR"
          set logfile_level_num = 2
        endif
        
	echo "nbx_share_slurm.csh: logfile_level = $logfile_level "

     else if ( "${argv[$iarg]}" == "-postproc" ) then
        @ iarg++ 
        set postproc = ${argv[$iarg]}
	echo " NUBEAM_POSTPROC: $postproc "

     else if ( "${argv[$iarg]}" == "init_only" ) then
        set init_only = "TRUE"
        echo " INIT_ONLY mode requested."

     else if ( "${argv[$iarg]}" == "cleanup" ) then
        set cleanup = "TRUE"
        echo " End of run CLEANUP requested."

     else if ( "${argv[$iarg]}" == "totalview" ) then
        set use_totalview = "TRUE"
	set ttv = "-tv"
        echo " TOTALVIEW requested."

     else if ( "${argv[$iarg]}" == "restart" ) then
        set restart = "TRUE"
        echo " RESTART mode requested."

     else if ( "${argv[$iarg]}" == "sbatch" ) then
        set sbatch = "TRUE"
        @ inext = $iarg + 1
        if ( $inext < $#argv ) then
          if ( "${argv[$inext]}" == "-p" ) then
             @ iarg = $iarg + 2
             set SLURMQUE = ${argv[$iarg]}
          endif
        endif
        if ($?SLURMQUE) then
           echo " SLURM (sbatch -p $SLURMQUE) job requested."
        else
           echo " SLURM (sbatch) job requested."
        endif

     else if ( "${argv[$iarg]}" == "no_sbatch" ) then
        set sbatch = "LOCAL"

     else if ( "${argv[$iarg]}" == "hold" ) then
        set rng_hold = "TRUE"
        echo " hold option OK: RNG seed held to namelist value"

     else if ( "${argv[$iarg]}" == "no_fran" ) then
        set fran = "FALSE"
        echo " NO FRANTIC mode"

     else

        echo " ?? argument $iarg = ${argv[$iarg]} unrecognized. "
	@ ier++

     endif
  end

  if ( $init_only == "TRUE" && $restart == "TRUE" ) then
    echo " ?? nbx_share_slurm.csh: incompatible options: input_only & restart."
    @ ier++
  endif

  if ( $restart == "TRUE" ) then
    if ( ! -d $wkpath ) then
      echo " ?? nbx_share_slurm.csh: restart mode: work directory missing: $wkpath "
      @ ier++
    endif
  endif

  if ( $input_state == "NONE" && $input_list == "NONE" ) then
    echo " ?? nubeam_comp_exec: input_state (or list of states) must be specified."
    @ ier++
  endif

  if ( $input_state != "NONE" && $input_list != "NONE" ) then
    echo " ?? nubeam_comp_exec: can specify an input state or an input list but not both."
    @ ier++
  endif

  if ( $input_list != "NONE" && $restart == "TRUE" ) then
    echo " ?? nbx_share_slurm.csh: incompatible options: input_list & restart."
    @ ier++
  endif

  if ( $wkpath == "NONE" ) then
    echo " ?? nubeam_comp_exec: wkpath must be specified."
    @ ier++
  endif

  if ( $nptcls > 0 && $nptcls < 100 ) then
    echo " ?? nubeam_comp_exec: nptcls < 100 not allowed."
    @ ier++
  endif

  if ( $ndep0 > 0 && $ndep0 < 100 ) then
    echo " ?? nubeam_comp_exec: ndep0 < 100 not allowed."
    @ ier++
  endif


# look for default namelists if necessary

  if ( $init_namelist == "DEFAULT" ) then
    set init_namelist = nubeam_init_example.dat
  endif

  if ( $step_namelist == "DEFAULT" ) then
    set step_namelist = nubeam_step_example.dat
  endif

  if ( $ier > 0 ) then
    echo " ?? nubeam_comp_exec/nbx_share_slurm.csh: see comments for valid args."
    exit 1
  endif

# apply nptcls and/or ndep0

  if ( $nptcls > 0 || $ndep0 > 0 ) then
    set init_prev = $init_namelist
    set step_prev = $step_namelist

    if ( $nptcls > 0 ) then
      set init_namelist = "nubeam_init_${nptcls}.dat"
      set step_namelist = "nubeam_step_${nptcls}.dat"
    else
      set init_namelist = "nubeam_init_${ndep0}.dat"
      set step_namelist = "nubeam_step_${ndep0}.dat"
    endif

    ./set_nptcls.csh $nptcls $ndep0 $init_prev $init_namelist
    set nstata = $status
    ./set_nptcls.csh $nptcls $ndep0 $step_prev $step_namelist
    set nstatb = $status
    if ( $nstata != 0 || $nstatb != 0) then
      echo " ?? nubeam_comp_exec/nbx_share_slurm.csh: set_nptcls error. "
      echo "    (also used to set ndep0) "
      exit 99
    endif
  endif

# check for required executable programs

  set proglist = ( nubeam_comp_exec  cstate  update_state )
  if ( $numprocs > 1 ) then
    set proglist = ( mpi_nubeam_comp_exec $proglist )
  endif

  @ ierr = 0
  foreach prog ( $proglist )
    set test_path = `which $prog`
    if ( $#test_path != 1 ) then
      echo " ? nbx_share_slurm.csh: required program not in PATH: $prog "
      @ ierr++
    else
      echo " %nbx_share_slurm.csh binary path: $test_path "
    endif
  end

  if ( $ierr > 0 ) then
    echo " ... early exit. "
    exit 1
  endif

  if ( ! -f $init_namelist ) then
    echo " ? nbx_share_slurm.csh: init namelist file not found: $init_namelist"
    exit 1
  endif

  if ( ! -f $step_namelist ) then
    echo " ? nbx_share_slurm.csh: step namelist file not found: $step_namelist"
    exit 1
  endif

  #-------------------------------
  # remove old work directory; make fresh copy

  if ( $restart != "TRUE" ) then
    echo " "
    echo " ---> nbx_share_slurm.csh: make work directory; copy files. "

    if ( -d $wkpath ) then
      rm -r -f $wkpath
    endif

    mkdir $wkpath
  else
    echo " "
    echo " ---> nbx_share_slurm.csh: restart mode; copy state file only. "

  endif

  echo "nbx_share_slurm.csh arguments:" >> $wkpath/nbx_share_slurm.args
  echo $argv >> $wkpath/nbx_share_slurm.args

  #-------------------------------
  # copy input data
  # NOTE: nubeam_comp_exec writes "state_changes.cdf" which are applied to
  #       a base state: either the input state, or the last state on the
  #       list, in case a list (time series) of input states is provided.

  if ( $input_state != "NONE" ) then
    echo " ...copy single state: $input_state "
    set base_state = $input_state

    cp $input_state $wkpath
    cat nubeam_init_files.dat | sed "s#my_old_state.cdf#$input_state#" \
                            | sed "s#nubeam_init_example.dat#$init_namelist#" \
        > $wkpath/nubeam_init_files.dat
    cat nubeam_step_files.dat | sed "s#my_old_state.cdf#$input_state#" \
                            | sed "s#nubeam_step_example.dat#$step_namelist#" \
        > $wkpath/nubeam_step_files.dat
    cat nubeam_serv_files.dat | sed "s#my_old_state.cdf#$input_state#" \
                            | sed "s#nubeam_init_example.dat#$init_namelist#" \
                            | sed "s#nubeam_step_example.dat#$step_namelist#" \
        > $wkpath/nubeam_serv_files.dat
  else
    echo " ...copy list of states: $input_list "
    # get list of states -- want the last one for base_state
    set slist = `cat $input_list | awk '{print $2}'`
    set nslist = $#slist
    set base_state = $slist[$nslist]

    # copy list
    cp $input_list $wkpath

    # copy state files; skip awk output from header i.e. start @ ict = 1, not 0
    @ ict = 1
    while ( $ict < $nslist ) 
      @ ict++
      cp $slist[$ict] $wkpath
    end

    cat nubeam_init_files.dat | sed "s#my_old_state.cdf#LIST:$input_list#" \
                            | sed "s#nubeam_init_example.dat#$init_namelist#" \
        > $wkpath/nubeam_init_files.dat
    cat nubeam_step_files.dat | sed "s#my_old_state.cdf#LIST#" \
                            | sed "s#nubeam_step_example.dat#$step_namelist#" \
        > $wkpath/nubeam_step_files.dat

    # (not for server mode; do not copy nubeam_serv_files.dat)

  endif
    
  cp $init_namelist $wkpath
  cp $step_namelist $wkpath
  cp *rf*data.cdf $wkpath
  cp *fppdata $wkpath
  cp *_rfxqlo_*.cdf $wkpath
  if ( $#status > 0 ) then
    echo "...OK, if no RF kick-off operator will be used"
  endif

  if ( $nptcls > 0 || $ndep0 > 0 ) then
    # cleanup of temporary namelists
    rm -f $init_namelist
    rm -f $step_namelist
  endif

  setenv NUBEAM_WORKPATH $wkpath
  setenv LOG_LEVEL $logfile_level_num

  if ( $init_only == "TRUE" ) then
    echo "# environment variables for testing... " > ./source_me_csh
    echo "setenv NUBEAM_WORKPATH $wkpath" >> ./source_me_csh
    echo "setenv LOG_LEVEL $logfile_level_num" >> ./source_me_csh
  endif

  if ( $cleanup == "TRUE" ) then
    echo "CLEANUP activated..."
    setenv NUBEAM_CLEANUP TRUE
    if ( $plot_script != "NONE" ) then
      set plot_script = "NONE"
      echo "...plot_script deactivated: CLEANUP in effect."
    endif
  else
    setenv NUBEAM_CLEANUP FALSE
  endif

  if ( $plot_script != "NONE" ) then
    cp $plot_script $wkpath
  endif

  #-------------------------------
  # initialize NUBEAM

  if ( $restart != "TRUE" ) then
    echo " "
    echo " ---> nbx_share_slurm.csh: nubeam_comp_exec with NUBEAM_ACTION value INIT "

    if ( $rng_hold == "TRUE" ) then
       setenv NUBEAM_ACTION init_hold
    else
       setenv NUBEAM_ACTION init
    endif

    # if input state does not already specify gas grid, it will be 
    # set for 50 zones

    unsetenv FRANTIC_ACTION
    if ( $fran == "TRUE" ) then
	setenv FRANTIC_INIT 50
    else
    	unsetenv FRANTIC_INIT
    endif

    echo " "
    echo "Serial execution of INIT"
    set source_dir = `pwd`
#    if ( "$use_totalview" == "TRUE" ) then
#      totalview $source_dir/nubeam_comp_exec.dbx
#    else
      nubeam_comp_exec
#    endif
    set nstat = $status

    if ( $nstat != 0 ) then
      echo " "
      echo " ?? nubeam_comp_exec INIT error! "
      exit 1
    endif

  endif

  echo " "
  echo " ---> nbx_share_slurm.csh: nubeam_comp_exec INIT OK "
  if ( $init_only == "TRUE" ) then
    echo " INIT_ONLY exit. "
    echo "   (see source_me.csh) "

    echo "setenv NUBEAM_ACTION step" >> ./source_me_csh
    echo "setenv NUBEAM_REPEAT_COUNT $repeat" >> ./source_me_csh
    echo "setenv NUBEAM_POSTPROC $postproc" >> ./source_me_csh
    if ( $fran == "TRUE" ) then
	echo "setenv FRANTIC_ACTION execute" >> ./source_me_csh
    endif


    exit 0
  endif

  #-------------------------------
  # run for indicated number of steps against target plasma; 
  # write distribution function

  # NOTE: see init_only code above; "source_me" script for setting environment
  #       variables should match...

  setenv NUBEAM_ACTION step
  setenv NUBEAM_REPEAT_COUNT $repeat
  setenv NUBEAM_POSTPROC $postproc

  unsetenv FRANTIC_INIT
  if ( $fran == "TRUE" ) then
     setenv FRANTIC_ACTION execute
  endif

  if ( $?MPI_CMD) then
     set mpx = "$MPI_CMD"
  else
     set mpx = `which mpirun`
  endif

  echo " "
  echo " ***> nbx_share_slurm.csh: nubeam_comp_exec STEP, NUBEAM_REPEAT_COUNT = $repeat "

  if ( $sbatch == "TRUE" ) then
    if ( $restart != "TRUE" ) then
      echo "1" > $NUBEAM_WORKPATH/Nnodes
      echo $numprocs > $NUBEAM_WORKPATH/Mpernode
    else
       if ( ! -f $NUBEAM_WORKPATH/Nnodes ) then
	    echo "1" > $NUBEAM_WORKPATH/Nnodes
	    echo $numprocs > $NUBEAM_WORKPATH/Mpernode
       endif	
    endif
    if ($?SLURMQUE) then
       nubeam_sbatch $base_state $output_state -p $SLURMQUE
    else
       nubeam_sbatch $base_state $output_state
    endif
    set squeue = $status

    echo " job submit status: $squeue"
    exit $squeue

    # post-processing done within batch script

  else if ( $sbatch == "LOCAL" ) then

    if ( $numprocs < 2 ) then    
      echo " "
      echo "Serial execution"

      nubeam_comp_exec
      set nstat = $status

    else

      echo " "
      echo "Parallel execution np = $numprocs "
    if ( "$use_totalview" == "TRUE" ) then
      totalview
    else	
      $mpx -np $numprocs $ttv `which mpi_nubeam_comp_exec`
      set nstat = $status
    endif

    endif

  else if ($numprocs > 1) then
    echo " "
    echo "MPI mode of execution"
    # ------------------------------------------------------------
# Log interesting information
#

    echo "-------------------"
    echo "This is a $SLURM_ENVIRONMENT job"
    echo "This job was submitted to the queue: $SLURM_QUEUE"
    echo "The job's id is: $SLURM_JOBID"
    echo "-------------------"
    echo "The master node of this job is: $SLURM_O_HOST"

# --- the nodes allocated to this job are listed in the
#     file SLURM_NODEFILE

# --- count the number of processors allocated to this run 
# --- $NPROCS is required by mpirun.
    set NPROCS=`wc -l < $SLURM_NODEFILE`

    set NNODES=`uniq $SLURM_NODEFILE | wc -l`

    echo "This job is using $NPROCS CPU(s) on the following $NNODES node(s):"
    echo "-----------------------"
    uniq $SLURM_NODEFILE | sort
    echo "-----------------------"

    set NODE_LIST = `uniq $SLURM_NODEFILE | sort`
    set suffix = `pwd | sed 's#^/local/##'`
    set source_dir = `pwd`
    set orig_wkpath = $source_dir/$NUBEAM_WORKPATH
    echo " "
    echo " get processor list... "
    set plist = $NUBEAM_WORKPATH/proclist.dat
    rm -f $plist

    $mpx -np $NPROCS `which mpi_proclist` $plist

   set cpnums = `cat $plist | awk '{print $1}'`
   set cpnames = `cat $plist | awk '{print $2}' | sed 's#\..*##' | uniq`


   echo " create work directories on each compute server node..."
   set targlist = ()
# 03/06/09 CLF: $NODE_LIST not usable on kruskal, use $cpnames
   foreach node ( $cpnames )
     set nnode = `echo $node | sed 's#\..*##'`
     set target = "/l/$nnode/$suffix"
     run_gmkdir $target/$NUBEAM_WORKPATH
     set targlist = ( $targlist $target )
   end

# check (due to paranoia about NFS...)

   echo " check work directories on each compute server node..."
   @ ier = 0
   set targlist = ()
   foreach node ( $cpnames )
     set nnode = `echo $node | sed 's#\..*##'`
     set target = "/l/$nnode/$suffix"
     set targlist = ( $targlist $target )
     echo " work directory: $target/$NUBEAM_WORKPATH "
     if ( ! -d $target/$NUBEAM_WORKPATH ) then
       @ ier++
       echo " ?? compute node work directory missing: $target/$NUBEAM_WORKPATH "
     else
    # delete any possibly left over files in the work directories
#       set dlist = `ls $target/$NUBEAM_WORKPATH`
#       @ j = 0
#       while ( $j < $#dlist )
#         @ j++
#         rm -f $target/$NUBEAM_WORKPATH/$dlist[$j]
#       end
     endif
   end

   if ( $ier > 0 ) then
     exit 1
   endif

##### 04/08/09 CLF
   set cmd=trmpi_mvnb
   if ( $?CODESYSDIR ) then
     if ( -f $CODESYSDIR/slurm/trmpi_mvnb ) then
         set cmd=$CODESYSDIR/slurm/trmpi_mvnb
     endif
   endif
   $cmd $NPROCS $NUBEAM_WORKPATH $source_dir $suffix $plist  
#####

# ------------------------------------------------------------
    $mpx -np $numprocs $ttv `which mpi_nubeam_comp_exec`
    set nstat = $status

   echo " "
   echo " pppl_mpi_test script: loop through compute node work directories..."
   echo " mv all results to: $orig_wkpath"
       @ j = 0
   while ( $j < $#targlist )
     @ j++
     set targdir = $targlist[$j]/$NUBEAM_WORKPATH
     echo " "
     echo " ------------------- "
     echo " contents of: $targdir (w/changes)"
     echo " "
     set zlist = `ls $targdir`
     set dlist = ( $init_namelist $step_namelist $plot_script)
     if ( $#zlist == 0 ) then
       echo " (empty) "
     else
       foreach file ( $zlist )
         if ( ! -f $orig_wkpath/$file ) then
           echo " moved: $file "
           mv $targdir/$file $orig_wkpath/$file
         else
           cmp -s $targdir/$file $orig_wkpath/$file
	   set cstat = $status
	   if ( $cstat == 0 ) then
	     # if identical: remove compute server copy
             echo " unchanged: $file"
	       @ kk = 0
	       while ( $kk < $#dlist )
		  @ kk++
		  if ( $file == $dlist[$kk])  then
		    echo "delete $file"
		    rm -f $targdir/$file
		  endif
	       end
           else
             echo " changed: $file"
             mv $targdir/$file $orig_wkpath/$file
           endif
         endif
       end
     endif
   end
   echo " "
   echo " ------------------------------------------------- "
   echo " *** mpi_nubeam_comp_exec exit status: $nstat *** "
   echo " ------------------------------------------------- "
   echo " "


 else
    echo " "
    echo "Serial execution"
    if ( "$use_totalview" == "TRUE" ) then
      set source_dir = `pwd`
      totalview $source_dir/nubeam_comp_exec.dbx
    else
      nubeam_comp_exec
    endif
    set nstat = $status

 endif
  # in non-batch mode some post-processing is done here...

  if ( $nstat != 0 ) then
    echo " "
    echo " ?? nubeam_comp_exec STEP error! "
    exit 1
  else

  echo " "
  date
  echo " ---> nbx_share_slurm.csh: nubeam_comp_exec STEP OK "

  # implement option to delete temporary files

  nubeam_cleanup_files

  # generate updated full state if requested

  if ( $output_state != "NONE" ) then
     update_state -input $NUBEAM_WORKPATH/$base_state \
                  -output $NUBEAM_WORKPATH/$output_state \
                  -updates $NUBEAM_WORKPATH/state_changes.cdf

     set ustat = $status
     if ( $ustat != 0 ) then
       echo " "
       echo "    attempt to merge state modifications failed... "
       echo " ?? nbx_share_slurm.csh error in update_state! "
       exit 1
     endif

     echo " "
     echo " --> nbx_share_slurm.csh: normal exit. "
     echo "     Plasma State changes in $NUBEAM_WORKPATH/state_changes.cdf"
     if ( $output_state != "NONE" ) then
       echo "     New Plasma State in $NUBEAM_WORKPATH/$output_state"
     endif
     echo " "

     exit 0
  endif
