subroutine nbi_avg_bsource(nbs_finish,nl_wrkpath)
!subroutine nbi_avg_bsource(nbs_finish,il_std)
! working with files related to beam ion birth source
! concatenate all  files on each cpu 
! then move to myid = 0 and concatenate files from all cpu
!
!  modifications
!    09may2014   jim.conboy@ccfe.ac.uk
!
!                implement $nbi_workdir ; if set, the value of this variable if prefixed to 
!                the workpath for merged files (only). this enables the merge to be performed 
!                on a shared device, if the different process cwds cant be accessed by other 
!                processes.
!                [ operation when $nbi_workdir is not set has not been tested - 9may14, jec ]
!
!                fix typos in initial infolog messages 
!                infolog message for file deletion
!
!-------------------------------------------------------------------------
!
  use nbi_com,only : workpath,runid,n_outtim,nbs_step,fp
  use nbi_mpi_com
  use logmod
  use netcdf
  use mpi_proc_data

  implicit none

  integer, intent(in) :: nbs_finish
  logical, intent(in) :: nl_wrkpath ! if .true. keep files in workpath directoy (nubeam_comp_exec)
                                    ! if .false. keep files in root directoy (transp)
  character*9 cstep
  character*300 :: suffix_in,suffix_out  ! birth source filenames
                                                    ! files with "suffix_in" need to be concatenated
                                                    ! 
                                                    ! files with "suffix__out" will be created and incorparated
                                                    ! data will be written
  character*300 :: suffix_del !suffix to the file/s to be deleted
  character*300 :: fname_av
  character*300 :: path_myidd0
  character*300, dimension(:), allocatable :: fname
  character*300 :: zcmd,fname_av0
  character(len=256)  :: cpath
  integer,dimension(:),allocatable :: icd_f
  integer :: ierr,icdf,i,ii,mpierr,i_err,ierr1,ier_sum,iersum,ilenc
  logical :: il_conc,il_moved,il_del
  logical :: lshared_wkdisk                         ! .t if files are merged on a shared drive
  integer :: len_swp                                ! length of the shared workpath string

  ! local

  logical  ::  have_group
  integer  ::  grp_comm
  integer  ::  grp_rank
  character(len=*), parameter         :: cshared_work_var  = 'nbi_workdir' &
  &                                     ,cdelim            = '/'
  character(len=256)                  :: cshared_work_path
  character(len=*), parameter         :: cr = 'nbi_avg_bsource'
  !---------------------------------


  ! manage fast ion birth source files
  call enterlog("nbi_avg_bsource") ! all returns through exitlog()

  ! check if in nubeam group (test programs dont use groups)
  call mpi_proc_info('nubeam',have_group,grp_comm,irank=grp_rank)
  if (have_group) then
     if (grp_rank.gt.0) then
        nbi_comm=grp_comm
     ! if rank == 0: nbi_comm was set in tr_output
     ! should not have been called if not a member of group 
     else if (grp_rank.lt.0) then
        call errorlog('nbi_avg_bsource called but not in group nubeam')
        ierr = 1
        return
     endif
  endif

  call nbi_mpi_com_init

  lshared_wkdisk     =  .false.
  cshared_work_path  =  ' '
  call infolog('nbi_avg_bsource nbi_comm after nbi_mpi_com_init = ',nbi_comm)
  if(nbs_finish.eq.0) return
#ifdef __MPI
  !let all cpu know 
  call infolog('nbi_avg_bsource mpi_bcast nbs_step = ',nbs_step )
  call mpi_bcast(nbs_step,1,MPI_INTEGER,0,nbi_comm,mpierr)
  if (mpierr.ne.0) then
     call errorlog('nbi_avg_bsource: nbs_step ierr broadcast  error ')
!     call mpi_finalize(mpierr)
!    call exit(1)
     call bad_exit
  endif
  call infolog('nbi_avg_bsource mpi_bcast  n_outtim = ',n_outtim )
  call mpi_bcast(n_outtim,1,MPI_INTEGER,0,nbi_comm,mpierr)
  if (mpierr.ne.0) then
     call errorlog('nbi_bsource_check: n_outtim ierr broadcast  error ')
!     call mpi_finalize(mpierr)
!     call exit(1)
     call bad_exit
  endif
  if(myidd.eq.0)then
     ilenc=len(trim(adjustl(runid)))
     call infolog('nbi_avg_bsource: bcast runid '//trim(runid))
  else
     ilenc=0
  endif
  call mpi_bcast(ilenc,1,MPI_INTEGER,0,nbi_comm,mpierr)
  call mpi_bcast(runid,ilenc,mpi_character,0,nbi_comm,mpierr)
  if (mpierr.ne.0) then
     call errorlog('nbi_avg_bsource: runid ierr broadcast  error ')
!     call mpi_finalize(mpierr)
!     call exit(1)
      call bad_exit
  endif
  if(myidd.eq.0)then
     ilenc=len(trim(adjustl(workpath)))
     call infolog('nbi_avg_bsource: bcast workpath '//trim(workpath))
  else
     ilenc=0
  endif
  call mpi_bcast(ilenc,1,MPI_INTEGER,0,nbi_comm,mpierr)
  call mpi_bcast(workpath,ilenc,mpi_character,0,nbi_comm,mpierr)
  if (mpierr.ne.0) then
     call errorlog('nbi_avg_bsource: workpath ierr broadcast  error ')
!     call mpi_finalize(mpierr)
!     call exit(1)
    call bad_exit
  endif
!
!     merge to directory on a shared disk, if cshared_work_var is defined
!     nb len_swp  is used later when generating filenames !!
!
  call mpi_sget_env( cshared_work_var, cshared_work_path, ierr )
  len_swp = len_trim(cshared_work_path)
  if( ierr == 0 .and. len_swp .gt. 0 )      then
     lshared_wkdisk = .true.
     if( cshared_work_path(len_swp:len_swp) /= cdelim )  then
        len_swp = len_swp + 1
        cshared_work_path(len_swp:) = cdelim
     endif
     call infolog( cr//' will use '//trim(cshared_work_path) )
     call mkdir( trim(cshared_work_path)//workpath, ierr )
     if( ierr /= 0 )                     then
        call errorlog( cr//' could not create'//trim(cshared_work_path)//workpath )
        call bad_exit
     endif
  else
     cshared_work_path = ' '
     len_swp = 0
  endif
!
#endif
  if(nbs_step.le.0) return
  ! checking if is is a restar then check if file exists
  suffix_in='_birth_cpu'
  suffix_out='_birth'
  ierr=0
  if(myidd.eq.0.) then
     write(cstep,'(i5)') n_outtim+1
        
     ! file  ==  <runid>_birth.cdf<nn>  --  to write data on myid=0
     call nbi_loc_cwd_myidd(workpath,path_myidd0,nl_wrkpath)
     fname_av=trim(path_myidd0)//trim(runid)//trim(suffix_out)//'.cdf'//trim(adjustl(cstep))
     ierr= nf90_open(trim(fname_av),nf90_nowrite,icdf) 
     if(ierr.eq.0) then
        !nothing to do, get ready for next outtim
        ierr= nf90_close(icdf)
        n_outtim=n_outtim+1
     endif
  endif

#ifdef __MPI
  !let all cpu know 
  call infolog('nbi_avg_bsource: bcast ierr (ok if .ne. 0) ',ierr)
  call mpi_bcast(ierr,1,MPI_INTEGER,0,nbi_comm,mpierr)
  if (mpierr.ne.0) then
     call errorlog('nbi_avg_bsource: ierr broadcast error')
!     call mpi_finalize(mpierr)
!     call exit(1)
      call bad_exit
  endif
#endif
  if(ierr.eq.0) then
     call warnlog(' nbi_avg_bsource: file already exists')
     call warnlog(' nbi_avg_bsource: cleanup is taken')
     nbs_step=0 
     !cleanup work dir on all cpu
     suffix_del=trim(runid)//trim(suffix_in)//'*'
     call fwc_delete(trim(workpath),trim(suffix_del),i_err) !delete fiels <runid>_birth_cpu*
     if(myidd.eq.0) then
        il_del=(trim(path_myidd0).ne.trim(workpath))
        if(il_del) then
           !cleanup dir where files are stored on all cpu=0
           call fwc_delete(trim(path_myidd0),trim(suffix_del),i_err) !delete fiels <runid>_birth*
        endif
     endif
     return
  endif
     

  if(allocated(icd_f)) deallocate(icd_f)
  allocate(icd_f(nbs_step))

  if(allocated(fname)) deallocate(fname)
  allocate(fname(nbs_step+1))

! fname(nbs_step+1) is template for fname(1 .. nbs_step)
!
  fname(nbs_step+1) = ' '
  call default_filename_ptcl(workpath,runid,fname(nbs_step+1),suffix_in,myidd,numprocd)
  if( lshared_wkdisk) then
     fname_av = trim(cshared_work_path)//fname(nbs_step+1)
  else
     fname_av =                          fname(nbs_step+1)
  endif
  !
  call infolog( cr, 'merging local files to '//trim(fname_av) )
  !
  ! averaged procedure needed
  ! in case of restar -- check if filew concatenated already 
  ! trying to open file
  ! <runid>_birth_cpu_<myidd>_<numprocd>.cdf
  il_moved=.false.
  il_conc=.false.

  ! checking if there are files to concatenate
  
  i_err=0
  
  do i=1,nbs_step
     write(cstep,'(i5)') i
     fname(i)=trim(fname(nbs_step+1))//'_'//trim(adjustl(cstep))
     ! trying to open file
     ! <runid>_birth_cpu_<myidd>_<numprocd>.cdf_<nn>
     ierr1 = nf90_open(trim(fname(i)),nf90_nowrite,icd_f(i))   
     if(ierr1.ne.0) then
        call warnlog(' nbi_avg_bsource: data missing or error to open file'//trim(fname(i)))
        i_err=i_err+1
     else
        call infolog( cr//' opened '//trim(fname(i)) )
     endif
  enddo
  ier_sum=0
  !check if all files are missing.
  if(i_err.gt.0.and.i_err.ne.nbs_step) then
     ! some files have been missed -- exit
     write(zcmd,'(i4,a,i4)') i_err,' of ',nbs_step
     call errorlog(cr//trim(zcmd)//' files have been missed')
#ifndef __MPI
     call bad_exit("nbi_avg_bsource : data missing or file open failed")
#endif
     ier_sum=1
  elseif(i_err.eq.nbs_step) then !all files are missed or delted due to 
     !fname_av has been moved to myidd=0 directory. 
     il_moved=.true. !let it be in myidd=0 directory. myidd will check.
     il_conc=.true.
  elseif(i_err .eq.0) then
     il_moved=.false.
     il_conc=.false.
  endif

#ifdef __MPI
  iersum=0
  call mpi_allreduce(ier_sum,iersum,1,MPI_INTEGER,MPI_SUM,nbi_comm,mpierr)
  if(iersum.gt.0) then
!     call mpi_finalize(mpierr
     call errorlog("nbi_avg_bsource : data missing or file open failed")
     call bad_exit
  endif
#endif

  i_err=0
  ierr=0
  ier_sum=0
  if(.not.il_conc) then
     !all files are present and opened, need to be concatenate
     !files need to be concatenated here, over all nbs_step, on each cpu
     if(nbs_step.eq.1) then
        i_err=nf90_close(icd_f(1))
        call frename(trim(fname(1)),trim(fname_av),i_err)
        if(i_err.ne.0) then
           call errorlog(' nbi_avg_bsource: error to move file on myidd=',myidd)
           ier_sum=1
           call fdelete(fname(1),i_err)
#ifndef __MPI
           call bad_exit("nbi_avg_bsource: error to move file")
#endif
        endif
     else
        !if file "fname_av" exists it will be overwritten.
        call nbi_bsource_conct(nbs_step,icd_f,trim(fname_av),ierr)
        
        if(ierr.ne.0) then
           ier_sum=1
           call errorlog(' nbi_avg_bsource: error to concatenate files on myidd=',myidd)
           do ii=1,nbs_step
              !close  opened files before exit
              ierr=nf90_close(icd_f(ii))
           enddo
#ifdef __MPI
#else
           call bad_exit("nbi_avg_bsource: unable to concatenate files on cpu")
#endif
        endif
     endif
     
#ifdef __MPI
     iersum=0
     call mpi_allreduce(ier_sum,iersum,1,MPI_INTEGER,MPI_SUM,nbi_comm,mpierr)
     if(iersum.gt.0) then
!        call mpi_finalize(mpierr)
        call errorlog("nbi_avg_bsource : nbi_avg_bsource: error to move file or unable to concatenate")
        call bad_exit
     endif
#endif
     !cleanup : close all files and delete them
     do ii=1,nbs_step
        ierr=nf90_close(icd_f(ii)) 
        call fdelete(fname(ii),ierr)
     enddo
     il_conc=.true.
     
     if(allocated(icd_f)) deallocate(icd_f)
     if(allocated(fname)) deallocate(fname)
  endif
  
  ! define "host" path
  call infolog('nbi_avg_bsource: call nbi_loc_cwd ')
  call nbi_loc_cwd(workpath,path_myidd0,.false.,il_moved) !get and broadcast myidd=0 directory
#ifdef __MPI
  ! move all files to myidd=0 directory
  ierr=0
  if(myidd.gt.0.and.(.not.il_moved)) then
     fname_av0=trim(path_myidd0)//trim(fname_av)
     call frename(trim(fname_av),trim(fname_av0),ierr)
     call infolog('nbi_avg_bsource: moved file '//trim(fname_av))
     call infolog('nbi_avg_bsource:  to '//trim(fname_av0))
     il_moved=.true.
  endif

  ier_sum=0
  if (ierr.ne.0) then
     ier_sum=1
     call errorlog('nbi_avg_bsource: not all files have been moved to myidd=0')
     call errorlog('nbi_avg_bsource: file has not been moved to myidd=0'//trim(fname_av))
  endif
  iersum=0
  call mpi_allreduce(ier_sum,iersum,1,MPI_INTEGER,MPI_SUM,nbi_comm,mpierr)
  if(iersum.gt.0) then
!     call mpi_finalize(mpierr)  ! must call  mpi_abort (done in bad_exit)
     call errorlog("nbi_avg_bsource : myidd=0 cannot find all needed files")
     call bad_exit
  endif

#endif
  
  !all files were moved to cpu=0
#ifdef __MPI
  call mpi_barrier(nbi_comm,mpierr)
#endif
  ier_sum=0
  if(myidd.eq.0) then
     write(cstep,'(i5)') n_outtim+1
     
     ! file  ==  <runid>_birth.cdf<nn>  --  to write data
     if( lshared_wkdisk)    then
        fname_av=trim(cshared_work_path)//trim(workpath)//trim(runid)//trim(suffix_out)//'.cdf'//trim(adjustl(cstep))
     elseif(nl_wrkpath)then
        call nbi_loc_cwd_myidd(workpath,path_myidd0,nl_wrkpath)

!mg        fname_av=trim(path_myidd0)//trim(workpath)//trim(runid)//trim(suffix_out)//'.cdf'//trim(adjustl(cstep))
        fname_av=trim(path_myidd0)//trim(runid)//trim(suffix_out)//'.cdf'//trim(adjustl(cstep))
     else
        fname_av=trim(path_myidd0)//trim(runid)//trim(suffix_out)//'.cdf'//trim(adjustl(cstep))
     endif
     call infolog( cr, 'merge to'//trim(fname_av) )
     if(allocated(icd_f)) deallocate(icd_f)
     allocate(icd_f(numprocd))
     
     if(allocated(fname)) deallocate(fname)
     allocate(fname(numprocd))

     if( lshared_wkdisk)   then
        cpath = trim(cshared_work_path)//workpath
     else
        cpath = workpath
     endif
     do i=1,numprocd
        ! read file/s  ==  <runid>_birth_cpu_<myidd>_<numprocd>.cdf 
        fname(i)=' '
        call default_filename_ptcl(cpath,runid,fname(i),suffix_in,i-1,numprocd)
        ierr = nf90_open(trim(fname(i)),nf90_nowrite,icd_f(i)) 
        if(ierr.ne.0) then
           call errorlog(' nbi_avg_bsource: data missing or error to open file')
           call errorlog(' nbi_avg_bsource: file '//trim(fname(i)))
           do ii=1,i-1
              !close  opened files before exit
              ierr=nf90_close(icd_f(ii))
           enddo
           ier_sum=1
#ifndef __MPI
           call bad_exit("nbi_avg_bsource : myidd=0 data missing or file open failed")
#endif
        endif
     enddo

     i_err=0
     if(numprocd.eq.1) then
        i_err=nf90_close(icd_f(1))
        call frename(trim(fname(1)),trim(fname_av),i_err)
        if(i_err.ne.0) then
           call errorlog(' nbi_avg_bsource: error to move file on myidd=0')
           call fdelete(fname(1),i_err)
           ier_sum=1
#ifdef __MPI
#else
           call bad_exit("nbi_avg_bsource: error to move file")
#endif
        endif
     else
        call nbi_bsource_conct(numprocd,icd_f,trim(fname_av),ierr) ! concatenate files over all cpu
                                                                   ! and write to "fname_av"
                                                                   !(already concatenated over nbs_step)
        if(ierr.ne.0) then
           ier_sum=1
           call errorlog(' nbi_avg_bsource: error to concatenate files on myidd=0')
           do ii=1,numprocd
              !close  opened files before exit
              ierr=nf90_close(icd_f(ii))
           enddo
#ifdef __MPI
#else
           call bad_exit("nbi_avg_bsource: unable to concatenate files on myidd=0")
#endif
        endif
        !cleanup : close all files and delete them
        !only one file is left runid>_birth.cdf<nn> 
        do ii=1,numprocd
           ierr=nf90_close(icd_f(ii)) 
           call fdelete(fname(ii),ierr)
           call infolog( cr,'deleting '//trim(fname(ii)) )
        enddo
        il_conc=.true.
     endif
   
     
     if(allocated(icd_f)) deallocate(icd_f)
     if(allocated(fname)) deallocate(fname)
     ! prepare to next outtim
     !for  cpu>0 no needs to know
     n_outtim=n_outtim+1
     
  endif
#ifdef __MPI
     iersum=0
     call mpi_allreduce(ier_sum,iersum,1,MPI_INTEGER,MPI_SUM,nbi_comm,mpierr)
     if(iersum.gt.0) then
!        call mpi_finalize(mpierr)
        call errorlog("nbi_avg_bsource : see log files")
        call bad_exit
     endif
#endif
  ! all cpu should know
  nbs_step=0 
  call exitlog("nbi_avg_bsource") 
     

end subroutine nbi_avg_bsource
