#!/bin/bash

# ICON
#
# ----------------------------------------------------------------------------
# Copyright (C) 2004-2024, DWD, MPI-M, DKRZ, KIT, ETH, MeteoSwiss
# Contact information: icon-model.org
# See AUTHORS.TXT for a list of authors
# See LICENSES/ for license information
# SPDX-License-Identifier: BSD-3-Clause
# ----------------------------------------------------------------------------

# ----------------------------------------------------------------------------
#
# Unified LES run script for all DEPHY cases
#
# Details: - Input data is only defined to a height of 20km.
#            Therefore use ICON-D2 vertical grid.
#
# Speed:     VEs         50x50x200m  100x100x100m  resolution
#            NE=1 CPE=8    305s        1551s
#  (1hour)   NE=4 CPE=8    234s         755s  (=5h for 24h LES, 1120s for L100)
#  (L65)     NE=8 CPE=8    233s         746s
#  (L100)    NE=4 CPE=8                 733s  (use levels_horizontal_mean for speed)
#
# Martin Koehler, Jan. 2022
# ----------------------------------------------------------------------------
#PBS -S /bin/bash
#PBS -q sx_norm
#PBS -v NE=4,CPE=8           # NE=1 or 4 or 8
##PBS -v NE=1,CPE=1          # NE=1,CPE=1 for VH-only runs
#PBS --venode=${NE}          # NE: number VE nodes, CPE: number cores on each VE
#PBS --venum-lhost=4         # Number of VE per logical host
#PBS -l cpunum_job=3         # number x86 cores per logical host (1ini+#IO)
#PBS -l coresz_prc=0
#PBS -T necmpi_hydra         # parallel job with NEC-MPIs
#PBS --use-hca=2             # number of HCA per logical host (2)
#PBS -l elapstim_req=24:00:00
#PBS -o /hpc/uwork/${USER}/wq/iconLES.%s
#PBS -j o
# ----------------------------------------------------------------------------

date
module load mpi/3.5.0
set -x
ulimit -s unlimited
. /etc/profile

# OpenMP settings
export OMP_SCHEDULE="static"
export OMP_DYNAMIC="false"
export OMP_NUM_THREADS=1                   # check total number of requested cores!
export VE_TRACEBACK=VERBOSE
export VE_FPE_ENABLE=DIV,FOF,INV

# Run information
let PPN=${CPE}*${NE}/${OMP_NUM_THREADS}    # no. of MPI procs. per NQS job
let NE1=${NE}-1
mpi_atm_io_procs=2                         # >0 for atmosphere plus land (not working for monitoring)
echo "PPN etc. $NN $NE $CPE $PPN $mpi_atm_io_procs"

ID=`echo $PBS_JOBID | cut -d: -f2 | cut -d. -f1`
ulimit -s unlimited

# ----------------------------------------------------------------------------
# specifiy experiment (idealized simulation)
# ----------------------------------------------------------------------------
CASE=FIRE                     # ARM, RICO, BOMEX, GABLS1, FIRE, MPACE, MAGIC
#EXPNAME=LES_100x100_${CASE}_dephy
EXPNAME=LES_50x50_${CASE}_dephy
#EXPNAME=LES_4x4_${CASE}_dephy

JOB_NAME="run_LES_dephycases_nec" # name of run script
RUNSCRIPTDIR=${PBS_O_WORKDIR}     # directory of run script

# case specific namelist parameters
. ${PBS_O_WORKDIR}/case_definition_dephy.s ${CASE}

# default LES physics parameters
inwp_convection=0              # 1:Tiedtke/Bechtold, 0:off
inwp_cldcover=5                # 3: clouds from COSMO SGS cloud scheme 0:off
inwp_turb=5                    # 1: TKE diffusion and transfer
inwp_sso=0
inwp_gwd=0

# ----------------------------------------------------------------------------
# path definitions
# ----------------------------------------------------------------------------

# base directory for ICON sources and binary:
ICONDIR=${PBS_O_WORKDIR%/*/*/*}    # local icon directory (up from scripts/scm/run)
#ICONDIR="/hpc/uhome/mkoehler/icon/icon-nwp-test2"

# SCM data directory (grids, init data, extpar)
SCMDATA=/hpc/uwork/mkoehler/scm/data       # at DWD on NEC

# directory with input grids:
GRIDDIR=${SCMDATA}/grid

# directory with init files:
INITDIR=${SCMDATA}/init_data/dephy_v1

# absolute path to output directory for results:
EXPDIR=${WORK}/run-icon/scm/${EXPNAME}

# path to model binary, including the executable:
MODEL_VH=${ICONDIR}/build/VH/bin/icon
MODEL_VE=${ICONDIR}/build/VE/bin/icon

# ----------------------------------------------------------------------------
# model timing
# ----------------------------------------------------------------------------

# 0.4 s for 70 m res., 0.5 for 100 m; 30 s for 2.5 km?
dtime=0.25                       # 0.5 200m, 0.25 100m
nhours=${nhours}
#nhours=1
#nsteps=`expr ${nhours} \* 3600 / ${dtime}`
nsteps=`echo ${nhours} \* 3600 / ${dtime} | bc`
nsteps=100

# set restart interval, checkpoint interval long
restart_interval="P1D"           #  restart interval: "P10D", PT25H", "P1D", "PT30M", 86400,
#restart_interval="PT02H"        #  restart interval: "P10D", PT25H", "P1D", "PT30M"
checkpoint_interval="P1Y"        #  write checkpoint (no need, can be very long)
restarts_onoff=0                 #  1: allow restarts, 0: prevent restarts

start_date=${start_date}
end_date=${end_date}

# ----------------------------------------------------------------------------
# grid namelist settings
# ----------------------------------------------------------------------------

# the grid parameters
#atmo_dyn_grids="Torus_Triangles_100x100_100m.nc"
atmo_dyn_grids="Torus_Triangles_50x50_200m.nc"
#atmo_dyn_grids="Torus_Triangles_4x4_2500m.nc"
atmo_rad_grids=""

# reconstruct the grid parameters in namelist form
dynamics_grid_filename=""
for gridfile in ${atmo_dyn_grids}; do
  dynamics_grid_filename="${dynamics_grid_filename} '${gridfile}',"
done
radiation_grid_filename=""
for gridfile in ${atmo_rad_grids}; do
  radiation_grid_filename="${radiation_grid_filename} '${gridfile}',"
done

# ----------------------------------------------------------------------------
# copy input data: grids, external parameters, model
# ----------------------------------------------------------------------------

# the directory for the experiment will be created, if not already there
if [ ! -d $EXPDIR ]; then
    mkdir -p $EXPDIR
fi
cd ${EXPDIR}

# grid
ln -sf ${GRIDDIR}/${atmo_dyn_grids} .

# initial condition and forcing
ln -sf ${INITDIR}/${file_dephy} init_SCM.nc

# files needed for radiation
ln -sf ${ICONDIR}/data/ECHAM6_CldOptProps.nc .
ln -sf ${ICONDIR}/data/rrtmg_lw.nc .
ln -sf ${ICONDIR}/data/rrtmg_sw.nc .

ecRad_data_path=${ICONDIR}'/externals/ecrad/data'

# ----------------------------------------------------------------------------
# restart setup: restart if Semaphore File exists
# ----------------------------------------------------------------------------

rm -f finish.status

lrestart=${lrestart:=".false."}
restartSemaphoreFilename='isRestartRun.sem'
if [ -f ${restartSemaphoreFilename} ] && [ restarts_onoff = 1 ] ; then
  lrestart=.true.
else
  lrestart=.false.
fi

# override lrestart:
#lrestart=.true.

# ----------------------------------------------------------------------------
# create ICON master namelist
# ----------------------------------------------------------------------------

cat > icon_master.namelist << EOF

&master_nml
 lrestart                    =                ${lrestart}         ! .TRUE.=current experiment is resumed
/
&master_model_nml
 model_type                  =                          1         ! identifies which component to run (atmosphere,ocean,...)
 model_name                  =                      "ATMO"        ! character string for naming this component.
 model_namelist_filename     =       "NAMELIST_${EXPNAME}"        ! file name containing the model namelists
 model_min_rank              =                          1         ! start MPI rank for this model
 model_max_rank              =                      65536         ! end MPI rank for this model
 model_inc_rank              =                          1         ! stride of MPI ranks
/

&master_time_control_nml
 restartTimeIntval    = "$restart_interval"
 checkpointTimeIntval = "$checkpoint_interval"
 experimentStartDate  = "$start_date"
 experimentStopDate   = "$end_date"
/

EOF

# ----------------------------------------------------------------------------
# model namelists
# ----------------------------------------------------------------------------
# For a complete list see doc/Namelist_overview.pdf

cat > NAMELIST_${EXPNAME} << EOF

&parallel_nml
 nproma         = 752
 p_test_run     = .false.
 num_io_procs   = ${mpi_atm_io_procs} ! number of I/O processors
 l_test_openmp  = .false.
 l_log_checks   = .false.
 proc0_shift    = 1                   ! 1: use first CPU for management, 0: only use VEs (delete VHs in mpirun)
 use_omp_input  = .true.
!l_fast_sum     = .TRUE.              ! .TRUE. speeds up global sums in apply_ls_forcing (most CPU time!)
/

&grid_nml
 dynamics_grid_filename = "${atmo_dyn_grids}",
 corio_lat              = ${corio_lat}
 is_plane_torus         = .TRUE.
 l_scm_mode             = .TRUE.      ! main logical to turn on SCM mode
/                                     ! note: if ldynamics off this transitions to LES or CRM

&SCM_nml
 i_scm_netcdf  = 2                    ! read initial profiles and forcings from netcdf - universal format
 lscm_read_tke = ${lscm_read_tke}     ! read initial tke from netcdf
 lscm_read_z0  = ${lscm_read_z0}      ! read initial z0 from netcdf
 lscm_random_noise = .TRUE.           ! add random noise at init
 scm_sfc_mom   = ${scm_sfc_mom}       ! 0: TURBTRANS: no prescribed u*
 scm_sfc_temp  = ${scm_sfc_temp}      ! 0: TERRA:     no prescribed sensible heat flux at surface
 scm_sfc_qv    = ${scm_sfc_qv}        ! 0: TERRA:     no prescribed latent heat flux at surface
/                                     ! 3: qv,sat(Tg)

&io_nml
 lkeep_in_sync  = .true.
 lflux_avg      = .FALSE.      ! false: accumulated fluxes
/

&run_nml
 num_lev        = 100          ! number of full levels of vertical grid
 dtime          = ${dtime}     ! timestep in seconds
 nsteps         = ${nsteps}
 ldynamics      = .TRUE.       ! compute adiabatic dynamic tendencies
 ltransport     = .TRUE.
 ntracer        = 5            ! default: 0
 iforcing       = 3            ! 3: NWP forcing; 6:inhecham forcing
 ltestcase      = .TRUE.       ! run testcase
 ltimer         = .TRUE.
 timers_level   = 10           ! 10: sufficiently detailed output
 msg_level      = 12           ! detailed report during integration (15 for SCM/LES output)
 output         = 'nml','totint'
/

&nwp_phy_nml
 inwp_gscp       = ${inwp_gscp}
 inwp_convection = ${inwp_convection} ! 1:Tiedtke/Bechtold, 0:off
 inwp_radiation  = ${inwp_radiation}  ! 1:RRTM radiation
 inwp_cldcover   = ${inwp_cldcover}   ! 3: clouds from COSMO SGS cloud scheme 0:off
 inwp_turb       = ${inwp_turb}       ! 1: TKE diffusion and transfer
 inwp_satad      = ${inwp_satad}
 inwp_sso        = ${inwp_sso}
 inwp_gwd        = ${inwp_gwd}
 inwp_surface    = 0            ! 0: none; 1: TERRA
 icapdcycl	 = 0            ! apply CAPE modification to improve diurnalcycle over tropical land (optimizes NWP scores)
 latm_above_top  = .TRUE.       ! needed for radiation routine
 itype_z0        = 2            ! 1: extpar, 2: land-cover dependent, 3: land-cover + sub-grid orography, see const_z0
 dt_rad	         = 1800.        ! Default: 1800   ! M. Koehler: 1440
 dt_conv         = 600.         ! Default: 600    ! M. Koehler: 360
 dt_sso	         = 600.         ! Default: 1200   ! M. Koehler: 720
 dt_gwd	         = 600.         ! Default: 1200   ! M. Koehler: 720
/

&radiation_nml
 irad_o3         = 9
 albedo_type     = 1            ! 1: table, 2: Modis albedo (attention: requires data update at midnight)
 direct_albedo   = 4
 direct_albedo_water = 3
 albedo_whitecap = 1
 vmr_co2         = 390.e-06     ! values representative for 2012
 vmr_ch4         = 1800.e-09
 vmr_n2o         = 322.0e-09
 vmr_o2          = 0.20946
 vmr_cfc11       = 240.e-12
 vmr_cfc12       = 532.e-12
 ecRad_data_path = '${ICONDIR}/externals/ecrad/data'
/

&les_nml
 isrfc_type      = 10           ! 10: boundary conditions for LES like in SCM
 ldiag_les_out   = .true.
/

&nh_testcases_nml
 w_perturb       = 0.1          ! random perturbation to start LES circulation (0.05 default, 0.1 faster start)
 th_perturb      = 0.2          ! random perturbation to start LES circulation (0.2 default)
/

&ls_forcing_nml
 is_subsidence_moment = ${is_subsidence_moment}
 is_subsidence_heat   = ${is_subsidence_heat}
 is_advection         = ${is_advection}
 is_advection_uv      = ${is_advection_uv}
 is_advection_tq      = ${is_advection_tq}
 is_geowind           = ${is_geowind}
 is_rad_forcing       = ${is_rad_forcing}
 is_nudging           = ${is_nudging}
 is_nudging_uv        = ${is_nudging_uv}
 is_nudging_t         = ${is_nudging_t}
 is_nudging_q         = ${is_nudging_q}
 nudge_start_height_uv = ${nudge_start_height_uv}
 nudge_full_height_uv = ${nudge_full_height_uv}
 nudge_start_height_t = ${nudge_start_height_t}
 nudge_full_height_t  = ${nudge_full_height_t}
 nudge_start_height_q = ${nudge_start_height_q}
 nudge_full_height_q  = ${nudge_full_height_q}
 dt_relax_uv          = ${dt_relax_uv}
 dt_relax_t           = ${dt_relax_t}
 dt_relax_q           = ${dt_relax_q}
 is_sim_rad           = ${is_sim_rad}
/

&turbdiff_nml
 tkhmin        = 0.6
 tkmmin        = 0.75
 pat_len       = 750.
 c_diff        = 0.2
 ltkesso       = .TRUE.
!frcsmot       = 0.2    ! these 2 switches together apply vertical smoothing of the TKE source terms
!imode_frcsmot = 2      ! in the tropics (only), which reduces the moist bias in the tropical lower troposphere
 ! use horizontal shear production terms with 1/SQRT(Ri) scaling to prevent unwanted side effects:
 itype_sher    = 0
 ltkeshs       = .false.
 a_hshr        = 0.0
 lconst_z0     = .TRUE.    ! TRUE: constant z0, typical for idealized LES cases ( or .FALSE. and itype_z0=2)
 const_z0      = 0.0001    ! ocean: about 0.0001m (better: user Charnock formula from turb_transder.f90)
 icldm_tran    = 2
 icldm_turb    = 2
 rlam_heat     = 10.0
 rat_sea       = 0.8
 alpha1        = 0.125
/

&nwp_tuning_nml
 itune_albedo     = 1
 tune_gkwake      = 1.5
 tune_gfrcrit     = 0.425
 tune_grcrit      = 0.5
 tune_gkdrag      = 0.16
 tune_zvz0i       = 0.85
 tune_box_liq_asy = 3.25
 tune_minsnowfrac = 0.2
 tune_gfluxlaun   = 3.75e-3
 tune_rcucov      = 0.075
 tune_rhebc_land  = 0.825
 lcalib_clcov     = .FALSE. !no tuning of cloud cover!
!tune_entrorg     = 7.4E-3 ! is 1.85e-3 normally
!tune_rdepths     = 60000.
!tune_texc        = 0.2 !0.125
!tune_qexc        = 1.e-4 !1.25e-2
!tune_gust_factor = 7.0
!icpl_turb_clc    = 2
!tune_box_liq     = 0.04
/

&diffusion_nml
 hdiff_order      = 5
 itype_vn_diffu   = 1
 itype_t_diffu    = 2
 hdiff_efdt_ratio = 24.
 hdiff_smag_fac   = 0.025
 lhdiff_vn        = .TRUE.
 lhdiff_temp      = .TRUE.
/

&nonhydrostatic_nml
 itime_scheme    = 4
 vwind_offctr    = 0.2
!damp_height     = 12250.
 damp_height     = 9000.
!rayleigh_coeff  = 0.5
 rayleigh_coeff  = 5.0
!divdamp_order   = 24
 divdamp_fac     = 0.004
 divdamp_type    = 32
 igradp_method   = 3
 l_zdiffu_t      = .true.
 thslp_zdiffu    = 0.02
 thhgtd_zdiffu   = 125.
 htop_moist_proc = 22500.
 hbot_qvsubstep  = 22500.
 exner_expol     = 0.333
 iadv_rhotheta   = 2
 ivctype         = 2
/

&sleve_nml
 decay_exp       = 1.2
 decay_scale_1   = 4000.
 decay_scale_2   = 2500.
 flat_height     = 16000.
 itype_laydistr  = 1
 min_lay_thckn   = 20.
 stretch_fac     = 0.65
 top_height      = 20000.
!top_height      = 22000.
!max_lay_thckn   = 400.
!htop_thcknlimit = 14000.
!stretch_fac     = 0.9
/

&extpar_nml
 itopo          = 0         ! 0: analytical topo; 1: topography/ext. data read from file
/

&dynamics_nml
 divavg_cntrwgt = 0.50
 lcoriolis      = .true.
/

&output_nml
 output_bounds    =  0., 0., 10800.           ! start, end, increment
 output_filename  = 'out_z_${EXPNAME}'
 steps_per_file   = 1000
 filename_format  = "<output_filename>_<levtype>_<datetime2>"
 ml_varlist       = 'z_mc', 'z_ifc'
 output_grid      = .TRUE.
/

&output_nml
 output_time_unit = 1                         ! 1: seconds
 output_bounds    = 0., 10000000., 3600.      ! start, end, increment
 mode             = 1                         ! 1: forecast
 steps_per_file   = 10000
 include_last     = .TRUE.
 output_filename  = 'out_ml_${EXPNAME}'
 filename_format  = "<output_filename>_<levtype>_<datetime2>"
 ml_varlist       = 'u','v','theta_v','tot_qv_dia','tot_qc_dia','tot_qi_dia','clc'
 output_grid      = .TRUE.
/

&output_nml
 output_time_unit = 1                         ! 1: seconds
 output_bounds    = 0., 10000000., 600.       ! start, end, increment
 mode             = 1                         ! 1: forecast
 steps_per_file   = 10000
 include_last     = .TRUE.
 output_filename  = 'out_sfc_${EXPNAME}'
 filename_format  = "<output_filename>_<levtype>_<datetime2>"
 ml_varlist       = 'shfl_s', 'lhfl_s','u_10m', 'v_10m', 'qv_2m','t_2m', 'clct',
                    'tqv_dia', 'tqc_dia', 'tqi_dia','tot_prec','sp_10m'
 output_grid      = .TRUE.
/

EOF


# --- 80m resolution to 7000m
# &run_nml
#  num_lev        = 87        ! number of full levels of vertical grid
# &nonhydrostatic_nml
#  ivctype        = 2         ! sleve vertical coordinate
#  damp_height    = 5000.     ! top_height-damp_height should be about 15km
#  l_open_ubc     = .false.   ! top open upper boundary condition. might help to go higher
#  rayleigh_coeff = 0.0
#  htop_moist_proc= 22500.
#  hbot_qvsubstep	= 22500.    ! Default: 22500  ! M. Koehler: 19000; at least as large as htop_moist_proc
# /
#
# &sleve_nml
#  top_height    = 7000.
#  min_lay_thckn = -40.       ! thickness of lowest model layer
# /


# ----------------------------------------------------------------------------
# run the model!
# ----------------------------------------------------------------------------

export VE_ERRCTL_ALLOCATE=MSG
export NMPI_PROGINF=YES
export VE_TRACEBACK=VERBOSE
export NMPI_SEPSELECT=3
export GMON_OUT_PREFIX=scal_prof
export VE_FPE_ENABLE=DIV,FOF,INV
export GFORTRAN_UNBUFFERED_PRECONNECTED=y
export NMPI_EXPORT="GFORTRAN_UNBUFFERED_PRECONNECTED"

date1=`date`

# --- VH only
#mpiexec -v -vh -node 0 -np 1 ${MODEL_VH}

# good: NE=1, 4, 8 or 16, CPE=8, venum-lhost=8, cpunum_job=2, use-hca=2

/opt/nec/ve/mpi/3.5.0/bin/runtime/mpirun                                       \
    -v -vh     -node 0        -np 1      -env OMP_NUM_THREADS 1  ${MODEL_VH} : \
    -x -venode -node 0-${NE1} -np ${PPN} -env OMP_NUM_THREADS 1  ${MODEL_VE} : \
       -vh     -node 0        -np 2      -env OMP_NUM_THREADS 1  ${MODEL_VH}

# --- other mpirun options:

# debug output: each processor with separate output in EXPDIR
# add /opt/nec/ve/bin/mpisep.sh before ${MODEL_V*}

# BAD: NE=4, CPE=8, venum-lhost=2, cpunum_job=2, use-hca=2

#/opt/nec/ve/bin/mpirun -v    -vh     -node 0        -np 1      -env OMP_NUM_THREADS 1 ${MODEL_VH} : \
#                          -x -venode -node 0-${NE1} -np ${PPN} -env OMP_NUM_THREADS 1 ${MODEL_VE} : \
#                             -vh     -node 0        -np 4      -env OMP_NUM_THREADS 1 ${MODEL_VH}

# NE=8
#mpirun -v -vh     -node 0        -np 1                   -env OMP_NUM_THREADS 1 /opt/nec/ve/bin/mpisep.sh ${MODEL_VH} : \
#       -x -venode -node 0-${NE1} -np ${PPN}              -env OMP_NUM_THREADS 1 /opt/nec/ve/bin/mpisep.sh ${MODEL_VE} : \
#          -vh     -node 0        -np ${mpi_atm_io_procs} -env OMP_NUM_THREADS 1 /opt/nec/ve/bin/mpisep.sh ${MODEL_VH}
# : \
#          -vh     -node 1        -np 3      ${MODEL_VH}

# NE=16, CPE=8, venum-lhost=8, cpunum_job=2, use-hca=2

#/opt/nec/ve/bin/mpirun -v    -vh     -node 0        -np 1      -env OMP_NUM_THREADS 1  ${MODEL_VH} : \
#                         -x -venode  -node 0-${NE1} -np ${PPN} -env OMP_NUM_THREADS 1  ${MODEL_VE} : \
#                             -vh     -node 0        -np 2      -env OMP_NUM_THREADS 1  ${MODEL_VH}

# NE=32
#mpiexec -v    -vh     -node 0        -np 1         ${MODEL_SCAL} : \
#        -x    -venode -node 0-${NE1} -np ${PPN}    ${MODEL}      : \
#        -v    -vh     -nn 4          -nnp 3        ${MODEL_SCAL}

#mpiexec -v    -node 0   -np 1        -vh          ${MODEL_SCAL} : \
#        -x    -venode   -node 0-${NE1} -np ${PPN} ${MODEL}      : \
#        -v    -nn 4     -nnp 3       -vh          ${MODEL_SCAL}


echo '---- time before/after: '$data1'   '`date`


# ----------------------------------------------------------------------------
# SCM mean over columns:
# ----------------------------------------------------------------------------

# merge currently available files for multiple restarts
#cdo mergetime *sfc*Z.nc out_sfc_LES_100x100_MAGIC_dephy_ML_20130720T173000Z_all.nc
#cod fldavg in out

outfile=out_z_${EXPNAME}_ML_`echo ${inidate} |cut -c 1-8`*Z.nc
meanfile=`echo ${outfile}| sed 's/.nc/_mean.nc/'`
cdo fldavg ${outfile} ${meanfile}

outfile=out_ml_${EXPNAME}_ML_`echo ${inidate} |cut -c 1-8`*Z.nc
meanfile=`echo ${outfile}| sed 's/.nc/_mean.nc/'`
cdo fldavg ${outfile} ${meanfile}

outfile=out_sfc_${EXPNAME}_ML_`echo ${inidate} |cut -c 1-8`*Z.nc
meanfile=`echo ${outfile}| sed 's/.nc/_mean.nc/'`
cdo fldavg ${outfile} ${meanfile}


# ----------------------------------------------------------------------------
# resubmit script
# ----------------------------------------------------------------------------

finish_status=`cat finish.status`
echo $finish_status
echo "============================"
echo "Script run successfully: $finish_status"
echo "============================"

if [ $finish_status = "RESTART" ] && [ $restarts_onoff = 1 ] ; then
  echo "restart next experiment..."
  this_script="${RUNSCRIPTDIR}/${JOB_NAME}"
  echo 'this_script: ' "$this_script"
  touch ${restartSemaphoreFilename}
  cd ${RUNSCRIPTDIR}
  qsub ${this_script}          # $run_param_0
else
  [[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
fi
