#! /usr/bash

# ICON
#
# ---------------------------------------------------------------
# Copyright (C) 2004-2026, DWD, MPI-M, DKRZ, KIT, ETH, MeteoSwiss
# Contact information: icon-model.org
# See AUTHORS.TXT for a list of authors
# See LICENSES/ for license information
# SPDX-License-Identifier: BSD-3-Clause
# ---------------------------------------------------------------

#______________________________________________________________________________
# Sets the parameters for various machine use_compiler configurations
# and creates the run script headers for these configurations
#
# Original version by Marco Giorgetta and Hermann Asensio
# Updated by Leonidas Linardakis, MPI-M, 2010-11-24
#______________________________________________________________________________
# Basic parameters (i.e. environment variables) in this script
#
# use_nodes  =  no of nodes to run
# use_mpi_procs_pernode = number of mpi procs per node
#
# output_script = absolute path of the job script that is generated
# output_folder = absolute path of the directory which will contain the job
#                 script
# ...
#______________________________________________________________________________

set -eu

BB_SYSTEM=${BB_SYSTEM:=""}
BB_NAME=${BB_NAME:=""}
DEADLINE="--deadline=now+130minutes"
MINIMAL_TIME="--time-min=00:10:00"
TIME_LIMITS="${MINIMAL_TIME} ${DEADLINE}"

add_free_var()
{
    # free variables are environment variables that will be added to the job
    # script
    varName=$1
    varValue=$2

    (( no_of_free_variables++ ))
    free_variable[$no_of_free_variables]=$varName
    free_variable_value[$no_of_free_variables]=$varValue
}

set_default()
{
    # Set variable with name $1 to $2 if it is not set or empty.
    eval : "\${$1:=$2}"
    # ":" is a shell build in that has no effect. It is used to make the variable expansion a valid statement.
    # ":="" assigns the default $2 value in place if parameter $1 was unset or null.
}

start_header()
{
    if [[ "$use_mpi" == "no" ]]
    then
        start=${start:=""}
    else
        start=${start:="$use_mpi_startrun"}
    fi

    if [[ ! -e "$output_script" ]]
    then
        echo "[ERROR] "$output_script" does not exist!"
        exit 1
    fi

    set_default use_ecrad_isolver 0 # Default: McICA (not for OpenACC)
    set_default use_proc0_shift 0 # VH offloading mode: off
    set_default use_num_io_procs 1

    cat >> "$output_script" << EOF
#-----------------------------------------------------------------------------
set +x
set -o pipefail
$(test "${use_ulimit:-yes}" != yes || printf 'ulimit -s $(ulimit -H -s)')
#-----------------------------------------------------------------------------
# handle job cancellation. Usually slurm return exit code 0 in that case
catchCancel() {
    echo "Job was canceled, setting non-zero exit code."
    exit 1
}

trap catchCancel SIGTERM SIGINT SIGKILL
#-----------------------------------------------------------------------------
# ICON run script:
# !ATTENTION! Do not change the format of the following lines.
#             They are evaluated by checksuite scripts.
# created by $0
# target machine is $use_target
# target use_compiler is $use_compiler
# with_mpi=$use_mpi
# with_openmp=$use_openmp
# memory_model=$use_memory_model
# submit with $use_submit
#
builder=${use_target}_${use_compiler}
with_openmp=$use_openmp
#-----------------------------------------------------------------------------
#
# OpenMP environment variables
# ----------------------------
export OMP_NUM_THREADS=${use_openmp_threads}
export ICON_THREADS=${use_openmp_threads}
export OMP_SCHEDULE=${use_OMP_SCHEDULE}
export OMP_DYNAMIC="false"
export OMP_STACKSIZE=${use_omp_stacksize:-200M}
#
# MPI variables
# -------------
proc0_shift=$use_proc0_shift
num_io_procs=${use_num_io_procs}

: \${no_of_nodes:=${use_nodes}} \${mpi_procs_pernode:=${use_mpi_procs_pernode}}
export no_of_nodes
export mpi_procs_pernode
((mpi_total_procs=no_of_nodes * mpi_procs_pernode))
export no_of_nodes mpi_procs_pernode mpi_total_procs

#
# blocking length
# ---------------
: \${nproma:=${use_nproma}} \${nproma_sub:=${use_nproma_sub}} \${nblocks_c:=${use_nblocks_c}}
export nproma nproma_sub nblocks_c

#
# Ecrad solver (0 for CPU/vector, 2 for GPU)
# ------------------------------------------
radiation_ecrad_isolver=${use_ecrad_isolver}
#
#-----------------------------------------------------------------------------

# load local setting, if existing
# -------------------------------
if [ -a ${use_builddir}/setting ]
then
  echo "Load Setting"
  . ${use_builddir}/setting
fi

# environment variables for the experiment and the target system
# --------------------------------------------------------------
EOF

# environment variables for the experiment and the target system

    i=1
    while [ $i -le ${no_of_free_variables} ]
    do
        echo "export ${free_variable[$i]}=\"${free_variable_value[$i]}\"" >> "$output_script"
        i=$((i+1));
    done

    # load addition profiles
    if [[ -n "$use_load_profile" ]]; then
      profile_filename=`echo $use_load_profile | cut -d ' ' -f2`
      cat >> "$output_script" << EOF
# load profile
# ------------
if [[ -a "$profile_filename" ]]
then
	$use_load_profile
fi
EOF
    fi

    # clean the module system
    if [[ -n "$use_purge_modules" ]]; then
      cat >> "$output_script" << EOF
#-----------------------------------------------------------------------------
# clean the module environment
${use_purge_modules}
EOF
      fi

    # load user defined module paths
    if [[ -n "$use_load_module_paths" ]]; then
      cat >> "$output_script" << EOF
#-----------------------------------------------------------------------------
# load modules
load_module_paths="$use_load_module_paths"
module use \$load_module_paths
#-----------------------------------------------------------------------------
EOF
    fi

    # load user defined modules
    if [[ -n "$use_load_modules" ]]; then
      cat >> "$output_script" << EOF
#-----------------------------------------------------------------------------
# load modules
loadmodule="$use_load_modules"
module load \$loadmodule
module list
#-----------------------------------------------------------------------------
EOF
    fi

    cat >> "$output_script" << EOF

#-----------------------------------------------------------------------------
# directories with absolute paths
# -------------------------------
thisdir=\$(pwd)
export basedir="${use_builddir}"
# experiments_dir can be predefined in a machine specific run_target_* header
experiments_dir="\${experiments_dir:=\${basedir}/experiments}"
export icon_data_rootFolder="${icon_data_rootFolder}"
export testingFolder_prefix="${testingFolder_prefix:=~/testingFolder}"

# how to start the icon model
# ---------------------------
export START="$start"
export MODEL="${use_builddir}/bin/icon"

set | grep SLURM

# how to submit the next job
# --------------------------
submit="$use_submit"
job_name="$job_name"

# cdo for post-processing
# -----------------------
cdo="${cdo}"
cdo_diff="${cdo} ${cdo_diff}"

# export the python interpreter
export PYTHON="${PYTHON}"
export PYTHONPATH="${BUNDLED_PYTHONPATH}${PYTHONPATH:+:$PYTHONPATH}"

# define script functions used in the experiment run script
# ---------------------------------------------------------
. \${basedir}/run/add_run_routines

#-----------------------------------------------------------------------------

EOF

}

# Limits the run time for slurm jobs started with BB
limit_time_bb()
{
    local limit=$1
    local default=$2

    if [[ "x$use_cpu_time" = "x" ]]; then # time is not set
        use_cpu_time=$default
        return 0
    fi
    if [[ ${use_cpu_time} =~ ^[0-9][0-9]:[0-5][0-9]:[0-5][0-9]$ ]]; then # check if time format equals xx:xx:xx
        if [[ ${use_cpu_time} > ${limit} ]]; then # string comparison
            echo "Requested time ${use_cpu_time} is bigger than the limit ${limit}. Using ${limit}."
            use_cpu_time=$limit
            return 0
        fi
    elif [[ ${use_cpu_time} =~ ^[0-5][0-9]:[0-5][0-9]$ ]]; then # check if time format equals xx:xx
        if [[ ${limit::2} == "00" ]]; then # only compare for limit < 1h
            if [[ ${use_cpu_time} > ${limit:3} ]]; then # string comparison
                echo "Requested time ${use_cpu_time} is bigger than the limit ${limit}. Using ${limit}."
                use_cpu_time=$limit
                return 0
            fi
        fi
    else
        echo "Requested time ${use_cpu_time} format not recognised (expected xx:xx:xx or xx:xx). Using default of ${default}."
        use_cpu_time=$default
        return 0
    fi
}

#-----------------------------------------------------------------------------
set_run_target_default()
{
    # Default configuration for unknown systems

    icon_data_rootFolder=${icon_data_rootFolder:=~/pool/data/ICON}

    set_default use_ulimit yes
    set_default use_nproma 48
    set_default use_nproma_sub 48
    set_default use_nblocks_c 0
    set_default use_nodes 1

    if [[ "$use_mpi" == "yes" ]]
    then
	set_default use_mpi_procs_pernode 4
    fi
    if [[ "$use_openmp" == "yes" ]]
    then
	set_default use_openmp_threads 2
    else
	set_default use_openmp_threads 1
    fi

    start_header

cat >> "$output_script" << EOF
$use_build_env
EOF

}

#-----------------------------------------------------------------------------
set_run_target_()
{
  echo "!! THIS IS A FALLBACK SETUP !!"
  echo "!! YOUR HOST/SITE IS unknown TO THE SCRIPTING ENGINE !!"
  set_run_target_default
}

#-----------------------------------------------------------------------------
# setup for levante.dkrz.de
# - compute partions is always exclusive
# - runscript generation should respect slurm settings when run by another job
# - default: single node job on compute (allthough this is way to much for 160km experiments)
set_run_target_bull_milan()
{
    icon_data_rootFolder=${icon_data_rootFolder:=/pool/data/ICON}

    set_default use_nproma 32
    set_default use_nblocks_c 0
    set_default use_nproma_sub 32
    set_default use_nodes ${SLURM_NNODES:-1}
    set_default use_num_io_procs 1
    set_default use_account_no "$(id -gn)"

    # set to 1 to use smt feature
    smt_adjust=1

    if [[ "x$BB_SYSTEM" != "x" ]]
    then
        queue=${use_queue:-shared}
        [[ ${use_nodes} -gt 1 ]] && queue='compute'

	# use the MPIMET testing account for buildbot
	use_account_no="mh0156"

	# use a fixed logfile name so that the buildbot server can identify it
	job_log_name="LOG.$job_name"

        # checksuite jobs are created ON shared, but may want to run on
        # 'compute' if it's specified in the experiment list
        [[ "${queue}" = 'shared' ]] && set_default use_tasks $((16 * ${smt_adjust}))
        set_default use_tasks $((128 * ${smt_adjust}))
    else
        queue=${use_queue:-compute}

        set_default use_tasks $((128 * ${smt_adjust}))
	job_log_name="LOG.$job_name.%j"
    fi
    #   use_submit="$use_submit -N \${SLURM_JOB_NUM_NODES:-1}"
    use_OMP_SCHEDULE="dynamic,1"
    use_omp_stacksize=200M

    is_serial_run="no"
    if [[ $use_mpi == "yes" ]]
    then
	if [[ $use_openmp == "yes" ]]
	then
	    set_default use_openmp_threads '"\$(( 4 * '"$smt_adjust"'))"'

            set_default use_mpi_procs_pernode '"\$(( '"${use_tasks}"' * '"${smt_adjust}"' / OMP_NUM_THREADS))"'
	else
	    set_default use_mpi_procs_pernode '"\$(( '"${use_tasks}"' * '"$smt_adjust"'))"'
	fi
    else
	start="srun -n 1"
	if [[ $use_openmp == "yes" ]]
        then
	    set_default use_openmp_threads 64
	    #      set_default use_mpi_procs_pernode 1
	    start+=" --cpus-per-task=\$(($smt_adjust * OMP_NUM_THREADS))"
	else
	    is_serial_run="yes"
	fi
    fi

    # FIXME: This probably still needs some improvement.
    # The main observation is: ICON runs consistently faster without SMT. Hence the '--hint=nomultithread' is used
    # if less than one node is needed, use_node = 0
    if [[ $use_tasks -lt 128 ]]
    then
        use_mpi_startrun="srun -l --kill-on-bad-exit=1 --distribution=plane --hint=nomultithread --ntasks=\${mpi_procs_pernode} --cpus-per-task=\${OMP_NUM_THREADS}"
    else
        use_mpi_startrun="srun -l --kill-on-bad-exit=1 --nodes=\${SLURM_JOB_NUM_NODES:-1} --distribution=plane --hint=nomultithread --ntasks=\$((no_of_nodes * mpi_procs_pernode)) --ntasks-per-node=\${mpi_procs_pernode} --cpus-per-task=\${OMP_NUM_THREADS}"
    fi

    add_free_var MALLOC_TRIM_THRESHOLD_ -1
    add_free_var SLURM_DIST_PLANESIZE 32
    case _$use_compiler in
	"_intel" )
            add_free_var KMP_AFFINITY "granularity=fine,scatter"
            add_free_var KMP_LIBRARY "turnaround"
            add_free_var MALLOC_TRIM_THRESHOLD_ "-1"
            add_free_var MKL_DEBUG_CPU_TYPE "5"
            add_free_var MKL_ENABLE_INSTRUCTIONS "AVX2"
	    ;;
    esac
    case $use_mpi_root in
	*openmpi*)
            add_free_var OMPI_MCA_btl "self"
            add_free_var OMPI_MCA_coll "^ml,hcoll"
            add_free_var OMPI_MCA_io "romio321"
            add_free_var OMPI_MCA_osc "ucx"
            add_free_var OMPI_MCA_pml "ucx"
            add_free_var UCX_HANDLE_ERRORS "bt"
            add_free_var UCX_TLS "shm,dc_mlx5,dc_x,self"
            add_free_var UCX_UNIFIED_MODE "y"
	    ;;
    esac

    if [[ "$is_serial_run" == "yes"  ]]
    then
	use_nodes=1
	tasks_per_node=1
	SBATCH_ntasks_per_node=""
    else
	job_type=parallel
	tasks_per_node=$use_mpi_procs_pernode
    fi

    s_cpu_time="${use_cpu_time:-00:30:00}"
    # ------------------------------------

    cat >> "$output_script" << EOF
#-----------------------------------------------------------------------------
#SBATCH --account=$use_account_no
$(if [[ "x$BB_SYSTEM" != 'x' ]]
 then
  printf "#SBATCH --qos=buildbot
#SBATCH ${TIME_LIMITS}
";fi)
#SBATCH --job-name=$job_name
#SBATCH --partition=$queue
#SBATCH --nodes=$use_nodes
#SBATCH --chdir=$output_folder
#SBATCH --output=$output_folder/$job_log_name.o
EOF
    if [[ -n "$use_memory" ]]; then
        echo "#SBATCH --mem=${use_memory}" >> "$output_script"
    fi
    if [[ $use_tasks -lt 128 ]]; then
      # here twice the given number of tasks is used to requesting resources, because --hint=nomultithread is used
      echo "#SBATCH --ntasks=$((2 * $use_tasks))" >> "$output_script"
    fi
    cat >> "$output_script" << EOF
#SBATCH --time=$s_cpu_time
EOF

    # for the rest use the SLURM_JOB_NUM_NODES by default
    use_nodes="\${SLURM_JOB_NUM_NODES:=${use_nodes}}"

    start_header

    cat >> "$output_script" <<EOF
ulimit -c 0
$(case $use_flags_group in
    (debug)
      printf 'ulimit -c unlimited'
      ;;
  esac)
EOF
}
#-----------------------------------------------------------------------------
set_run_target_bullx_gpu()
{
    icon_data_rootFolder=${icon_data_rootFolder:=/pool/data/ICON}

    set_default use_nproma 0
    set_default use_nproma_sub 8000
    set_default use_nblocks_c 1
    set_default use_nodes 1
    set_default use_num_io_procs 1
    set_default use_account_no "$(id -gn)"
    set_default use_ecrad_isolver 2 # McICA for OpenACC

    if [[ "x$BB_SYSTEM" != "x" ]]
    then
	# this buildbot, do not use job id
	use_account_no="mh0156"
    fi
    use_submit="$use_submit -N \${SLURM_JOB_NUM_NODES:-1}"
    use_OMP_SCHEDULE="dynamic,1"
    use_omp_stacksize=200M

    # set to 1 to use smt feature
    smt_adjust=2
    is_serial_run="no"
    if [[ $use_mpi == "yes" ]]
    then
	num_gpus_pernode=4
	set_default use_mpi_procs_pernode '"\$(( $num_gpus_pernode + 1 ))"'
	srun_cmd="srun -l --kill-on-bad-exit=1 --nodes=\${SLURM_JOB_NUM_NODES:-1} --gpus-per-node=${num_gpus_pernode}"
    srun_cmd="${srun_cmd} --ntasks=\$((no_of_nodes * mpi_procs_pernode))"
    srun_cmd="${srun_cmd} \${basedir}/run/run_wrapper/levante.sh -n \${mpi_total_procs} -o \${num_io_procs} -e"
    use_mpi_startrun="${srun_cmd}"
    else
	start="srun --cpu-freq=HighM1 -n 1"
	is_serial_run="yes"
    fi

    if test "x$use_queue" != 'x' -a "x$use_queue" != 'xcompute' -a "x$use_queue" != 'xshared' ; then queue=$use_queue; fi

    add_free_var MALLOC_TRIM_THRESHOLD_ -1
    if [[ "$is_serial_run" == "yes"  ]]
    then
	queue=${queue:="gpu"}
	use_nodes=1
	tasks_per_node=1
	SBATCH_ntasks_per_node=""
    else
        if [[ "x$BB_SYSTEM" != "x" ]]
        then
            queue="${queue:-gpu}"
            #TODO queue="${queue:-gpu-devel,gpu}" # needs slurm config update
        else
            queue=${queue:="gpu"}
            job_type=parallel
            tasks_per_node=$use_mpi_procs_pernode
        fi
    fi

    if [[ "x$BB_SYSTEM" != "x" ]]
    then
	# this buildbot, do not use job id
	job_log_name="LOG.$job_name"
    else
	job_log_name="LOG.$job_name.%j"
    fi

    # ------------------------------------
    s_cpu_time="00:30:00"
    if [[ "x$use_cpu_time" != "x" ]]
    then
	s_cpu_time=$use_cpu_time
    fi
    # ------------------------------------

    cat >> "$output_script" << EOF
#-----------------------------------------------------------------------------

# levante gpu batch job parameters
# --------------------------------
#SBATCH --account=$use_account_no
$(if [[ "x$BB_SYSTEM" != 'x' ]]
 then
  printf "#SBATCH --qos=buildbot
#SBATCH ${TIME_LIMITS}
";fi)
#SBATCH --job-name=$job_name
#SBATCH --partition=$queue
#SBATCH --chdir=$output_folder
#SBATCH --nodes=$use_nodes
#SBATCH --gpus-per-node=$num_gpus_pernode
#SBATCH --mem=0
#SBATCH --output=$output_folder/$job_log_name.o
#SBATCH --error=$output_folder/$job_log_name.o
#SBATCH --exclusive
#SBATCH --time=$s_cpu_time
$(if [[ "x${BB_SYSTEM}" = 'x' ]]
  then
  printf '#SBATCH --constraint=a100_80
';fi)
$(if [[ "$use_mpi_root" == *intel*mpi* ]]
 then
  printf '#----------------------------------------
# the following line is only needed for srun to work with Intel MPI
# but should be commented when using BullX MPI
export I_MPI_PMI_LIBRARY=/usr/lib64/libpmi.so
#----------------------------------------' ; fi)
EOF

    # for the rest use the SLURM_JOB_NUM_NODES by default
    use_nodes="\${SLURM_JOB_NUM_NODES:=${use_nodes}}"

    start_header

    cat >>"$output_script" << EOF
ulimit -c 0
$(case $use_flags_group in
    (debug)
      printf 'ulimit -c unlimited'
      ;;
  esac)

# Enable CUDA graphs support. Setting this is harmless when the graphs are not used
export NVCOMPILER_ACC_USE_GRAPH=1
EOF
}

#-----------------------------------------------------------------------------
set_run_target_levante_aurora()
{
    # Aurora nodes at DKRZ

    icon_data_rootFolder=${icon_data_rootFolder:=/pool/data/ICON}

    if [[ "$use_compiler" == "nec" ]]; then
      set_default use_nproma 752
      set_default use_nproma_sub 752
    else
      set_default use_nproma 48
      set_default use_nproma_sub 48
    fi
    set_default use_nblocks_c 0
    set_default use_nodes 1

    if [[ "$use_mpi" == "yes" ]]
    then
	set_default use_mpi_procs_pernode 4
    fi

cat >>"${output_script}" <<EOF
${use_build_env}
EOF

    start_header

}

#-----------------------------------------------------------------------------
set_run_target_mpipc()
{
    # *.mpimet.mpg.de with Linux (workstations and breeze)
    icon_data_rootFolder=${icon_data_rootFolder:=/pool/data/ICON}

    # only user-local mount/copys of /pool from dkrz
    [[ "Darwin" = "$(uname -o)" ]] && icon_data_rootFolder="${HOME}/pool/data/ICON"

    set_default use_nproma 64
    set_default use_nproma_sub 64
    set_default use_nblocks_c 0
    use_nodes=1

    if [[ $use_mpi == "yes" ]]
    then
	set_default use_mpi_procs_pernode 2
    fi
    if [[ $use_openmp == "yes" ]]
    then
	set_default use_openmp_threads 2
    else
	set_default use_openmp_threads 1
    fi

    start_header
}

#-----------------------------------------------------------------------------
set_run_target_lumi_gpu()
{
    # LUMI at CSC
    icon_data_rootFolder=${icon_data_rootFolder:=/appl/local/climatedt/pool/data/ICON}

    set_default use_nproma 0
    set_default use_nproma_sub 0
    set_default use_nblocks_c 1
    set_default use_prefetch_procs 0
    set_default use_account_no ${SBATCH_ACCOUNT:-project_465000454}
    set_default use_nodes 1
    if [[ "x$BB_SYSTEM" != "x" ]]; then
        # this buildbot, use debug queue
        set_default use_queue "dev-g"
    else
        set_default use_queue "standard-g"
    fi

    use_OMP_SCHEDULE="static,1"

    is_serial_run="no"

    set_default use_mpi_procs_pernode $((8+use_prefetch_procs))
    set_default use_openmp_threads 1

    job_type=parallel
    tasks_per_node=$use_mpi_procs_pernode
    SBATCH_nodes="SBATCH --nodes=$use_nodes"
    SBATCH_ntasks_per_node="SBATCH --ntasks-per-node=${tasks_per_node}"
    cpu_bind='map_cpu:49,57,17,25,1,9,33,41'

    # ------------------------------------
    job_log_name=""
    if [[ "x$BB_SYSTEM" != "x" ]]
    then
	# this buildbot, do not use job id
	job_log_name="LOG.%x"
        # do not bind cpus
        cpu_bind=
    else
	job_log_name="LOG.%x.%j"
    fi
    # ------------------------------------
    s_cpu_time="00:30:00"
    if [[ "x$use_cpu_time" != "x" ]]
    then
      s_cpu_time="$use_cpu_time"
    fi
    # ------------------------------------

    cat >> $output_script << EOF
#-----------------------------------------------------------------------------

# LUMI gpu batch job parameters
# ------------------------------
#SBATCH --job-name=$job_name
#SBATCH --account=$use_account_no
#SBATCH --output=$output_folder/$job_log_name.o
#SBATCH --error=$output_folder/$job_log_name.o
#SBATCH --chdir=$output_folder/
#$SBATCH_nodes
#$SBATCH_ntasks_per_node
#SBATCH --gpus-per-node=${tasks_per_node}
#SBATCH --mem=0
#SBATCH --time=${s_cpu_time}
#SBATCH --partition=${use_queue}
$(if [[ "x$BB_SYSTEM" != 'x' ]]
 then
  printf "#SBATCH ${TIME_LIMITS}
";fi)
EOF

    start_header
    cat >>$output_script <<EOF

export FI_MR_CACHE_MONITOR=memhooks
export MPICH_OFI_NIC_POLICY=GPU
export MPICH_GPU_SUPPORT_ENABLED=1
${cpu_bind:+"export SLURM_CPU_BIND='${cpu_bind}'"}
EOF
}

#-----------------------------------------------------------------------------
set_run_target_lumi_cpu()
{
    # LUMI at CSC
    icon_data_rootFolder=${icon_data_rootFolder:=/appl/local/climatedt/pool/data/ICON}

    set_default use_nproma 32
    set_default use_nblocks_c 0
    set_default use_account_no ${SBATCH_ACCOUNT:-project_462000911}
    set_default use_nodes 1
    if [[ "x$BB_SYSTEM" != "x" ]]; then
        # this buildbot, use debug queue
        set_default use_queue "debug"
    else
        set_default use_queue "standard"
    fi

    use_OMP_SCHEDULE="static,1"

    is_serial_run="no"

    set_default use_mpi_procs_pernode 32
    set_default use_openmp_threads 4

    job_type=parallel
    tasks_per_node=$use_mpi_procs_pernode
    SBATCH_nodes="SBATCH --nodes=$use_nodes"
    SBATCH_ntasks_per_node="SBATCH --ntasks-per-node=${tasks_per_node}"
    SBATCH_cpus_per_task="SBATCH --cpus-per-task=${use_openmp_threads}"

    use_openmp_threads=\$SLURM_CPUS_PER_TASK

    # ------------------------------------
    job_log_name=""
    if [[ "x$BB_SYSTEM" != "x" ]]
    then
	# this buildbot, do not use job id
	job_log_name="LOG.%x"
    else
	job_log_name="LOG.%x.%j"
    fi
    # ------------------------------------
    s_cpu_time="00:30:00"
    if [[ "x$use_cpu_time" != "x" ]]
    then
      s_cpu_time="$use_cpu_time"
    fi
    # ------------------------------------

    cat >> $output_script << EOF
#-----------------------------------------------------------------------------

# LUMI cpu batch job parameters
# ------------------------------
#SBATCH --job-name=$job_name
#SBATCH --account=$use_account_no
#SBATCH --output=$output_folder/$job_log_name.o
#SBATCH --error=$output_folder/$job_log_name.o
#SBATCH --chdir=$output_folder/
#$SBATCH_nodes
#$SBATCH_ntasks_per_node
#$SBATCH_cpus_per_task
#SBATCH --time=${s_cpu_time}
#SBATCH --partition=${use_queue}
$(if [[ "x$BB_SYSTEM" != 'x' ]]
 then
  printf "#SBATCH ${TIME_LIMITS}
";fi)
EOF

    start_header
    cat >>$output_script <<EOF

export OMP_PLACES=cores
export FI_MR_CACHE_MONITOR=memhooks
EOF
}

#=============================================================================
set_run_target_mn5_cpu()
{
    # Mare Nostrum 5 at BSC
    icon_data_rootFolder=${icon_data_rootFolder:=/gpfs/projects/ehpc01/models/icon/inidata}

    set_default use_nproma 32
    set_default use_nblocks_c 0
    set_default use_nproma_sub 32
    set_default use_account_no "ehpc01"
    set_default use_nodes 1
    set_default use_queue "gpp"
    set_default use_qos "gp_ehpc"

    use_OMP_SCHEDULE="static"

    is_serial_run="no"

    set_default use_mpi_procs_pernode 28
    set_default use_openmp_threads 4

    job_type=parallel
    tasks_per_node=$use_mpi_procs_pernode
    SBATCH_nodes="SBATCH --nodes=$use_nodes"
    SBATCH_ntasks_per_node="SBATCH --ntasks-per-node=${tasks_per_node}"
    SBATCH_cpus_per_task="SBATCH --cpus-per-task=${use_openmp_threads}"

    use_openmp_threads=\$SLURM_CPUS_PER_TASK
    use_mpi_startrun="srun --cpus-per-task=\${SLURM_CPUS_PER_TASK}"

    # ------------------------------------
    job_log_name=""
    if [[ "x$BB_SYSTEM" != "x" ]]
    then
	# this buildbot, do not use job id
	job_log_name="LOG.%x"
    else
	job_log_name="LOG.%x.%j"
    fi
    # ------------------------------------
    s_cpu_time="00:30:00"
    if [[ "x$use_cpu_time" != "x" ]]
    then
      s_cpu_time="$use_cpu_time"
    fi
    # ------------------------------------

    cat >> $output_script << EOF
#-----------------------------------------------------------------------------

# MN5 cpu batch job parameters
# ------------------------------
#SBATCH --job-name=$job_name
#SBATCH --account=$use_account_no
#SBATCH --output=$output_folder/$job_log_name.o
#SBATCH --error=$output_folder/$job_log_name.o
#SBATCH --chdir=$output_folder
#$SBATCH_nodes
#$SBATCH_ntasks_per_node
#$SBATCH_cpus_per_task
#SBATCH --time=${s_cpu_time}
#SBATCH --partition=${use_queue}
#SBATCH --qos=${use_qos}
$(if [[ "x$BB_SYSTEM" != 'x' ]]
 then
  printf "#SBATCH ${TIME_LIMITS}
";fi)
EOF

    start_header
    cat >>$output_script <<EOF

export OMP_PLACES=cores
EOF
}

#-----------------------------------------------------------------------------
set_run_target_mn5_gpu()
{
    # Mare Nostrum 5 at BSC
    icon_data_rootFolder=${icon_data_rootFolder:=/gpfs/projects/ehpc01/models/icon/inidata}

    set_default use_nproma 0
    set_default use_nblocks_c 1
    set_default use_nproma_sub 0
    set_default use_account_no "ehpc01"
    set_default use_nodes 1
    set_default use_queue "acc"
    set_default use_qos "acc_ehpc"

    use_OMP_SCHEDULE="static"

    is_serial_run="no"

    set_default use_mpi_procs_pernode 4
    set_default use_cpus_per_task 20
    set_default use_openmp_threads 1

    job_type=parallel
    tasks_per_node=$use_mpi_procs_pernode
    SBATCH_nodes="SBATCH --nodes=$use_nodes"
    SBATCH_ntasks_per_node="SBATCH --ntasks-per-node=${tasks_per_node}"
    SBATCH_cpus_per_task="SBATCH --cpus-per-task=${use_cpus_per_task}"

    # ------------------------------------
    job_log_name=""
    if [[ "x$BB_SYSTEM" != "x" ]]
    then
	# this buildbot, do not use job id
	job_log_name="LOG.%x"
    else
	job_log_name="LOG.%x.%j"
    fi
    # ------------------------------------
    s_cpu_time="00:30:00"
    if [[ "x$use_cpu_time" != "x" ]]
    then
      s_cpu_time="$use_cpu_time"
    fi
    # ------------------------------------

    cat >> $output_script << EOF
#-----------------------------------------------------------------------------

# MN5 gpu batch job parameters
# ------------------------------
#SBATCH --job-name=$job_name
#SBATCH --account=$use_account_no
#SBATCH --output=$output_folder/$job_log_name.o
#SBATCH --error=$output_folder/$job_log_name.o
#SBATCH --chdir=$output_folder
#$SBATCH_nodes
#$SBATCH_ntasks_per_node
#$SBATCH_cpus_per_task
#SBATCH --gpus-per-node=${tasks_per_node}
#SBATCH --time=${s_cpu_time}
#SBATCH --partition=${use_queue}
#SBATCH --qos=${use_qos}
$(if [[ "x$BB_SYSTEM" != 'x' ]]
 then
  printf "#SBATCH ${TIME_LIMITS}
";fi)
EOF

    start_header
    cat >>$output_script <<EOF

export SLURM_CPU_BIND=none
EOF
}

#-----------------------------------------------------------------------------
set_run_target_docker_cpu()
{
    # Docker container
    icon_data_rootFolder=${icon_data_rootFolder:=/icon-data-pool}

    set_default use_nproma 16
    set_default use_nproma_sub 16
    set_default use_nblocks_c 0
    use_nodes=1

    set_default use_mpi_procs_pernode 1
    set_default use_openmp_threads 1

    use_OMP_SCHEDULE="static,12"

    start_header
}

#-----------------------------------------------------------------------------
set_run_target_docker_gpu()
{
    # Docker container with GPU
    icon_data_rootFolder=${icon_data_rootFolder:=/icon-data-pool}

    set_default use_nproma 20480
    set_default use_nproma_sub 800
    set_default use_nblocks_c 0
    use_nodes=1

    set_default use_mpi_procs_pernode 1
    set_default use_openmp_threads 1

    use_OMP_SCHEDULE="static,12"

    start_header
}

#-----------------------------------------------------------------------------
set_run_target_rcl()
{
  # rcl.dwd.de
    icon_data_rootFolder=${icon_data_rootFolder:="/hpc/rwork0/routfor/test/icon"}

    if [[ "$use_mpi" == "yes" ]]
    then
        # Careful, this is the number of threads per VE, not the number of MPI processes.
        set_default use_mpi_procs_pernode 8
    else
        set_default use_mpi_procs_pernode 1
    fi

    if [[ "$use_openmp" == "yes" ]]
    then
        set_default use_openmp_threads 2
    else
        set_default use_openmp_threads 1
    fi

    use_OMP_SCHEDULE="dynamic,1"
    set_default use_omp_stacksize 2G

    set_default use_nproma 752
    set_default use_nproma_sub $use_nproma
    set_default use_nblocks_c 0
    set_default use_nodes 1
    set_default use_venum_lhost 1
    set_default use_cpunum_job 6
    set_default use_proc0_shift 1 # VH offloading mode: on with 1 proc
    set_default use_num_io_procs 1

    # $use_memory can be used to set memsz_job. e.G. "24gb"

    # absolute path to cdo
    cdo="/hpc/sw/cdo/2.2.1/x86/gnu/bin/cdo"
    cdo_diff="diffn"

    add_free_var VE_ERRCTL_ALLOCATE "MSG"
    add_free_var NMPI_PROGINF "YES"
    add_free_var VE_TRACEBACK "VERBOSE"
    add_free_var NMPI_SEPSELECT "3"
    add_free_var GMON_OUT_PREFIX "scal_prof"
    add_free_var VE_FPE_ENABLE "DIV,FOF,INV"
    add_free_var GFORTRAN_UNBUFFERED_PRECONNECTED "y"
    add_free_var NMPI_EXPORT "GFORTRAN_UNBUFFERED_PRECONNECTED"
    add_free_var NMPI_SWAP_ON_HOLD "OFF"

    # try to guess the location of the Vector Host binary
    #
    # make_runscripts and create_target_header must be run from the VectorEngine build directory.
    # Only then, it is possible to choose the correct MPI version depending on the nfort version.
    #
    # In the following $use_builddir from set-up.info is used because this is directory
    # were the target binary that was created during compilation.
    if [[ "$use_compiler" == "nec" ]]; then
        # "nec" is the VE compiler
        MODEL="${use_builddir}/bin/icon"
        if [[ -n "$secondary_build_dir" ]]; then
            MODEL_SCAL="${secondary_build_dir}/bin/icon"
        else
            # Default fallback
            # if "VE" is used as build directory, look for "../VH", otherwise use "../host"
            case "$use_builddir" in
                */VE )
                    MODEL_SCAL="${use_builddir}/../VH/bin/icon"
                    ;;
                */vector )
                    MODEL_SCAL="${use_builddir}/../host/bin/icon"
                    ;;
                * )
                    # fallback assumption:
                    # host build was configured with '--prefix=$PWD/host' as in buildbot
                    MODEL_SCAL="${use_builddir}/host/bin/icon"
                    ;;
            esac
            if [[ ! -x "${MODEL_SCAL}" ]]; then
                echo "[ERROR] create_target_header: Automatic detection of vector host binary has failed."
                echo "Could not find host executable '${MODEL_SCAL}'."
                echo "Use a default build directory structure ('VE/' + ' VH/' or 'vector/' + 'host/') or set the option 'secondary_build_dir'."
                exit 1
            fi
        fi
    else
        echo "[ERROR] create_target_header:"
        echo "make_runscripts and create_target_header must be run from the VectorEngine build directory."
        echo "Only then, it is possible to choose the correct MPI version depending on the nfort version."
        exit 1
    fi

    cat >> "$output_script" << EOF
#-----------------------------------------------------------------------------
#PBS -q ${use_queue:-sx_norm}
#PBS -v NE=${use_nodes},CPE=${use_mpi_procs_pernode}
#PBS -v BB_NAME
#PBS -l elapstim_req=00:40:00
#PBS --venode=\${NE}
#PBS --venum-lhost=${use_venum_lhost}     # Number of VE per logical host
EOF
    if [[ -n "$use_memory" ]]; then
        echo "#PBS -l memsz_job=${use_memory}" >> "$output_script"
    fi
    cat >> "$output_script" << EOF
#PBS -l cpunum_job=${use_cpunum_job}
#PBS -l coresz_prc=0
#PBS --use-hca=2             # Number of HCA per logical host
#PBS -j o
#PBS -o LOG.${job_name}.o
#PBS -T necmpi_hydra
# ----------------------------------------------------------------------------
date

EOF

    start_header

    cat >> "$output_script" << EOF
set -x

export VE_OMP_STACKSIZE=\$OMP_STACKSIZE

# Run information
((PPN=CPE * NE / OMP_NUM_THREADS))         # no. of MPI procs. per NQS job
echo "NE CPE PPN: \$NE \$CPE \$PPN"

ID=\$(echo \$PBS_JOBID | cut -d: -f2 | cut -d. -f1)

# For PBS: change to directory where job was submitted
# (otherwise job would run in HOME)
if [[ -n "\${PBS_O_WORKDIR}" ]] ; then
  cd "\${PBS_O_WORKDIR}"
fi

venum_lhost=$use_venum_lhost # Number of VE per logical host

# set path variables used in add_run_routines (not used on RCL, thus set to null)
current_status_file=/dev/null #\$PWD/${job_name}.status
final_status_file=/dev/null #\$PWD/${job_name}.final_status

# path to model binary, including the executable:
#MODEL=\${PBS_O_WORKDIR}/../../build/VE/bin/icon
#MODEL_SCAL=\${PBS_O_WORKDIR}/../../build/VH/bin/icon

# how to start the icon model on the NEC
# --------------------------------------
MODEL_SCAL="${MODEL_SCAL}"
EOF
cat "${use_builddir}/run/add_START_MODEL_function" >> "$output_script"

}

#-----------------------------------------------------------------------------
set_run_target_gpnl()
{
  # gpnl*.dwd.de GPU Node with Linux (NQSV)
    icon_data_rootFolder=${icon_data_rootFolder:="/hpc/rwork0/routfor/test/icon"}

    # Use same nproma for GPU and CPU for development purposes
    set_default use_nproma 8000
    set_default use_nproma_sub 800
    set_default use_nblocks_c 0
    set_default use_ecrad_isolver 2 # McICA for OpenACC
    use_nodes=1

    # absolute path to cdo
    cdo="/hpc/sw/cdo/2.2.1/x86/gnu/bin/cdo"

    if [[ "$use_mpi" == "yes" ]]
    then
        set_default use_mpi_procs_pernode 8
        set_default use_num_io_procs 3
    fi

    # no OpenMP Threads
    set_default use_openmp_threads 1

    cat >> "$output_script" <<EOF
#PBS -q ${use_queue:-gp_norm_all}
#PBS -l elapstim_req=00:20:00
#PBS -v gpu_per_lhost=${use_mpi_procs_pernode}
#PBS --gpunum-lhost=\${gpu_per_lhost}
#PBS --cpunum-lhost=${use_mpi_procs_pernode}
# Request 100Gb of memory per GPU (100Gb out ouf 240Gb possible)
#PBS -l memsz_job=${use_memory:-\${gpu_per_lhost\}00Gb}
#PBS -l vmemsz_job=${use_memory:-\${gpu_per_lhost\}00Gb}
#PBS -l vmemsz_prc=240Gb
#PBS -j o
#PBS -o LOG.${job_name}.o
#-----------------------------------------------------------------------------
EOF

    start_header

    cat >> "$output_script" <<EOF
set +x
ulimit -Sv unlimited

# OpenMPI get confused by this environment variable.
unset PMIX_INSTALL_PREFIX

if [[ -n "\${PBS_O_WORKDIR}" ]] ; then
  cd "\${PBS_O_WORKDIR}"
fi
EOF
    if [[ "$use_gpu" == "yes" ]]
    then
        echo "export ENABLE_NVIDIA_SMI_LOGGER=yes" >> "$output_script"
    fi
}
#-----------------------------------------------------------------------------
set_run_target_oflws()
{
  # *.dwd.de with Linux (workstations)
    icon_data_rootFolder=${icon_data_rootFolder:=""}

    set_default use_nproma 64
    set_default use_nproma_sub 64
    set_default use_nblocks_c 0
    use_nodes=1

    if [[ "$use_mpi" == "yes" ]]
    then
    set_default use_mpi_procs_pernode 4
    fi
    if [[ "$use_openmp" == "yes" ]]
    then
    set_default use_openmp_threads 2
    fi

    start_header
}

#-----------------------------------------------------------------------------
set_run_target_hpc()
{
    # xce.dwd.de
    icon_data_rootFolder=${icon_data_rootFolder:=""}

    set_default use_nproma 64
    set_default use_nproma_sub 64
    set_default use_nblocks_c 0
    set_default use_nodes 1
    set_default use_mpi_procs_pernode 1
    set_default use_openmp_threads 1

    cdo="/e/uhome/hanlauf/X86_64/bin/cdo"
    cdo_diff="diff"


    cat >> "$output_script" << EOF
#### BATCH_SYSTEM=PBS ####
#-----------------------------------------------------------------------------
#PBS -q lang
#PBS -j oe
#PBS -o LOG.$job_name.o
#PBS -l select=$use_nodes:ncpus=$use_mpi_procs_pernode
#PBS -m n
# ---------------------------
#-----------------------------------------------------------------------------
# for PBS change to directory where job was submitted
# (without this job is started in HOME)
if [[ -n \${PBS_O_WORKDIR} ]]
 then
  cd \${PBS_O_WORKDIR}
fi
export F_PROGINF=DETAIL
#-----------------------------------------------------------------------------
EOF

    start_header
}

set_run_target_alps_mch_gpu()
{
    rootFolder_prefix=/scratch/mch/jenkins/icon
    icon_data_rootFolder=${icon_data_rootFolder:=${rootFolder_prefix}/pool/data/ICON}

    testingFolder_prefix=/scratch/mch/icontest

    set_default use_nodes 1
    set_default use_nproma 0
    set_default use_nblocks_c 1
    set_default use_prefetch_procs 0
    set_default use_num_io_procs 1
    use_OMP_SCHEDULE="static,1"

    is_serial_run="no"

    # hardcode number of nodes
    set_default use_nodes 1
    set_default use_mpi_procs_pernode $((5+use_prefetch_procs))

    job_type=parallel
    tasks_per_node=$use_mpi_procs_pernode
    SBATCH_nodes="SBATCH --nodes=$use_nodes"
    SBATCH_ntasks_per_node="SBATCH --ntasks-per-node=${tasks_per_node}"

    if [[ "x$BB_SYSTEM" != "x" ]]
    then
        # this buildbot, do not use job id
        job_log_name="LOG.$job_name"
    else
        job_log_name="LOG.$job_name.%j"
    fi

    # add compute-sanitizer
    # can be enabled with exp.abc.run --compute-sanitizer
    use_mpi_startrun="${use_mpi_startrun} \${1-}"

    # ------------------------------------
    s_cpu_time=""
    if [[ "x$use_cpu_time" != "x" ]]
    then
      s_cpu_time="SBATCH --time=$use_cpu_time"
    fi
    if [[ "x$BB_SYSTEM" != "x" ]]
    then
        # this is buldbot, do not run more than 120 minutes
        limit_time_bb "02:00:00" "00:15:00"
        s_cpu_time="SBATCH --time=$use_cpu_time"
    fi

    # ------------------------------------

    use_nodes=${use_nodes}
    use_mpi_procs_pernode=${use_mpi_procs_pernode}

    set_default partition "short"

    cat >> "$output_script" << EOF
#-----------------------------------------------------------------------------

# alps_mch gpu batch job parameters
# ------------------------------
#SBATCH --job-name=$job_name
#SBATCH --output=$output_folder/$job_log_name.o
#SBATCH --error=$output_folder/$job_log_name.o
#SBATCH --chdir=$output_folder
#SBATCH --gres=gpu:4
#SBATCH --exclusive
#SBATCH --partition=$partition
$(if [[ "x$BB_SYSTEM" != 'x' ]]
 then
  printf "#SBATCH ${TIME_LIMITS}
";fi)
#$SBATCH_nodes
#$SBATCH_ntasks_per_node
#$s_cpu_time
EOF

    start_header
    cat >> "$output_script" <<EOF
EOF
}

#-----------------------------------------------------------------------------
set_run_target_alps_mch_cpu()
{
    rootFolder_prefix=/scratch/mch/jenkins/icon
    icon_data_rootFolder=${icon_data_rootFolder:=${rootFolder_prefix}/pool/data/ICON}

    testingFolder_prefix=/scratch/mch/icontest

    set_default use_nproma 8
    set default use_nproma_sub 8
    set_default use_nblocks_c 0
    set_default use_nodes 1
    set_default use_num_io_procs 1
    use_OMP_SCHEDULE="static,12"

    is_serial_run="no"
    if [[ $use_openmp == "yes" ]]
    then
        set_default use_mpi_procs_pernode 4
        set_default use_openmp_threads 32
    else
        set_default use_mpi_procs_pernode 64
    fi

    job_type=parallel
    tasks_per_node=$use_mpi_procs_pernode
    SBATCH_nodes="SBATCH --nodes=${use_nodes}"
    SBATCH_ntasks_per_node="SBATCH --ntasks-per-node=${tasks_per_node}"

    if [[ "x$BB_SYSTEM" != "x" ]]
    then
        # this buildbot, do not use job id
        job_log_name="LOG.$job_name"
    else
        job_log_name="LOG.$job_name.%j"
    fi
    # ------------------------------------
    s_cpu_time=""
    if [[ "x$use_cpu_time" != "x" ]]
    then
        s_cpu_time="SBATCH --time=$use_cpu_time"
    fi
    if [[ "x$BB_SYSTEM" != "x" ]]
    then
        # this is buildbot, do not run more than 15 minutes on Alps (SLURM delays)
        # Exception: member selection for generating tolerances and tolerance generation
        case $job_name in
            pp.select_members*)
                limit="01:00:00"
            ;;
            *)
                limit="00:40:00"
            ;;
        esac
        limit_time_bb "${limit}" "${limit}"
        s_cpu_time="SBATCH --time=$use_cpu_time"
    fi
    # ------------------------------------

    use_nodes=${use_nodes}
    use_mpi_procs_pernode=${use_mpi_procs_pernode}

    target_partition=pp-short
    # check if pp-short partition exists
    partition=`sinfo --noheader --partition $target_partition --format %R`

    if test -z $partition; then
        partition="normal"
    else
        # in case not normal partition is used: Check partition status
        status=`sinfo --noheader --partition $target_partition --format %a`
        if [[ "$status" != "up" ]]; then
            partition="normal"
        fi
    fi


    cat >> "$output_script" << EOF
#-----------------------------------------------------------------------------

# alps_mch cpu batch job parameters
# ------------------------------
#SBATCH --job-name=$job_name
#SBATCH --output=$output_folder/$job_log_name.o
#SBATCH --error=$output_folder/$job_log_name.o
#SBATCH --chdir=$output_folder
#SBATCH --partition=$partition
#SBATCH --exclusive
#SBATCH --hint=nomultithread
$(if [[ "x$BB_SYSTEM" != 'x' ]]
 then
  printf "#SBATCH ${TIME_LIMITS}
";fi)
#$SBATCH_nodes
#$SBATCH_ntasks_per_node
#$s_cpu_time
EOF

  start_header

  cat >> "$output_script" << EOF
EOF
}

set_run_target_tave_knl()
{
    rootFolder_prefix=/users/icontest
    icon_data_rootFolder=${icon_data_rootFolder:=${rootFolder_prefix}/pool/data/ICON}

    set_default use_nproma 16
    set_default use_nproma_sub 16
    set_default use_nblocks_c 0
    set_default use_nodes 1
    use_OMP_SCHEDULE="static,2"

    is_serial_run="no"
    if [[ $use_mpi == "yes" ]]
    then
	if [[ $use_openmp == "yes" ]]
	then
	    set_default use_mpi_procs_pernode 60
	    set_default use_openmp_threads 2
	else
	    set_default use_mpi_procs_pernode 60
	fi
    else
	if [[ $use_openmp == "yes" ]]
	then
	    set_default use_openmp_threads 60
	    set_default use_mpi_procs_pernode 1
	else
	    is_serial_run="yes"
	fi
    fi

    case _$use_compiler in
	"_intel" )
            use_mpi_startrun="$use_mpi_startrun -npernode \$mpi_procs_pernode -cpus-per-proc \$OMP_NUM_THREADS --bind-to-core --verbose --report-bindings"
            add_free_var KMP_AFFINITY "verbose,granularity=core,compact,1,1"
            add_free_var KMP_LIBRARY "turnaround"
            add_free_var KMP_KMP_SETTINGS "1"
            add_free_var OMP_WAIT_POLICY "active"
	    ;;
    esac

    if [[ "$is_serial_run" == "yes"  ]]
    then
	use_nodes=1
	tasks_per_node=1
	SBATCH_ntasks_per_node=""
    else
	job_type=parallel
	tasks_per_node=$use_mpi_procs_pernode
	SBATCH_ntasks_per_node="SBATCH --ntasks-per-node=32"
    fi

    if [[ "x$BB_SYSTEM" != "x" ]]
    then
	# this buildbot, do not use job id
	job_log_name="LOG.$job_name"
    else
	job_log_name="LOG.$job_name.%j"
    fi
    # ------------------------------------
    s_cpu_time=""
    if [[ "x$use_cpu_time" != "x" ]]
    then
	s_cpu_time="SBATCH --time=$use_cpu_time"
    fi
    if [[ "x$BB_SYSTEM" != "x" ]]
    then
        limit_time_bb "00:45:00" "00:45:00"
        s_cpu_time="SBATCH --time=$use_cpu_time"
    fi
    # ------------------------------------

    use_nodes=${use_nodes}
    use_mpi_procs_pernode=${use_mpi_procs_pernode}

    cat >> "$output_script" << EOF
#-----------------------------------------------------------------------------
# tave batch job parameters
#--------------------------
#SBATCH --account=csstaff
$(if [[ "x$BB_SYSTEM" != 'x' ]]
 then
  printf "#SBATCH ${TIME_LIMITS}
";fi)
#SBATCH --job-name=$job_name
#SBATCH --output=$output_folder/$job_log_name.o
#SBATCH --error=$output_folder/$job_log_name.o
#SBATCH --chdir=$output_folder
#SBATCH --nodes=$use_nodes
#$SBATCH_ntasks_per_node
#$s_cpu_time
EOF
    start_header
}

#-----------------------------------------------------------------------------
set_run_target_euler()
{
  # euler.ethz.ch
  typeset poolFolder_prefix=/cluster/work/climate
  icon_data_rootFolder=${icon_data_rootFolder:=${poolFolder_prefix}/icon_input}
  icon_data_poolFolder=${icon_data_poolFolder:=${poolFolder_prefix}/icon_input}

  testingFolder_prefix=/cluster/work/climate/icon_testing_input

  set_default use_nproma 64
  set_default use_rrtmgp_columns_chunk 64
  set_default use_nodes 1
  set_default use_openmp_threads 1
  set_default use_mpi_procs_pernode 12

  job_log_name="LOG.$job_name.%J"
  s_cpu_time="SBATCH --time=00:15:00"
  SBATCH_nodes="SBATCH --ntasks=${use_mpi_procs_pernode}"

  cat >> "$output_script" << EOF
# ---------------------------
#### BATCH_SYSTEM=SBATCH ####
#-----------------------------------------------------------------------------
#SBATCH --job-name=$job_name
#SBATCH --output=$output_folder/$job_log_name.o
#SBATCH --error=$output_folder/$job_log_name.o
#SBATCH --chdir=$output_folder
#SBATCH --constraint=EPYC_7H12
$(if [[ "x$BB_SYSTEM" != 'x' ]]
 then
  printf "#SBATCH ${TIME_LIMITS}
";fi)
#$SBATCH_nodes
#$s_cpu_time
# ---------------------------
EOF

  # create header
  start_header

}

#-----------------------------------------------------------------------------
set_run_target_pacluster()
{
    set_default use_nproma 32
    set_default use_nproma_sub 32
    set_default use_nblocks_c 0
    set_default use_nodes 1
    set_default use_cpu_time 04:00:00

    if [[ $use_openmp == "yes" ]]
    then
	use_mpi="no"
	set_default use_openmp_threads 24
	queue=""
	resources=""
    elif [[ $use_mpi == "yes" ]]
    then
	set_default use_mpi_procs_pernode 12
	if [[ "$use_mpi_procs_pernode" == "1" ]]
	then
	    queue=""
	    resources=""
	else
	    queue=""
	    resources=""
	fi
    else
	queue="-q s8"
	resources=""
    fi

    cat >> "$output_script" << EOF
#############################################################################
# DLR Linux Cluster batch job parameters
# EMBEDDED FLAGS FOR PBS Pro
#############################################################################
################# shell to use
#PBS -S /bin/ksh
################# export all  environment  variables to job-script
#PBS -V
################# name of the log file
#PBS -o ./${job_name}.\${PBS_JOBID}.log
################# join standard and error stream (oe, eo) ?
#PBS -j oe
################# do not rerun job if system failure occurs
#PBS -r n
################# send e-mail when [(a)borting|(b)eginning|(e)nding] job
### #PBS -m ae
### #PBS -M my_userid@my_institute.my_toplevel_domain
################# always ppn=12 tasks per node!
#PBS -l nodes=$use_nodes:ppn=$use_mpi_procs_pernode
#PBS -l walltime=$use_cpu_time
#############################################################################
#-----------------------------------------------------------------------------
# for PBS change to directory where job was submitted
# (without this job is started in HOME)
if [[ -n \${PBS_O_WORKDIR} ]]
 then
  cd \${PBS_O_WORKDIR}
fi
#-----------------------------------------------------------------------------
EOF

    start_header
}

#-----------------------------------------------------------------------------
set_run_target_hk()
{
    icon_data_rootFolder=${icon_data_rootFolder:=/hkfs/home/dataset/datasets/icon/pool/data/ICON}

    if [[ "$use_gpu" == "yes" ]]; then
        set_default use_queue accelerated
        set_default use_nproma 8192
        set_default use_nproma_sub 512
        set_default use_ecrad_isolver 2 # McICA for OpenACC
        if [[ $use_mpi == "yes" ]]; then
            set_default use_mpi_procs_pernode 4
        fi
    else
        set_default use_queue cpuonly
        set_default use_nproma 32
        set_default use_nproma_sub 8
        if [[ $use_mpi == "yes" ]]; then
            set_default use_mpi_procs_pernode 76
        fi
        if [[ $use_openmp == "yes" ]]; then
            set_default use_openmp_threads 2
        else
            set_default use_openmp_threads 1
        fi
    fi
    set_default use_nodes 1
    set_default use_num_io_procs 1
    set_default use_cpu_time 01:00:00

    if [[ -n "$BB_SYSTEM" ]]; then
        # this buildbot, do not use job id
        job_log_name="LOG.$job_name"

        use_queue="dev_${use_queue}"

        limit_time_bb "00:45:00" "00:45:00"
    else
        job_log_name="LOG.$job_name.%j"
    fi

    cdo='cdo'
    cdo_diff='diffn'

    cat >> "$output_script" << EOF
#SBATCH --output=$output_folder/$job_log_name.o
#SBATCH --error=$output_folder/$job_log_name.o
#SBATCH --chdir=$output_folder
#SBATCH --job-name=$job_name
#SBATCH --nodes=${use_nodes}
#SBATCH --account=hk-project-glori-init
#SBATCH --ntasks-per-node=$use_mpi_procs_pernode
#SBATCH --time=$use_cpu_time
#SBATCH --partition=$use_queue
EOF

    if [[ "x$BB_SYSTEM" != 'x' ]]; then
        echo "#SBATCH --qos cx" >> "$output_script"
        echo "#SBATCH ${TIME_LIMITS}" >> "$output_script"
    fi

    if [[ "$use_gpu" == "yes" ]]; then
        echo '#SBATCH --gres=gpu:4' >> "$output_script"
    fi

    start_header

    cat >> "$output_script" << EOF
export ECCODES_DEFINITION_PATH=/hkfs/home/dataset/datasets/icon/bacy_data/definitions/2.32.0-2/definitions.edzw:/hkfs/home/dataset/datasets/icon/bacy_data/definitions/2.32.0-2/definitions
EOF
    if [[ "$use_compiler" == "pgi" ]]; then
        echo 'export NVCOMPILER_TERM=trace' >> "$output_script"
        if [[ "$use_compiler_version" == 'nv:24.7.0' ]]; then
            echo "source /hkfs/home/dataset/datasets/icon/sw/buildenv/nvidia-24.7/environment.rc" >> "$output_script"
        fi
    fi
}
#-----------------------------------------------------------------------------
set_run_target_juwels()
{
    icon_data_rootFolder=${icon_data_rootFolder:=/p/data1/slmet/model_data/ICON-BB}

    set_default use_nproma 8
    set_default use_nproma_sub 8
    set_default use_nblocks_c 0
    set_default use_nodes 1
    set_default use_num_io_procs 1
    set_default use_cpu_time 01:00:00

    if [[ $use_mpi == "yes" ]]; then
	if [[ $use_openmp == "yes" ]]; then
		set_default use_mpi_procs_pernode 24
	else
		set_default use_mpi_procs_pernode 48
	fi
    fi
    if [[ $use_openmp == "yes" ]]; then
        set_default use_openmp_threads 2
    else
        set_default use_openmp_threads 1
    fi

    if [[ -n "$BB_SYSTEM" ]]; then
        job_log_name="LOG.$job_name"
        use_cpu_time="00:15:00"
    else
        job_log_name="LOG.$job_name.%j"
    fi

    if [ "$use_gpu" == "yes" ]; then
      set_default use_queue 'booster'
      set_default use_nproma 0
      set_default use_nblocks_c 1
    else
      set_default use_queue 'batch'
      set_default use_nproma 8
      set_default use_nblocks_c 0
    fi

    cdo='cdo'
    cdo_diff='diffn'

    cat >> "$output_script" << EOF
#SBATCH --output=$output_folder/$job_log_name.o
#SBATCH --error=$output_folder/$job_log_name.o
#SBATCH --chdir=$output_folder
#SBATCH --job-name=$job_name
#SBATCH --nodes=${use_nodes}
#SBATCH --partition=${use_queue}
$(if [[ "x$BB_SYSTEM" != 'x' ]]
 then
  printf "#SBATCH ${TIME_LIMITS}
";fi)
#SBATCH --account=exaww
#SBATCH --ntasks-per-node=$use_mpi_procs_pernode
#SBATCH --time=$use_cpu_time

$use_build_env
EOF

    start_header

}
#-------------------------------------------------------------------------------
set_run_target_jupiter()
{
    icon_data_rootFolder=${icon_data_rootFolder:=/p/data1/slmet/model_data/ICON}

    set_default use_nodes 1
    set_default use_num_io_procs 1
    set_default use_cpu_time 01:00:00
    set_default use_queue 'booster'

    if [[ -n "$BB_SYSTEM" ]]; then
        job_log_name="LOG.$job_name"
        use_cpu_time="00:15:00"
    else
        job_log_name="LOG.$job_name.%j"
    fi

    if [ "$use_gpu" == "yes" ]; then
      set_default use_nproma 0
      set_default use_nblocks_c 1
      set_default use_mpi_procs_pernode 4
    else
      set_default use_nproma 8
      set_default use_nblocks_c 0
      set_default use_mpi_procs_pernode 112
    fi

    cdo='cdo'
    cdo_diff='diffn'

    cat >> "$output_script" << EOF
#SBATCH --output=$output_folder/$job_log_name.o
#SBATCH --error=$output_folder/$job_log_name.o
#SBATCH --chdir=$output_folder
#SBATCH --job-name=$job_name
#SBATCH --nodes=${use_nodes}
#SBATCH --partition=${use_queue}
$(if [[ "x$BB_SYSTEM" != 'x' ]]
 then
  printf "#SBATCH ${TIME_LIMITS}
";fi)
#SBATCH --account=jureap22
#SBATCH --ntasks-per-node=$use_mpi_procs_pernode
#SBATCH --time=$use_cpu_time

$use_build_env
EOF

    start_header

}
#-----------------------------------------------------------------------------
set_run_target_leo()
{
    icon_data_rootFolder=${icon_data_rootFolder:=/leonardo_work/DE360_GLORI/pool/data/ICON/}

    if [[ "$use_gpu" == "yes" ]]; then
        set_default use_nproma 8192
        set_default use_nproma_sub 512
        set_default use_ecrad_isolver 2 # McICA for OpenACC
        if [[ $use_mpi == "yes" ]]; then
            set_default use_mpi_procs_pernode 4
        fi
    else
        set_default use_nproma 8
        set_default use_nproma_sub 8
        if [[ $use_mpi == "yes" ]]; then
            set_default use_mpi_procs_pernode 32
        fi
        if [[ $use_openmp == "yes" ]]; then
            set_default use_openmp_threads 2
        else
            set_default use_openmp_threads 1
        fi
    fi
    set_default use_nblocks_c 0
    set_default use_nodes 1
    set_default use_num_io_procs 1
    set_default use_cpu_time 01:00:00


    if [[ -n "$BB_SYSTEM" ]]; then
        job_log_name="LOG.$job_name"
        limit_time_bb "00:45:00" "00:45:00"
    else
        job_log_name="LOG.$job_name.%j"
    fi

    cdo='cdo'
    cdo_diff='diffn'

    cat >> "$output_script" << EOF
#SBATCH --output=$output_folder/$job_log_name.o
#SBATCH --error=$output_folder/$job_log_name.o
#SBATCH --chdir=$output_folder/
#SBATCH --job-name=$job_name
#SBATCH --nodes=${use_nodes}
#SBATCH --partition=boost_usr_prod
$(if [[ "x$BB_SYSTEM" != 'x' ]]
 then
  printf "#SBATCH ${TIME_LIMITS}
";fi)
#SBATCH --account=DE360_GLORI
#SBATCH --ntasks-per-node=$use_mpi_procs_pernode
#SBATCH --time=$use_cpu_time
EOF
    if [[ "$use_gpu" == "yes" ]]; then
    cat >> "$output_script" << EOF
#SBATCH --gres=gpu:4
EOF
    fi

    start_header

    cat >> "$output_script" << EOF
export PATH=/leonardo/prod/spack/03/install/0.19/linux-rhel8-icelake/gcc-11.3.0/cdo-2.1.0-2ltng25xbm7yakxoon4dyehxev5ynaw2/bin/:\$PATH
export ECCODES_DEFINITION_PATH=/leonardo_scratch/fast/DE360_GLORI/definitions/2.32.0-2/definitions.edzw:/leonardo_scratch/fast/DE360_GLORI/definitions/2.32.0-2/definitions
export NVCOMPILER_TERM=trace
export OMPI_MCA_btl=self,tcp
EOF
}
set_run_target_santis_cpu()
{
    icon_data_rootFolder="/capstor/store/cscs/userlab/cws01/pool/data/ICON/"
    testingFolder_prefix="/capstor/store/cscs/userlab/cws01/ci"

    set_default use_nodes 1
    set_default use_nproma 48
    set_default use_nproma_sub 48
    set_default use_nblocks_c 0
    set_default use_prefetch_proc 0
    set_default use_num_io_procs 0
    set_default use_account_no "$(id -gn)"
    use_OMP_SCHEDULE="static,1"

    is_serial_run="no"

    # only single processor runs work
    set_default use_mpi_procs_pernode 280

    job_type=parallel
    tasks_per_node=$use_mpi_procs_pernode
    SBATCH_nodes="SBATCH --nodes=$use_nodes"
    SBATCH_ntasks_per_node="SBATCH --ntasks-per-node=${tasks_per_node}"

    job_log_name="LOG.$job_name.%j"

    s_cpu_time=""
    if [[ "x$use_cpu_time" != "x" ]]
    then
      s_cpu_time="SBATCH --time=$use_cpu_time"
    fi

    use_nodes=${use_nodes}
    use_mpi_procs_pernode=${use_mpi_procs_pernode}

    cat >> "$output_script" << EOF
#=============================================================================

# santis cpu batch job parameters
# ------------------------------
#SBATCH --job-name=$job_name
#SBATCH --output=$output_folder/$job_log_name.o
#SBATCH --error=$output_folder/$job_log_name.o
#$SBATCH_nodes
#$SBATCH_ntasks_per_node
#$s_cpu_time
EOF

    start_header
    cat >> "$output_script" <<EOF
export CUDA_BUFFER_PAGE_IN_THRESHOLD_MS=0.001
export FI_CXI_SAFE_DEVMEM_COPY_THRESHOLD=0
export FI_CXI_RX_MATCH_MODE=software
export FI_MR_CACHE_MONITOR=disabled
export MPICH_GPU_SUPPORT_ENABLED=1
export NVCOMPILER_ACC_DEFER_UPLOADS=1
export NVCOMPILER_TERM=trace

EOF
}

set_run_target_santis_gpu()
{
    icon_data_rootFolder="/capstor/store/cscs/userlab/cws01/pool/data/ICON/"
    testingFolder_prefix="/capstor/store/cscs/userlab/cws01/ci"

    set_default use_nodes 1
    set_default use_nproma 0
    set_default use_nproma_sub 8000
    set_default use_nblocks_c 1
    set_default use_prefetch_proc 0
    set_default use_num_io_procs 0
    set_default use_account_no "$(id -gn)"
    use_OMP_SCHEDULE="static,1"

    case $job_name in
        *dwd_run_ICON_09_R2B4N5_EPS.run*|*dwd_run_ICON_09_R2B4N5_EPS_member_id*)
                use_nproma=8000
                use_nproma_sub=0
            ;;
    esac

    is_serial_run="no"

    # hardcode number of nodes
    set_default use_mpi_procs_pernode 4

    job_type=parallel
    tasks_per_node=$use_mpi_procs_pernode
    SBATCH_nodes="SBATCH --nodes=$use_nodes"
    SBATCH_ntasks_per_node="SBATCH --ntasks-per-node=${tasks_per_node}"

    job_log_name="LOG.$job_name.%j"

    s_cpu_time=""
    if [[ "x$use_cpu_time" != "x" ]]
    then
      s_cpu_time="SBATCH --time=$use_cpu_time"
    fi

    use_nodes=${use_nodes}
    use_mpi_procs_pernode=${use_mpi_procs_pernode}

    cat >> "$output_script" << EOF
#=============================================================================

# santis gpu batch job parameters
# ------------------------------
#SBATCH --job-name=$job_name
#SBATCH --output=$output_folder/$job_log_name.o
#SBATCH --error=$output_folder/$job_log_name.o
#$SBATCH_nodes
#$SBATCH_ntasks_per_node
#$s_cpu_time
EOF

    start_header
    cat >> "$output_script" <<EOF
export CUDA_BUFFER_PAGE_IN_THRESHOLD_MS=0.001
export FI_CXI_SAFE_DEVMEM_COPY_THRESHOLD=0
export FI_CXI_RX_MATCH_MODE=software
export FI_MR_CACHE_MONITOR=disabled
export MPICH_GPU_SUPPORT_ENABLED=1
export NVCOMPILER_ACC_DEFER_UPLOADS=1
export NVCOMPILER_TERM=trace

# Enable CUDA graphs support. Setting this is harmless when the graphs are not used
export NVCOMPILER_ACC_USE_GRAPH=1

EOF
}


#=============================================================================
create_target_header()
{
    cdo="cdo"
    cdo_diff="diffn"
    use_OMP_SCHEDULE="static"

    if [[ "$use_mpi" == "no" ]]
    then
	use_nodes=1
	use_mpi_procs_pernode=1
    fi

    if [[ "$use_openmp" == "no" ]]
    then
	use_openmp_threads=1
    fi

    use_prefetch_procs=0
 	case $job_name in
        *dwd_run_ICON_09_R2B4N5_EPS.run*|*dwd_run_ICON_09_R2B4N5_EPS_member_id*)
            case $use_target in
                alps_mch_gpu*)
                    use_nodes=1
                    use_mpi_procs_pernode=4
                    use_nproma=8000
                    use_nblocks_c=0
                ;;
            esac
            ;;
        *mch_ch_lowres.run|*mch_ch_lowres_member_id*)
            case $use_target in
                alps_mch_gpu*)
                    use_nodes=1
                    use_mpi_procs_pernode=4
                ;;
            esac
            ;;
        *mch_*_small.run|*mch_*_small_member_id*)
            case $use_target in
                alps_mch_gpu*)
                    use_nodes=1
                    use_mpi_procs_pernode=4
                ;;
            esac
            ;;
        *atm_tracer_Hadley*|*atm_cph_nest*)
            case $use_target in
                bullx_gpu*)
                    use_nproma=8000
                    use_nblocks_c=0
                ;;
            esac
            ;;
        *mch_opr_r04b07.run|*mch_opr_r04b07_member_id*.run)
            use_nodes=1
            use_mpi_procs_pernode=3
            case $use_target in
                *_cpu*)
                    use_nproma=4250
                ;;
            esac
            ;;
        *mch_opr_r04b07_lhn_00.run|*mch_opr_r04b07_lhn_00_member_id*.run)
            use_nodes=2
            use_mpi_procs_pernode=3
            ;;
        *mch_opr_r04b07_nest.run|*mch_opr_r04b07_nest_member_id*.run)
            use_nodes=1
            use_mpi_procs_pernode=4
            use_nproma=1600
            ;;
        *mch_opr_r04b07_performance.run|*mch_opr_r04b07_performance_member_id*.run)
            use_nodes=1
            case $use_target in
                *_cpu*)
                    use_mpi_procs_pernode=12
                    use_nproma=16
                ;;
                *_gpu*)
                    use_mpi_procs_pernode=2
                    # no nproma needed since this uses nlbocks_c anyway
                ;;
            esac
            ;;
        *mch_icon-ch1.run)
            case $use_target in
                alps_mch_gpu*)
                    use_nodes=2
                    use_mpi_procs_pernode=5
                    use_nproma_sub=6054
                    partition="normal"
                ;;
                alps_mch_cpu*)
                    use_nodes=4
                    use_mpi_procs_pernode=128
                ;;
            esac
            ;;
        *mch_kenda-ch1.run|*mch_kenda-ch1_member_id*.run)
            case $use_target in
                alps_mch_cpu*)
                    use_nodes=2
                    use_mpi_procs_pernode=128
                    partition="pp-short"
                ;;
                alps_mch_gpu*)
                    use_nodes=1
                    use_mpi_procs_pernode=6
                    use_nproma_sub=1128
                    partition="normal"
                ;;
            esac
            ;;
        *mch_icon-ch2.run)
            case $use_target in
                alps_mch_gpu*)
                    use_nodes=1
                    use_mpi_procs_pernode=6
                    use_nproma_sub=17757
                    partition="normal"
                ;;
            esac
            ;;
        *mch_ch_r04b09_dace.run|*mch_ch_r04b09_dace_seed_*.run)
            use_nodes=2
            case $use_target in
                alps_mch*)
                    use_mpi_procs_pernode=4 # Use 4 procs pernode for GPU as well until bug is fixed
                    ;;
            esac
            ;;
	    *ocean_omip_R2B4_V*)
            case $use_target in
                bullx_gpu*)
                    use_nproma=8000
                    use_nblocks_c=0
                ;;
            esac
            ;;
        *run_ICON_01_R3B9_lam*)
            case $use_target in
                rcl)
                    use_nodes=4
                    use_venum_lhost=4
                    use_cpunum_job=6
                    ;;
            esac
            ;;
        *run_ICON_02_R2B13_lam*)
            case $use_target in
                rcl)
                    use_nodes=1
                    use_venum_lhost=1
                    use_cpunum_job=2
                    ;;
            esac
            ;;
        *run_ICON_03_R19B7N8-ID2_ID1_lam*)
            use_memory="24gb"
            case $use_target in
                rcl)
                    use_nodes=6
                    use_venum_lhost=$use_nodes # all vector engines must be on one host for NEC.mem_check.py
                    use_cpunum_job=6
                    ;;
            esac
            ;;
        *run_ICON_05_R02B06N08_ifsinit_restarttest*)
            case $use_target in
                rcl)
                    use_nodes=4
                    use_venum_lhost=2
                    use_cpunum_job=5
                    ;;
            esac
            ;;
        *run_ICON_06_R02B06N07_UPATMO_ifsinit_restarttest*)
            case $use_target in
                rcl)
                    use_nodes=6
                    use_venum_lhost=2
                    use_cpunum_job=2
                    add_free_var VE_FORT_NML_REPEAT_FORM NO
                    ;;
            esac
            ;;
        *run_ICON_07_R02B04N06M_restarttest*)
            case $use_target in
                rcl)
                    use_nodes=2
                    use_venum_lhost=2
                    use_cpunum_job=3
                    ;;
            esac
            ;;
        *run_ICON_08_R19B7-ID2_oper*)
            case $use_target in
                rcl)
                    use_nodes=4
                    use_venum_lhost=2
                    use_cpunum_job=4
                    ;;
            esac
            ;;
        *run_ICON_09_R2B6N7_oper_EPS*)
            case $use_target in
                rcl)
                    use_nodes=3
                    use_venum_lhost=$use_nodes # all vector engines must be on one host for NEC.mem_check.py
                    use_cpunum_job=5
                    use_nproma=504 # fails with regular nproma=752 at a certain allocation
                    add_free_var NMPI_COLLORDER ON
                    ;;
            esac
            ;;
        *run_ICON_10_R3B7N8_oper_mfr*)
            use_memory="24gb"
            case $use_target in
                rcl)
                    use_nodes=32
                    use_venum_lhost=8
                    use_cpunum_job=6
                    ;;
            esac
            ;;
        *run_ICON_11_R3B08_lam_initmode7_restarttest*)
            case $use_target in
                rcl)
                    use_nodes=1
                    use_venum_lhost=1
                    use_cpunum_job=4
                    use_omp_stacksize=3G
                    ;;
            esac
            ;;
        *run_ICON_12_R3B08_lam_initmode4*)
            case $use_target in
                rcl)
                    use_nodes=1
                    use_venum_lhost=1
                    use_cpunum_job=4
                    ;;
            esac
            ;;
        *run_ICON_13_R2B08-dkltest*)
            case $use_target in
                rcl)
                    use_nodes=2
                    use_venum_lhost=2
                    use_cpunum_job=6
                    ;;
            esac
            ;;
        *run_ICON_14_R2B6N7_oper_IAU_and_restarttest*)
            case $use_target in
                rcl)
                    use_nodes=4
                    use_venum_lhost=$use_nodes # all vector engines must be on one host for NEC.mem_check.py
                    use_cpunum_job=3
                    use_omp_stacksize=3G
                    ;;
            esac
            ;;
        *run_ICON_15_R19B7-ID2_ass*)
            case $use_target in
                rcl)
                    use_nodes=6
                    use_venum_lhost=3
                    use_cpunum_job=5
                    ;;
            esac
            ;;
        *run_ICON_16_R19B7-ID2_ass*)
            case $use_target in
                rcl)
                    use_nodes=6
                    use_venum_lhost=3
                    use_cpunum_job=5
                    ;;
            esac
            ;;
        *run_ICON_27_R2B6_mvstream_qbudget*)
            case $use_target in
                rcl)
                    use_nodes=2
                    use_venum_lhost=2
                    use_cpunum_job=5
                    use_nproma=504
                    add_free_var NMPI_COLLORDER ON
                    ;;
            esac
            ;;
        *run_ICON_17_R2B4_AO_coupled*)
            case $use_target in
                rcl)
                    use_nodes=4
                    use_venum_lhost=2
                    use_cpunum_job=3
#                    elapstim_req=01:00:00 # this variable does not exist yet
                    ;;
            esac
            ;;
        *run_ICON_20_R2B4_R2B6_AO_coupled*)
            case $use_target in
                rcl)
                    use_nodes=4
                    use_venum_lhost=2
                    use_cpunum_job=3
#                    elapstim_req=01:00:00 # this variable does not exist yet
                    ;;
            esac
            ;;
        *run_ICON-ART_01_R3B08_lam_initmode7_pollen*)
            use_memory="18gb"
            case $use_target in
                hk)
                    use_nodes=4
                    ;;
                rcl)
                    use_nodes=2
                    use_venum_lhost=2
                    use_cpunum_job=6
#                    elapstim_req=43300 # this variable does not exist yet
                    ;;
            esac
            ;;
        *run_ICON_23_R2B4_atmo_waves_coupled*|*run_ICON_28_R2B6_IAU_atmo_waves_*_cpl*)
            case $use_target in
                rcl)
                    use_nodes=4
                    use_venum_lhost=2
                    use_cpunum_job=3
                    ;;
            esac
            ;;
        *run_ICON_24_R19B7_RUC_ass*)
            case $use_target in
                rcl)
                    use_nodes=6
                    use_venum_lhost=3
                    use_cpunum_job=5
                    ;;
            esac
            ;;
        *run_ICON_25_R19B7_RUC_fc*)
            case $use_target in
                rcl)
                    use_nodes=6
                    use_venum_lhost=3
                    use_cpunum_job=5
                    ;;
            esac
            ;;
        *run_ICON_26_R2B4_R2B6_AO-HD_coupled*)
            case $use_target in
                rcl)
                    use_nodes=4
                    use_venum_lhost=2
                    use_cpunum_job=6 # 2x proc0 + 2x2 IO
                    ;;
            esac
            ;;
        *run_ICON_29_R2B5_R2B6_AO_coupled_gribInitOce*)
            case $use_target in
                rcl)
                    use_nodes=8
                    use_venum_lhost=2
                    use_cpunum_job=12 # 2x proc0 + 2 atm-IO + 8 oce-IO
                    ;;
            esac
            ;;
        *atm_nwp_jsbach_test*)
            use_memory="18gb"
            ;;
        *land*sitelevel*)
	    # ICON-Land experiments on one or more single sites (cells)
            # These run only on one MPI proc, on a grid with one cell and need the bash shell
            use_nodes=1
            use_tasks=1
            #use_mpi_procs_pernode=1
            use_nproma=1
            use_shell="/bin/bash"
            use_cpu_time="00:10:00"
            case $use_target in
                bull_milan)
                    use_queue="shared"
                ;;
            esac
            case $job_name in
                *land*iqj*sitelevel*)
                # when running with quincy configuration in land mode -- in addition -- memory and cpu time need to be increased
                use_memory="2gb"
                use_cpu_time="00:29:00"
                ;;
            esac
            case $job_name in
                *land*sitelevel*test)
                # test scripts only run for a couple of days and need less time
                use_cpu_time="00:02:00"
                ;;
            esac
            ;;
        *land_*_test*)
            case $use_target in
                *_gpu*)
                    use_num_io_procs=0
                ;;
            esac
            ;;
        *aes_bubble*)
            case $use_target in
            bull_milan*)
                use_queue=shared
                use_tasks=16
                ;;
            esac
            ;;
        *art_oem*.run)
            use_nodes=4
            case $use_target in
                *_cpu*)
                    use_mpi_procs_pernode=12
                    use_nproma=16
                ;;
                *_gpu*)
                    use_mpi_procs_pernode=1
                    # no nproma needed since this uses nlbocks_c anyway
                ;;
            esac
            ;;
        *ICON_CLM*)
            case $use_target in
            bullx_gpu*)
                use_mpi_procs_pernode=4
                ;;
            esac
            ;;
        *atm_msgwam_gc*)
            use_nodes=2
            ;;
        esac

    # set more default values
    set_default use_shell "/bin/bash"
    set_default use_memory_model "default"

    cat > "$output_script" << EOF
#! $use_shell
EOF
    set_run_target_${use_target}
}
