#!/bin/ksh

# ICON
#
# ------------------------------------------
# Copyright (C) 2004-2026, DWD, MPI-M, DKRZ, KIT, ETH, MeteoSwiss
# Contact information: icon-model.org
# See AUTHORS.TXT for a list of authors
# See LICENSES/ for license information
# SPDX-License-Identifier: BSD-3-Clause
# ------------------------------------------

#-------------------------------------------------------------------------------------=
# ICON testbed
# Creates the setup for a given target
#
# Leonidas Linardakis, MPI-M, 2010-11-24
#
#-------------------------------------------------------------------------------------=

#-------------------------------------------------------------------------------------
set_target_machine_supermuc()
{
  # default compile values
  use_fortran=${use_fortran:="intel"}
  optimize_level=${optimize_level:="std"}
  optimize_level=${optimize_level:="high"}
  debug_mode=${debug_mode:="no"}
  trace_mode=${trace_mode:="no"}

  # default paralleization values
  use_mpi=${use_mpi:="yes"}
  use_openmp=${use_openmp:="yes"}

  # system parameters
  use_shell="/bin/bash"
  use_load_profiles="/etc/profile /etc/profile.d/modules.sh"
  use_make_command="make -j 16"
  use_submit="llsubmit"
  use_sync_submit="llsubmit -s"
}

#-------------------------------------------------------------------------------------
set_standard_lib_paths_supermuc()
{
 # the modules must be load before calling this method

 # NETCDF_INC is a system enviroment variable set by the load modules
 #NETCDF_LIB is a system enviroment variable, altenrativle use NETCDF_F90_LIB
 NETCDF_LIB=$NETCDF_F90_LIB
 #MPI_INC is a system enviroment variable set by the load modules
 # MPI_INC="-I/lrz/sys/parallel/mpi.ibm/PMR-fixes/include/ibmmpi/thread64"
  # MPI_LIB=$MPI_F90_LIB
  # this is for the intel mpi, mpi_mt has to be loaded manually
  MPI_LIB="-ldl -L/lrz/sys/intel/mpi_40_3_008/lib64 -Xlinker --enable-new-dtags -Xlinker -rpath -Xlinker /lrz/sys/intel/mpi_40_3_008/lib64 -Bstatic -lmpi_mt -lmpi -lmpigf -lmpigi -Bdynamic -lpthread -lrt"
  # MPI_LIB=""

#   IHPCT_BASE="/usr/lpp/ppe.hpct"
#   PROFILE_LIB="-L$IHPCT_BASE/lib64 -lhpc -lm -lpmapi"
#   PROFILE_INCLUDE="-I$IHPCT_BASE/include"
}

#-------------------------------------------------------------------------------------
create_target_setup_supermuc_intel()
{
  use_config_h="config_linux_std.h"

  # create load modules include script
  use_load_modules_script="$base_folder/config/load_modules_script"
  cat > $use_load_modules_script << EOF
#--------------------------------
. /etc/profile
. /etc/profile.d/modules.sh
module unload  mpi.altix mpi.mpt mpi.ibm mpi.parastation mpi.mpich2 poe
module load  fortran/intel/12.1
#module unload mpi.ibm
module load mpi.intel/4.0
module load netcdf
module show mpi_pinning/hybrid_socket
module add  mpi_pinning/hybrid_socket
module add  lrztools
module list
#--------------------------------
EOF
  load_modules

  set_fortran_target_linux_${use_fortran}
  FC=mpif90
#  HI_OPT_FLAGS="$HI_OPT_FLAGS  -fma -xSSE4.2"
#  STD_OPT_FLAGS="$STD_OPT_FLAGS -mt-mpi -static-mpi -xSSE4.2 -ip -ipo"
#  STD_OPT_FLAGS="$STD_OPT_FLAGS -xSSE4.2-ip -ipo"
#   OMP_FLAGS="$OMP_FLAGS -mt_mpi"
   OMP_FLAGS="$OMP_FLAGS"
  set_standard_lib_paths_supermuc

  use_mpi_startrun="poe"
  use_mpi_startrun="mpiexec -n \\\$mpi_total_procs"
}

#=============================================================================
set_run_target_supermuc()
{
  set_default use_nproma 32
  default_nodes=1

  use_OMP_SCHEDULE="dynamic,1"

#   if [[ $use_mpi == "yes" ]]; then
#     if [[ $use_openmp == "yes" ]]; then
#       default_mpi_procs_pernode=4
#       use_openmp_threads=${use_openmp_threads:=10}
#     else
#       default_mpi_procs_pernode=40
#     fi
#   else
#     use_openmp_threads=${use_openmp_threads:=40}
#   fi

  calculate_mpi_procs

  # check if we are in smt mode
  task_affinity=""
  mcm_affinity_options=""
  use_resources=""
  rset=""

  if [[  $run_mode = "seq" ]] ; then
    job_type=serial
    node_usage=shared
    networkMPI=""
    node="node             = 1"
    tasks_per_node="tasks_per_node   = $use_mpi_procs_pernode"
 else
    job_type=parallel
    job_type=mpich
    node_usage=not_shared
    networkMPI="network.MPI  = sn_all,not_shared,us"
    node="node             = $use_nodes"
    tasks_per_node="tasks_per_node   = $use_mpi_procs_pernode"
  fi

  #-------------------------------------
  if [[ "x$use_cpu_time" != "x" ]] ; then
    wall_clock_limit="@ wall_clock_limit = $use_cpu_time"
  else
    wall_clock_limit=""
  fi

  if [ $use_nodes -le 4 ] ; then
    class="class = test"
    island_count="island_count = 1"
  else
    class="class = general"
    island_count="island_count = 1"
  fi

  if [ $use_nodes -ge 1000 ] ; then
    class="class = large"
    island_count="island_count = 2,3"
  fi

  if [ $use_nodes -ge 2000 ] ; then
    class="class = large"
    island_count="island_count = 4"
  fi
  #-------------------------------------


  # create header
  start_header

#  create_llsubmit_header
  cat >> $output_script << EOF
# =====================================
# @ shell        = $use_shell
# @ job_type     = $job_type
# @ $class
# @ $island_count
# @ $networkMPI
# @ $node
# @ $tasks_per_node
# $wall_clock_limit
# @ initialdir   = $run_folder
# @ job_name     = $job_name
# @ output       = LOG.\$(job_name).\$(jobid).o
# @ error        = LOG.\$(job_name).\$(jobid).o
# @ notification=always
# @ notify_user=adamidis@dkrz.de
# @ queue
EOF

  if [ $use_openmp = "yes" ] ; then
  cat >> $output_script << EOF
# =====================================
#export MP_SINGLE_THREAD=no
#export MP_TASK_AFFINITY=core:$use_openmp_threads

# pinning for intel
#export KMP_LIBRARY=turnaround
#export KMP_AFFINITY="verbose,granularity=fine,compact"
#export KMP_AFFINITY="verbose,granularity=core,compact,1,0"
#export KMP_AFFINITY="verbose,granularity=fine,compact,0,0"
export I_MPI_PIN_DOMAIN=omp
export I_MPI_PIN_MODE=mpd
export I_MPI_DEBUG=4
#========================================

printenv |grep -i "I_MPI_"
printenv |grep -i "KMP_"

placementtest-mpi.intel

EOF
  fi
}
