#!/bin/bash

# ICON
#
# ------------------------------------------
# Copyright (C) 2004-2026, DWD, MPI-M, DKRZ, KIT, ETH, MeteoSwiss
# Contact information: icon-model.org
# See AUTHORS.TXT for a list of authors
# See LICENSES/ for license information
# SPDX-License-Identifier: BSD-3-Clause
# ------------------------------------------

## ----------------------------------------------------------------------------
##
## SCM run script for LANFEX/demistify fog case
##
#-----------------------------------------------------------------------------
#PBS -q sx_norm
#PBS -v NE=1,CPE=1,NMPI_ROOT=/opt/nec/ve/mpi/2.3.1
#PBS -l elapstim_req=00:10:00
#PBS --venode=${NE}
#PBS --venum-lhost=1     # Number of VE per logical host
#PBS --cpunum-lhost=2    # Number of VH-x86-Cores per log. host
#PBS -T necmpi           # necessary for parallel jobs with NEC-MPI
#PBS -N icon-scm-ideal
##PBS -o /hpc/uwork/${USER}/wq/iconSCM.%s
#PBS -j o
##-----------------------------------------------------------------------------

## ----------------------------------------------------------------------------
##PBS -q sx_norm
##PBS -v NE=2,CPE=8,NMPI_ROOT=/opt/nec/ve/mpi/2.3.1
##PBS -v NMPI_MALLOC_HEAP_EXPANSION_SIZE=512
##PBS -v NMPI_MALLOC_MMAP_THRESHOLD=512
##PBS -v MPI_IB_VBUF_TOTAL_SIZE=131072
##PBS -v NMPI_DAEMON_PATH=${NMPI_ROOT}/libexec/mpid
##PBS -l elapstim_req=00:40:00
##PBS --venode=${NE}
##PBS --venum-lhost=2     # Number of VE per logical host
##PBS -l cpunum_job=2
##PBS -l coresz_prc=0
##PBS -T necmpi
##PBS --use-hca=2             # Number of HCA per logical host
##PBS -j o
##PBS -N icon-scm-ideal
## ----------------------------------------------------------------------------
# ============================================================================

source ${NMPI_ROOT}/bin/necmpivars.sh

set -x

# OpenMP settings
export OMP_SCHEDULE="static"
export OMP_DYNAMIC="false"
export OMP_NUM_THREADS=1   # check total number of requested cores!
export ATP_ENABLED=1

# Run information
let PPN=${CPE}*${NE}/${OMP_NUM_THREADS}         # no. of MPI procs. per NQS job
let NE1=${NE}-1

ID=`echo $PBS_JOBID | cut -d: -f2 | cut -d. -f1`
ulimit -s unlimited


# ----------------------------------------------------------------------------
# specifiy experiment (idealized simulation)
# ----------------------------------------------------------------------------
#EXPNAME=LANFEX
EXPNAME=test_scm_new_tg

# ----------------------------------------------------------------------------
# path definitions
# ----------------------------------------------------------------------------

# base directory for ICON sources and binary:
#ICONDIR="$PWD/.."
ICONDIR=${PBS_O_WORKDIR%/*/*/*}       # local icon directory (up from SCM-ideal)
#ICONDIR="/e/uhome/mkoehler/icon/icon-nwp-test3"
#ICONDIR="${HOME}/icon-nwp-scm-new"

# directory with input grids:
#GRIDDIR=/hpc/uhome/tgoecke/icon-nwp/SCM-ideal
#GRIDDIR=/hpc/uwork/tgoecke/LANFEX/indata
GRIDDIR=/hpc/uwork/mkoehler/scm/data/grid

# absolute path to directory with plenty of space:
#EXPDIR=${GRIDDIR}/${EXPNAME}
#EXPDIR=/hpc/gtmp/tgoecke/experiments/SCM-ideal/${EXPNAME}
EXPDIR=${WORK}/LANFEX/experiments/${EXPNAME}

# path to model binary, including the executable:
#MODEL=${ICONDIR}/build/x86_64-unknown-linux-gnu/bin/icon
MODEL=${ICONDIR}/build/VE/bin/icon
MODEL_SCAL=${ICONDIR}/build/VH/bin/icon


# ----------------------------------------------------------------------------
# copy input data: grids, external parameters, model
# ----------------------------------------------------------------------------

# the directory for the experiment will be created, if not already there
if [ ! -d $EXPDIR ]; then
    mkdir -p $EXPDIR
fi
cd ${EXPDIR}

# files needed for radiation
ln -sf ${ICONDIR}/data/ECHAM6_CldOptProps.nc .
ln -sf ${ICONDIR}/data/rrtmg_lw.nc .
# needed for 2 moment microphysics
ln -sf ${ICONDIR}/data/dmin_wetgrowth_lookup.nc #.dat .

# APE BCs
#ln -sf ${ICONDIR}/data/external/ape_o3_R2B04_1Pa_spr0.90-cell.nc o3_icon_DOM01.nc

# link binary
ln -sf $MODEL icon


#-----------------------------------------------------------------------------
# model timing
#-----------------------------------------------------------------------------

# 0.4 s for 70 m res., 0.5 for 100 m; 30 s for 2.5 km?
dtime=60
ndyn_substeps=3
dt_checkpoint=`expr 36 \* 3600`  # write restart file every hours (when lrestart = TRUE)
nhours=19
#nhours=1
nsteps=`expr ${nhours} \* 3600 / ${dtime}`

start_date="2014-11-24T17:00:00Z"
end_date="2014-11-25T12:00:00Z"

#-----------------------------------------------------------------------------
# output
#-----------------------------------------------------------------------------
DT_DATA=`expr 1 \* 3600`      # output each n hours
DT_DATA=${dtime}              # output every time step
#n_in_ofile=60                # number of time steps per output file
n_in_ofile=10000                # number of time steps per output file

# ----------------------------------------------------------------------------
# grid namelist settings
# ----------------------------------------------------------------------------

# the grid parameters
atmo_dyn_grids="Torus_Triangles_4x4_2500m.nc"
#atmo_dyn_grids="Torus_Triangles_4x4_10000m.nc"
atmo_rad_grids=""

# reconstruct the grid parameters in namelist form
dynamics_grid_filename=""
for gridfile in ${atmo_dyn_grids}; do
  dynamics_grid_filename="${dynamics_grid_filename} '${gridfile}',"
done
radiation_grid_filename=""
for gridfile in ${atmo_rad_grids}; do
  radiation_grid_filename="${radiation_grid_filename} '${gridfile}',"
done

ln -sf ${GRIDDIR}/${atmo_dyn_grids} .

#forcings
#ln -sf ${GRIDDIR}/py/init_SCM_GASS_DCP_GoAmazon.nc init_SCM.nc
#ln -sf ${GRIDDIR}/py/init_SCM_GASS_DCP_GoAmazon_IOP1.nc init_SCM.nc
#ln -sf /e/uhome/mkoehler/icon/icon-nwp-test3/SCM-ideal/py/init_SCM_GASS_DCP_GoAmazon.nc init_SCM.nc
ln -sf ${ICONDIR}/data/scm/init_data/init_SCM_LANFEX.nc init_SCM.nc

# ----------------------------------------------------------------------------
# create ICON master namelist
# ----------------------------------------------------------------------------

cat > icon_master.namelist << EOF

! master_nml: ----------------------------------------------------------------
&master_nml
 lrestart                    =                     .FALSE.        ! .TRUE.=current experiment is resumed
/

! master_model_nml: repeated for each model ----------------------------------
&master_model_nml
 model_type                  =                          1         ! identifies which component to run (atmosphere,ocean,...)
 model_name                  =                      "ATMO"        ! character string for naming this component.
 model_namelist_filename     =       "NAMELIST_${EXPNAME}"        ! file name containing the model namelists
 model_min_rank              =                          1         ! start MPI rank for this model
 model_max_rank              =                      65536         ! end MPI rank for this model
 model_inc_rank              =                          1         ! stride of MPI ranks
/

! time_nml: specification of date and time------------------------------------
&time_nml
 ini_datetime_string         =               "$start_date"        ! initial date and time of the simulation
 end_datetime_string         =                 "$end_date"        ! initial date and time of the simulation
/

EOF


# ----------------------------------------------------------------------------
# model namelists
# ----------------------------------------------------------------------------
# For a complete list see Namelist_overview.pdf

cat > NAMELIST_${EXPNAME} << EOF

&parallel_nml
 nproma         =  8 ! 32
 p_test_run     = .false.
 num_io_procs   =  1         ! number of I/O processors
 l_test_openmp  = .false.
 l_log_checks   = .false.
/

&grid_nml
 !cell_type = 3            ! triangular cells
 dynamics_grid_filename = "${atmo_dyn_grids}",
 corio_lat = 52.1         ! Cardington UK, 0.42W 52.1M
 is_plane_torus = .TRUE.
 l_scm_mode = .TRUE.       ! main logical to turn on SCM mode
/

&run_nml
 num_lev        = 200 !90           ! number of full levels of vertical grid
 dtime          = ${dtime}     ! timestep in seconds
 nsteps         = ${nsteps}
 ldynamics      = .FALSE.      ! compute adiabatic dynamic tendencies
 ltransport     = .FALSE.
 ntracer        = 5            ! default: 0
 iforcing       = 3            ! 3: NWP forcing; 6:inhecham forcing
 ltestcase      = .TRUE.       ! run testcase
 ltimer         = .FALSE.      !
 msg_level      = 12 !25           ! detailed report during integration
 output         = 'nml'
/

&SCM_nml
 i_scm_netcdf = 1            ! read initial profiles and forcings from netcdf
 lscm_read_tke=.False.     ! read initial tke from netcdf
 lscm_read_z0=.TRUE.       ! read initial z0 from netcdf
 scm_sfc_mom=0 !0             ! 0=TERRA,1=turb_trans,2=prescribed u*, 0
                           ! 0 , combined with inwp_surface = 0 (TERRA off)
			   ! to let turb_trans determine u* (do not overwrite flux)
 scm_sfc_temp=1            ! 0=TERRA,1=turb_trans+t_g,prescribed sensible heat flux at surface
 scm_sfc_qv=3              ! 0=TERRA,1=turb_trans+q_vs,prescribed latent heat flux at surface,3=qv_s at saturation

/

&io_nml
 dt_checkpoint  = ${dt_checkpoint}
 lkeep_in_sync    = .true.
 lflux_avg = .FALSE. ! use accumulated rather than averages variables where possible
/



&nwp_phy_nml
 inwp_gscp       = 4 !
 icalc_reff = 4 ! use effective radius calculation consitent between micro and cloud, use same as inwp_gscp
 inwp_convection = 0 ! 1:Tiedtke/Bechtold 0:apdf
 inwp_radiation  = 1 ! 1:RRTM radiation
 inwp_cldcover   = 1 ! 3: clouds from COSMO SGS cloud scheme 0:apdf
 inwp_turb       = 1 ! 1: TKE diffusion and transfer
 inwp_satad      = 1 !
 inwp_sso        = 0
 inwp_gwd        = 0
 inwp_surface    = 0 ! 0: none; 1: TERRA
 icapdcycl	 = 3            ! apply CAPE modification to improve diurnalcycle over tropical land (optimizes NWP scores)
 latm_above_top  = .TRUE.  ! needed for radiation routine
 itype_z0        = 1 ! 1: default, 2: turn off SSO part of z0
 dt_rad				= 1800.        ! Default: 1800   ! M. Koehler: 1440
 dt_conv			= 600.         ! Default: 600    ! M. Koehler: 360
 dt_sso				= 600.         ! Default: 1200   ! M. Koehler: 720
 dt_gwd				= 600.         ! Default: 1200   ! M. Koehler: 720
/



&radiation_nml
 ecRad_data_path = '${ICONDIR}/externals/ecrad/data'
  irad_co2 = 2 ! 2         ! constant from vmr_co2
  vmr_co2 = 0.00039145 != volume mixing ration to get mass mixing ratio of 5.941E-4 kg/kg
  irad_o3 = 11 ! 0: switch off ozone and initialize in test_case setup instead
  !irad_aero = 6 ! 6: Tegen aerosol climatology in radiation (not workin with testcase?
  !izenith = 6 ! same lat lon for all grid points in zenith angle calculation, rad_lat, rad_lon set in grid_nml
  albedo_type = 1 ! 2: MODIS albedo, together with inwp_surface = 0
                  ! 1: albedo based on soil type, see mo_phyparam_soil
 vmr_ch4                      = 1800.e-09
 vmr_n2o                      = 322.0e-09
 vmr_o2                       = 0.20946
 vmr_cfc11                    = 240.e-12
 vmr_cfc12                    = 532.e-12
 direct_albedo                = 4
 direct_albedo_water          = 3
 albedo_whitecap              = 1
 llw_cloud_scat               = .true.
/

 only if TERRA is needed?
&lnd_nml
  ntiles         = 1 !3
  nlev_snow      = 3
  lmulti_snow    = .false.
  itype_heatcond = 3
  idiag_snowfrac = 20
  lsnowtile      = .true.
  lseaice        = .true.
  llake          = .true.
  itype_lndtbl   = 4
  itype_evsl     = 4
  itype_trvg     = 3
  itype_root     = 2
  cwimax_ml      = 5.e-4
  c_soil         = 1.25
  c_soil_urb     = 0.5
  sstice_mode    = 2
  lprog_albsi    = .true.
  itype_snowevap = 2
/

&ls_forcing_nml
 is_subsidence_moment = .FALSE.
 is_subsidence_heat   = .FALSE.
 is_advection         = .FALSE.
 is_advection_uv      = .FALSE.
 is_advection_tq      = .FALSE.
 is_geowind           = .FALSE.
 is_rad_forcing       = .FALSE.
 is_nudging           = .FALSE.
 is_nudging_uv        = .FALSE.
 is_nudging_t         = .FALSE.
 is_nudging_q         = .FALSE.
 is_theta             = .FALSE.
/

&turbdiff_nml
 tkhmin  = 0.6 !0.75  ! new default since rev. 16527
 tkmmin  = 0.75  !
 tkhmin_strat                 = 1.0
 tkmmin_strat                 = 4
 alpha0                       = 0.0123
 alpha0_max                   = 0.0335
 alpha1                       = 0.125
 pat_len = 750.
 c_diff  = 0.2
 rat_sea = 0.8 !8.5  ! ** new value since rev. 25646 (reduced saturation vapor pressure over salt water) **
 ltkesso = .true.
 frcsmot = 0.2      ! these 2 switches together apply vertical smoothing of the TKE source terms
 imode_frcsmot = 2  ! in the tropics (only), which reduces the moist bias in the tropical lower troposphere
 ! use horizontal shear production terms with 1/SQRT(Ri) scaling to prevent unwanted side effects:
 itype_sher = 3
 ltkeshs    = .true.
 a_hshr     = 2.0
 lconst_z0 = .TRUE.
 const_z0  = 0.1  ! is this really needed, if scm mode overwrites z0 any way?
 icldm_turb                   = 1       ! 2: Gauss clouds for turbulence    1: grid scale clouds
 icldm_tran                   = 2       ! 2: Gauss clouds for surface layer 1: grid scale clouds
/

&nwp_tuning_nml
 itune_albedo                 = 0       ! somewhat reduced albedo (w.r.t. MODIS data) over Sahara in order to reduce cold bias
 tune_zceff_min               = 0.01    ! ** default value to be used for R3B7; use 0.025 for R2B6 in order to get similar temperature biases in upper troposphere **
 tune_gkdrag                  = 0.075   ! R2B6: 0.075
 tune_gkwake                  = 1.5     ! R2B6: 1.5
 tune_gfrcrit                 = 0.425   ! R2B6: 0.425
 tune_dust_abs                = 0.
 tune_zvz0i                   = 0.85
 tune_box_liq_asy             = 3.25    ! oper global: 3.0 , oper D2: 3.25, default: 2.5
 tune_box_liq                 = 0.05
 tune_rcucov                  = 0.075
 tune_rhebc_land              = 0.825
 tune_gust_factor             = 7.0
 icpl_turb_clc                = 1
 lcalib_clcov                 = .false. ! turn off TCC, HCC, MCC, LCC tuning
/

&diffusion_nml
 lhdiff_temp  = .TRUE.
 lhdiff_vn    = .TRUE.
 lhdiff_w     = .TRUE.

 hdiff_order                  = 5
 itype_vn_diffu               = 1
 itype_t_diffu                = 2
 hdiff_efdt_ratio             = 24.0   ! for R2B6; recommendation for R3B7: 30.0
 hdiff_smag_fac               = 0.025  ! for R2B6; recommendation for R3B7: 0.02
/

&nonhydrostatic_nml
 ivctype        = 2          ! sleve vertical coordinate
 damp_height    = 44000.     ! top_height-damp_height should be about 15km
 ndyn_substeps  = ${ndyn_substeps}
 rayleigh_coeff = 0.5 !0.0
 htop_moist_proc= 22500.
 hbot_qvsubstep	= 16000.    ! Default: 22500  ! M. Koehler: 19000; at leas as large as htop_moist_proc

 iadv_rhotheta                = 2
 itime_scheme                 = 4
 exner_expol                  = 0.333
 vwind_offctr                 = 0.2         ! 0.2 for R2B6 and higher resolution, 0.3 for lower resolution
 divdamp_order                = 24          ! 2 ass, 24 fc
 divdamp_type                 = 32          ! optional: 2 assimilation cycle, 32 forecast
 divdamp_fac                  = 0.004       ! 0.004 for R2B6; recommendation for R3B7: 0.003
 divdamp_trans_start          = 12500
 divdamp_trans_end            = 17500
 igradp_method                = 3
 l_zdiffu_t                   = .true.
 thslp_zdiffu                 = 0.02
 thhgtd_zdiffu                = 125.
/

&sleve_nml
 min_lay_thckn			= 5.       ! Default: 50     ! M. Koehler: 20
 max_lay_thckn			= 400.      ! maximum layer thickness below htop_thcknlimit: 400 recommended for NWP
 htop_thcknlimit		= 14000.    ! this implies that the upcoming COSMO-EU nest will have 60 levels; Default: 15000 ! M. Koehler: 14000
 top_height			= 75000.
 stretch_fac			= 0.9       ! Default: 1      ! M. Koehler: 0.9
 decay_scale_1			= 4000.
 decay_scale_2			= 2500.
 decay_exp			= 1.2
 flat_height			= 16000.
/



&extpar_nml
 itopo          = 0 ! 0: analytical topo; 1: topography/ext. data read from file
/

&dynamics_nml
 divavg_cntrwgt = 0.50
 lcoriolis      = .TRUE.  ! SCM attention: .TRUE. distroys U-profile!!
/

!&output_nml
! mode             = 1 ! 1: nwp mode
! output_start     = "${start_date}"
! output_end       = "${end_date}"
! output_interval  =  "PT05M" !"PT05M"
! !output_interval  = "PT60S"
! !file_interval    = "P01H"
! steps_per_file   = ${n_in_ofile}
! include_last     = .TRUE.
! output_filename  = 'prog_vars'
! filename_format  = "<output_filename>_DOM<physdom>_<levtype>_<datetime2>"
! ml_varlist       = 'group:all', 'rh' , ! 'ddt_temp_gscp'#,'ddt_tke_therm'
!                    'z_ifc','z_mc','u','v','w','temp','pres','rho','theta_v','pres_sfc','div',
!                    'qv','qc','qi','qs','qr','rh',
!                    !'ashfl_s', 'alhfl_s', 'athb_s', 'athb_t', 'asob_s', 'asob_t',
!                    'ddt_temp_radsw', 'ddt_temp_radlw', 'ddt_temp_turb', 'ddt_temp_drag', 'ddt_temp_pconv','ddt_temp_gscp',
!                    'ddt_qv_turb','ddt_qc_turb','ddt_qv_conv','ddt_qc_conv','u_10m', 'v_10m', 't_2m', 't_g',
!                    'qv_s','z_mc','lhfl_s','shfl_s','umfl_s','vmfl_s','tcm','tch','clc','tke','rcld','qhfl_s',
!                    'sob_s', 'thb_s','sob_t', 'lwflx_dn',
!	       !'tet_flux','vap_flux','liq_flux'
!	       'tetfl_turb', 'vapfl_turb', 'liqfl_turb',
!	       !'ddt_tke_therm', 'ddt_tke_mech', 'ddt_tke_shear','ddt_tke_pconv','ddt_tke_hsh', ! ,'edr',
!               'qnc','qnr','qni','qns','qng','qnh','ninact', ! for 2mom scheme, cnnc when aerosol?
!	       !'t_g','qv_s','t_2m'
!
! output_grid      = .TRUE.
!/


!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
! output SCALAR fields as demanded by Ian Boutle every minute
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
&output_nml
 mode             = 1 ! 1: nwp mode
 output_start     = "${start_date}"
 output_end       = "${end_date}"
 output_interval  =  "PT01M" !"PT05M"
 steps_per_file   = ${n_in_ofile}
 output_filename  =  'scalars_${EXPNAME}_ICON_SCM_DWD'
 output_grid      =  .TRUE.
 include_last     =  .TRUE.
 ml_varlist = 'sou_s','accsodird_s','accsodifd_s','accsodifu_s','accsob_s','accthu_s','accthd_s','t_g','accshfl_s','acclhfl_s','clct','tqv','tqc_dia','tqc','tqr','rain_gsp_rate','rain_con_rate','pres_sfc','u_10m','v_10m','t_2m','qv_2m','rh_2m','rh_2m_land','tvm', 'qv_s','accthb_s', 'accthb_t', 'accsob_s', 'accsob_t','theta_v','clc','tot_qv_dia','tot_qc_dia','tot_qi_dia','qr','qs','temp','pres','u','v','rho','z_mc','qhfl_s','qcfl_s','umfl_s','vmfl_s' !,'cloud_gsp_rate'
/
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
! output PROFILES as demanded by Ian Boutle every 15 min
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
&output_nml
 filetype         =  4
 dom              = -1
 output_start     = "${start_date}"
 output_end       = "${end_date}"
 output_interval  =  "PT15M" !"PT05M"
 steps_per_file   = ${n_in_ofile}
 include_last     =  .TRUE.
 remap            =  0   ! triangular grid
 output_filename  =  'profs_${EXPNAME}_ICON_SCM_DWD'
 output_grid      =  .TRUE.
 mode             =  1      ! 1: forecast mode (relative t-axis), 2: climate mode (absolute t-axis)
  ml_varlist = 'z_mc','z_ifc','pres','exner','theta_v','temp','rh','qv','qc','qr','qi','qs','rho','tot_qv_dia','tot_qi_dia','tot_qc_dia',
               'clc','u','v','tkvm','tkvh','tke','ddt_qv_turb','ddt_qc_turb','ddt_tke',
	       'ddt_temp_gscp','ddt_temp_radsw','ddt_temp_radlw','ddt_temp_turb','lwflxall','trsolall','pres_sfc','t_g',
	       'qnc','qnr','qni','qns','qng','qnh','ninact',
	       'tetfl_turb','vapfl_turb','liqfl_turb',
	       'lwflx_up','lwflx_dn','swflx_up','swflx_dn',
	       'reff_qc','reff_qr'
	       ! 'ddt_tke_therm','ddt_tke_mech','ddt_tke_shear','ddt_tke_pconv','ddt_tke_hsh','edr',


	          !!!'vfl_q01','vfl_q02','vfl_q04',
	          !!!!,'ddt_qv_gscp','ddt_qc_gscp','ddt_qr_gscp',
	      !'ddt_temp_gscp','ddt_temp_radsw','ddt_temp_radlw','ddt_temp_turb','lwflxall','trsolall','pres_sfc','t_g', !'re_diag',
              ! 'qnc','qnr','qni','qns','qng','qnh','ninact', ! for 2mom scheme, cnnc when aerosol?
              ! 'tetfl_turb', 'vapfl_turb', 'liqfl_turb',
	      ! 'lwflx_up','lwflx_dn','swflx_up','swflx_dn',
	      ! 'lwflx_up_clr','lwflx_dn_clr','swflx_up_clr','swflx_dn_clr',
	      !    !'tet_flux','vap_flux','liq_flux','lwup','lwdn','swup','swdn' ,
	      !    !'ddt_tke_therm', 'ddt_tke_mech', 'ddt_tke_shear','ddt_tke_pconv','ddt_tke_hsh' ,'edr'
/
EOF


# ----------------------------------------------------------------------------
# run the model!
# ----------------------------------------------------------------------------

#./icon > out.txt 2>out2.txt

# select=4 : Number of nodes - see header (n/N)
# -n 48    : Number of MPI Tasks (N*select)
# -N 12    : Number of MPI Tasks/Node            (12: haswell, 18: broadwell)
# -d 4     : Number of threads Threads/MPI Task
# -j 2     : Hyperthreading enabled: 24 physical cores -> 48 "virtual" cores
# -m 3g    : 3GB memory, 64GB available shared on one node (m x N < 64)

#aprun -n 18 -N 18 -j 2 -d 4 -m 3g ./icon

ls -l $MODEL

# VE with openMP only (PPN = 1!)
date1=`date`

#mpiexec -v    -vh     -node 0        -np 1      ${MODEL_SCAL} : \
#           -x -venode -node 0-${NE1} -np ${PPN} ${MODEL}    #  : \
mpiexec -v -x -venode -np ${PPN} ${MODEL}


#/opt/nec/ve/bin/mpirun -v    -vh     -node 0        -np 1      -env OMP_NUM_THREADS 1 ${MODEL_SCAL} : \
#                             -x      -venode        -np ${PPN} -env OMP_NUM_THREADS 1 ${MODEL} : \
#                             -vh     -node 0        -np 4      -env OMP_NUM_THREADS 1 ${MODEL_SCAL}
#

echo $data1 ; date
