# ICON
#
# ---------------------------------------------------------------
# Copyright (C) 2004-2026, DWD, MPI-M, DKRZ, KIT, ETH, MeteoSwiss
# Contact information: icon-model.org
# See AUTHORS.TXT for a list of authors
# See LICENSES/ for license information
# SPDX-License-Identifier: BSD-3-Clause
# ---------------------------------------------------------------

require 'minitest/autorun'
require 'open3'
require 'jobqueue'
require 'facets'
require 'iconPlot'
require 'rake/clean'
require 'tempfile'
require 'test/unit/assertions'
include Test::Unit::Assertions

# ============================================================================
# Please copy this file to the opt directory of you main working copy and
# adjust the remote and local settings
# usage with the given configuration:
# * parallel build with gcc (locally) and nag (remotely)
#   rake parX localGcc_build remoteNag_build
# * synchronization of several remote build locations
#   rake par remoteDef_sync remoteIntel_sync remoteGccHi_sync
# * print our the commands before executing:
#   DEBUG=1 rake remoteGccHi_conf
# * print the commands without executing them
#   DRYRUN=1 rake remoteGccHi_conf
# ============================================================================
# AUTHOR: Ralf Mueller, ralf.mueller@zmaw.de
# REQUIREMENTS:
# * ruby-1.9.* or higher (use with 'module load ruby')
# * jobQueue (rubygem, install with 'gem install jobQueue --user-install)
# ============================================================================
# user account on all remote machines
REMOTE_USER = 'm300064'
@fileList   = FileList.new("src/**/*.{F,F90,f,h,c,inc,f90}","support/*.{c,h}","externals/**/*.{c,h}")

# local master source code directory
SRCDIR     = ENV.has_key?('SRCDIR')? ENV['SRCDIR'] : '.'
WORKDIR    = File.expand_path('.')
OCEAN_ONLY = ( not ENV.has_key?('OCEAN_ONLY') or 'false' != ENV['OCEAN_ONLY']) ? true : false
# remote runscripts
DEFAULT_EXP = 'test_oce_default'
#plotter
@plotter = IconPlot.new

# load user settings
RC = "#{ENV['HOME']}/.rake.rc"
load(RC) if File.exist?(RC)


# basic host setup
$hosts = {
  # local port forwarding setup for thunder4 and 5
  # cat .ssh/config
  #Host thunder
  #HostName login.zmaw.de
  #LocalForward 50022 thunder5.zmaw.de:22
  #LocalForward 40022 thunder4.zmaw.de:22

  :thunder4 => {
    :user     => REMOTE_USER,
    :hostname => 'localhost',
    :port     => 40022,
   #:hostname => 'thunder4.zmaw.de', # use from internal net if login node is down
    :dir      => "/scratch/mpi/CC/mh0287/users/#{REMOTE_USER}/builds/remote/icon" },
  :thunder5 => {
    :user     => REMOTE_USER,
    :hostname => 'localhost',
    :port     => 50022,
   #:hostname => 'thunder5.zmaw.de',# use from internal net if login node is down
    :dir      => "/scratch/mpi/CC/mh0287/users/#{REMOTE_USER}/builds/remote/icon" },
  :passat => {
    :user     => REMOTE_USER,
    :hostname => 'passat.dkrz.de',
    :dir      => '/work/mh0287/users/ram/builds'},
  :wizard => {
    :user     => REMOTE_USER,
    :hostname => 'wizard.dkrz.de',
    :dir      => '/work/mh0287/users/ram/builds'},
  # laptop on my desk
  :thingol => {
    :dir => '/home/ram/builds/icon' }
}
# naming conventions:
# * local builds start with 'local'
# * remote builds start with 'remote'
$builds = {
  :localGcc          => {:host => :thingol , :subdir => 'gcc'}     ,
  :localGccBranch    => {:host => :thingol , :subdir => 'gcc-branch'}     ,
  :localIntel        => {:host => :thingol , :subdir => 'intel',:CC => 'intel'}     ,
  :localIntelHi      => {:host => :thingol , :subdir => 'intel-hiopt',:CC => 'intel'}     ,
  :localCmake        => {:host => :thingol , :subdir => 'cmake'}     ,
  :localGccHi        => {:host => :thingol , :subdir => 'gcc-hiopt', :CC => 'gcc'      , :FLAGS => 'hiopt'},
  :localGprof        => {:host => :thingol , :subdir => 'gprof'}   ,
  :localOpenMP       => {:host => :thingol , :subdir => 'openmp'   ,                     :FLAGS => 'hiopt' , :openmp => true}  ,
  :localNag          => {:host => :thingol , :subdir => 'nag'      , :CC => 'nag'}     ,
  :localSun          => {:host => :thingol , :subdir => 'sun'      , :CC => 'sun'      ,:preConf => 'PATH=$HOME/local/solarisstudio12.3/bin:$PATH', :preBuild => 'PATH=$HOME/local/solarisstudio12.3/bin:$PATH'},
  :localTest         => {:host => :thingol , :subdir => 'test'     , :CC => 'nag'}     ,
  :remoteSerial      => {:host => :thunder4, :subdir => 'serial'   ,                    :confOpts => '--without-mpi'}    ,
  :remoteCmake       => {:host => :thunder4, :subdir => 'cmake'}   ,
  :remoteGccHi       => {:host => :thunder5, :subdir => 'gcc'      , :CC => 'gcc'      , :FLAGS => 'hiopt'},
  :remoteGccHiOpenMP => {:host => :thunder5, :subdir => 'gcc-opemp', :CC => 'gcc'      , :FLAGS => 'hiopt' , :openmp => true},
  :remoteNag         => {:host => :thunder4, :subdir => 'nag'      , :CC => 'nag'}     ,
  :remoteIntel       => {:host => :thunder4, :subdir => 'intel'    , :CC => 'intel'}   ,
  :remotePgi         => {:host => :thunder4, :subdir => 'pgi'      , :CC => 'pgi'}     ,
  :remoteXlf         => {:host => :passat  , :subdir => 'default'} ,
  :remoteIntelW      => {:host => :wizard  , :subdir => 'intel', :CC => 'intel' } ,
  :remoteXlfHi       => {:host => :passat  , :subdir => 'hiopt'    , :FLAGS => 'hiopt'}
}
# ============================================================================

# run the commands and try to get back stdout or possible error messages
def call(cmd,unthreadded=false)
  unless ENV['DRYRUN'].nil?
    puts cmd
    return
  end
  dbg(cmd)
  if unthreadded then
    puts IO.popen(cmd).read
  else
    Open3.popen3(cmd) do |stdin, stdout, stderr, external|
      # read from stdout and stderr in parallel
      { :out => stdout, :err => stderr }.each {|key, stream|
        Thread.new do
          until (line = stream.gets).nil? do
            puts line
          end
        end
      }

      # Don't exit until the external process is done
      external.join
    end
  end
end

# debug output for any type of object
def dbg(msg)
  pp msg unless ENV['DEBUG'].nil?
end

# check for the naming convention to distinguish remote from local builds
def isLocal?(build);  'local' == build.to_s[0,5]; end
def isRemote?(build); not isLocal?(build); end
def getBranch
  if File.exists?('.svn') then
    url = `svn info | grep URL`.chomp.split(' ')[-1]
    branch = /branches/.match(url) ? url.split('/')[-1] : master
  else
    branch = `git branch`.split("\n").grep(/^\*/)[0].split[-1]
  end
  return branch == 'master' ? nil : branch
end

# hide some complex hash accesses
# {
$hostOf = lambda {|build|
  host = $builds[build][:host]
}
$hostnameOf = lambda {|build|
  host = $hostOf[build]
  $hosts[host][:hostname].nil? ? host.to_s : $hosts[host][:hostname]
}
# }

# compute the directory where to (possible remote) code is
$targetDir = lambda {|build,withHost=true,revision=nil|
  dir      = ''
  host     = $hostOf[build]
  hostname = $hostnameOf[build]
  dir << "#{REMOTE_USER}@#{hostname}:" if isRemote?(build) and withHost
  dir << $hosts[host][:dir] + '/' + $builds[build][:subdir]
  dir << "/r#{revision.to_s}" unless revision.nil?
  dir << "/#{getBranch}" unless getBranch.nil?

  dir
}

# ssh connection command line
$remoteConnection = lambda {|build|
  hostname  = $hostnameOf[build]
  portSpec  = $hosts[$hostOf[build]][:port].nil? ? '' : "-p #{$hosts[$hostOf[build]][:port]}"
  "ssh #{portSpec}"
}

# configure for a given build call wrt. to given options
$configureCall = lambda {|build|
  configureCall = ''
  if /cmake/i.match(build.to_s) then
  configureCall << './cmake.configure'
  else
  configureCall << "#{$builds[build][:preConf]};" unless $builds[build][:preConf].nil?
  configureCall << "COMPILER=#{$builds[build][:CC].nil? ? 'gcc' : $builds[build][:CC]} "
  configureCall << './configure'
  configureCall << " --with-fortran=#{$builds[build][:CC]}" unless $builds[build][:CC].nil?
  configureCall << " --with-flags=#{$builds[build][:FLAGS]}" unless $builds[build][:FLAGS].nil?
  configureCall << " --with-openmp" unless $builds[build][:openmp].nil?
  configureCall << " --disable-atmo --disable-jsbach" if OCEAN_ONLY
  configureCall << " " << $builds[build][:confOpts] if $builds[build].has_key?(:confOpts)
  configureCall << " ;"
  end
  configureCall
}

# build call
$buildCall = lambda {|build|
  procs = ENV.has_key?('PROCS') ? ENV['PROCS'] : 8
  buildCall = ''
  if /cmake/i.match(build.to_s) then
  buildCall << "make -j #{procs};"
  else
  buildCall << $builds[build][:preBuild]  << ';' unless $builds[build][:preBuild].nil?
  buildCall << 'perl -pi.bak -e "s/make.*/make -j '+procs.to_s+'/" build_command ; cat build_command;'
  buildCall << './build_command;'
  end
  buildCall
}
# create/submit runscripts
$runscriptCreateCall = lambda {|build,exp=''| "./make_runscripts #{exp}" }
$runscriptSubmitCall = lambda {|build,exp=''| "cd run; sbatch exp.#{exp}.run" }

# ============================================================================
# global tempfile array: little hack because the call generator does not really
# call the system command
$_temps = []

def setSelection
  var  = ENV.has_key?('VAR') ? ENV['VAR']      : '__ALL'
  ts   = ENV.has_key?('TS')  ? ENV['TS'].to_i  : 0
  lev  = ENV.has_key?('LEV') ? ENV['LEV'].to_i : 0
  show = ENV.has_key?('SHOW') #ENV.values_at('VAR','TS','LEV').uniq != [nil]
  max  = ENV.has_key?('MAX') ? ENV['MAX'].to_f : ''
  min  = ENV.has_key?('MIN') ? ENV['MIN'].to_f : ''
  mod  = ENV.has_key?('MOD') ? ENV['MOD']      : 'auto'
  showgrid  = ENV.has_key?('SHOWGRID') ? true : false


  [var,ts,lev,show,min,max,mod,showgrid]
end
# optional connection command
def connect(build,command,targetDir=$targetDir[build,false])
  cmd      = ''
  hostname = $hostnameOf[build]
  cmd << $remoteConnection[build] + " #{REMOTE_USER}@#{hostname} " if isRemote?(build)
  cmd << "'" if isRemote?(build)
  cmd << "source /etc/profile; [[ -f .profile ]] && source .profile; " if isRemote?(build)
  cmd << "cd #{targetDir};"

  # add module command - needed by ICONs configure
  cmd << command
  cmd << "'" if isRemote?(build)

  cmd
end

# check all possible accumulated variables
def checkAcc(build,exp,var)
  call(connect(build,<<-EOS
      cd experiments/#{exp};
      var=#{var} ruby -rcdo <<-EOR
      ofiles        = Dir.glob("#{exp}*")
      names         = Cdo.showname(input: ofiles[0])[0].split
      acc_names     = names.grep(/_acc/)
      non_acc_names = acc_names.map {|n| n[0..-5]}
      # collect variable with have acc and spot values in the output file
      vars2check    = non_acc_names.select {|name| names.include?(name) }
      vars2check    = vars2check.grep(/\#{ENV["var"]}/) if ENV["var"] != "__ALL"
      vars2check.each {|var2check|
        ofiles.each {|ofile|
          result = Cdo.diffv(input: " -selname," + var2check + " " + ofile +
                              " -selname," + var2check + "_acc" + " " + ofile)
          unless result.empty? then
            puts "CHECK ACCUMULATION in FILE:" + ofile + " with VAR:" + var2check
            pp result
          else
            puts "ZERO diff for " + var2check + " (file:" + ofile + ")"
          end
      }}
EOR
   EOS
  ))
end

# simple varnames check
def checkNames(build,exp)
  call(connect(build,<<-EOS
      cd experiments/#{exp};
      for file in $(ls -rc #{exp}*.nc*); do
        echo '#===========================================================================';
        echo " $file";echo
        cdo showname  $file
      done
      EOS
      ),true)
end
# cat output together to a fiven file
# output file variable is #{exp}_Output with value `tempfile`_#{exp}.nc
def joinModelOutput(exp: exp,ifilePattern: "#{exp}*.nc",expTag: '')
  if ifilePattern.nil? then
    "
    cd experiments;
    joinModelOutput_#{expTag}=$(tempfile -s _#{exp}.nc);
    [[ -f ${#{exp}_Output} ]] && rm ${joinModelOutput_#{exp}};
    cdo cat #{exp}/#{ifilePattern} ${joinModelOutput_#{exp}};
    cd -;
    "
  else
    "
    cd #{Dir::Tmpname.tmpdir}
    joinModelOutput_#{exp}_#{expTag}=$(tempfile -s _#{exp}.nc);
    [[ -f ${#{exp}_Output} ]] && rm ${joinModelOutput_#{exp}};
    cdo cat '#{ifilePattern}' ${joinModelOutput_#{exp}_#{expTag}};
    cd -;
    "
  end
end
# rm output from joinModelOutput
def rmJoinedModelOutput(*exps)
  out = ""
  exps.each {|exp|
    out << " [[ -f ${joinModelOutput_#{exp}} ]] && rm ${joinModelOutput_#{exp}};"
  }
  out
end
def selectBy(varname,timestep,level)
  out = <<-EOS
    selectBy='';
    if [ #{timestep} -ne 0 ]; then
      selectBy="$selectBy -seltimestep,#{timestep}";
    fi
    if [ #{level} -ne 0 ]; then
      selectBy="$selectBy -sellevidx,#{level}";
    fi
    set -x
    if [ "#{varname}" != '__ALL' ]; then
      selectBy="$selectBy -selname,#{varname}";
    fi
  EOS
end
def diffThese(fileA,fileB,diffThese='diffThese')
  out = <<-EOS
      #{diffThese}=$(tempfile -s _diffThese)
      cdo diffv #{fileA} #{fileB} > $#{diffThese}
      echo "# ==========================================================="
      if test -s $#{diffThese}
      then
        echo "DIFF found in OUTPUT!!"
        [[ $debug = 'true' ]] && cat $#{diffThese}
      else
        echo "no DIFF found in OUTPUT"
        cat $#{diffThese}
      fi
      echo "# ==========================================================="
      [[ -f ${#{diffThese}} ]] && rm ${#{diffThese}}
  EOS
end
# check a single variable with infov
def checkVar(build,exp,varname,timestep,level)
  call(connect(build,<<-EOS
      cd experiments/#{exp};
      for file in $(ls -rc #{exp}*); do
        echo "CHECKING FILE: $file   >>>>>>>>>>>>>>>>>>>>> "
        #{selectBy(varname,timestep,level)}
        cdo infov ${selectBy} $file
      done
      EOS
  ))
end
def checkOutputByDir(build,dirA,dirB)
    call(connect(build,<<-EOS
      selection='-select,name=forc_wind_u,forc_wind_v,timestep=1,2,3,4'
      selection=''
      debug="#{ENV.has_key?('DEBUG')}"
      [[ $debug = 'true' ]] && set -x
      fileA=/tmp/#{expA}.nc
      fileB=/tmp/#{expB}.nc
      [[ -f ${fileA} ]] && rm ${fileA}
      [[ -f ${fileB} ]] && rm ${fileB}
      cdo cat #{dirA}/#{expA}*.nc ${fileA}
      cdo cat #{dirB}/#{expB}*.nc ${fileB}
      #{diffThese("/tmp/#{expA}.nc","/tmp/#{expB}.nc")}
      echo "FILE:  $fileA ========================"
      [[ ${debug} = 'true' ]] && cdo infov ${selection} $fileA
      echo "FILE:  $fileB ========================"
        [[ ${debug} = 'true' ]] && cdo infov ${selection} $fileB
               EOS
              ),true)
end
# plot a plot from output file
def showPlot(build,exp,var: 't',timestep:  0,level:  0,min: '',max: '',mod: 'auto',showGrid: false)
  call(connect(build,<<-EOS
      cd experiments/#{exp};
      var=#{var} timestep=#{timestep} level=#{level} min=#{min} max=#{max} mod=#{mod} showGrid=#{showGrid} ruby -rcdo -riconPlot -rjobqueue <<-EOR
      ofiles        = Dir.glob("#{exp}*.nc")
      names         = Cdo.showname(input: ofiles[0])[0].split
      var          = ENV["var"]
      timestep     = ENV["timestep"]
      level        = ENV["level"]
      min          = ENV["min"]
      max          = ENV["max"]
      selMode      = ENV["mod"]
      showGrid     = ENV["showGrid"]
      plotter      = IconPlot.new()
      q = JobQueue.new()
      plotter.display = "sxiv"
      plotter.debug = true
      ofiles.each {|ofile|
        puts "Check file " +ofile
        q.push {
          plotter.show(plotter.scalarPlot(ofile,"NH_"+ofile,var,mapType: "NHps",timeStep: timestep,levIndex: level, selMode: selMode, minVar: min,maxVar: max,showGrid: showGrid))
        }
        q.push {
          plotter.show(plotter.scalarPlot(ofile,"SH_"+ofile,var,mapType: "SHps",timeStep: timestep,levIndex: level, selMode: selMode, minVar: min,maxVar: max,showGrid: showGrid))
        }
        q.push {
          plotter.show(plotter.scalarPlot(ofile,"GLOB_"+ofile,var,                timeStep: timestep,levIndex: level, selMode: selMode, minVar: min,maxVar: max,showGrid: false))
        }
      }
      q.run
EOR
   EOS
  ))
end

# display the online diagnostics
def showOnlineDiag(build,exp)
  diagFile    = 'oce_diagnostics.txt'
  diagPlotter = 'scripts/postprocessing/tools/oceDiag.gp'
  call(connect(build,<<-EOS
      cd experiments/#{exp};
      if test -f #{diagFile}; then
        LD_LIBRARY_PATH=/usr/lib SHOW=1 FILE=#{diagFile} gnuplot ../../#{diagPlotter};
      else echo 'No diagnostics avialable!';
      fi
   EOS
  ))
end
# show diff between two files
def doDiffThis(build,fA,fB,tool='meld')
  dbg("tool=#{tool}")
  dbg("fA=#{fA}")
  dbg("fB=#{fB}")
  call(connect(build,<<-EOS
     #{[tool,fA,fB].join(' ')}
   EOS
  ))
end
def getFilesBy(dir,subdirs,pattern: '**/*.{f90,inc}',filelist: [])
  files = []
  if filelist.empty? then
  subdirs.each {|subdir|
    files << Dir.glob([dir,subdir,pattern].join(File::SEPARATOR))
  }
  else
  subdirs.each {|subdir|
    filelist = [filelist] unless filelist.kind_of?(Array)
    filelist.each {|file| files << [dir,subdir,file].join(File::SEPARATOR) }
  }
  end
  files
end
def getOceanOnlyfilelist(build)
  oceanFiles = []
  oceanFiles << getFilesBy("src"   ,%w[ocean sea_ice parallel_infrastructure include shared])

  oceanFiles << getFilesBy("src/io",%w[shared])

  oceanFiles << getFilesBy("src",%w[advection], filelist: "mo_advection_utils.f90")

  oceanFiles << getFilesBy("src",%w[configure_model],
             filelist: %w[mo_dynamics_config.f90 mo_gribout_config.f90 mo_grid_config.f90 mo_io_config.f90 mo_name_list_output_config.f90 mo_parallel_config.f90 mo_run_config.f90 mo_time_config.f90])

  oceanFiles << getFilesBy("src",%w[drivers],
                           filelist: %w[control_model.f90 mo_master_control.f90])

  oceanFiles << getFilesBy("src", %w[namelists],
                           filelist: %w[mo_dbg_nml.f90  mo_dynamics_nml.f90 mo_gribout_nml.f90 mo_grid_nml.f90 mo_io_nml.f90 mo_master_nml.f90 mo_parallel_nml.f90 mo_run_nml.f90 mo_sea_ice_nml.f90 mo_time_nml.f90])

  oceanFiles << getFilesBy("src", %w[shr_horizontal],
                           filelist: %w[mo_alloc_patches.f90 mo_ext_data_types.f90 mo_grid_geometry_info.f90 mo_grid_tools.f90 mo_intp_data_strc.f90 mo_lonlat_grid.f90 mo_model_domain.f90 mo_model_domimp_patches.f90 mo_model_domimp_setup.f90 mo_reorder_patches.f90 mo_icon_interpolation_scalar.f90])

  oceanFiles << getFilesBy("src", %w[testcases],
                           filelist: "mo_ape_params.f90")

  oceanFiles << Dir.glob("run/*oce*")
  oceanFiles << getFilesBy("run",[''],
                           filelist: %w[exec.iconrun add_run_routines post.test_compare_restarts])


  oceanFiles << Dir.glob("config/*")

  %w[data doc include scripts support create_builds build_all aclocal.m4 configure.ac configure make_my_runscript make_runscripts Makefile.in externals].each {|item|
    if File.directory?(item) then
      oceanFiles <<  Dir.glob("#{item}/**/*")
    else
      oceanFiles << item
    end
  }

  oceanFiles.flatten!

  # remove nc files
  oceanFiles.delete_if {|f| %w[.nc .grb .png .eps .ps .pdf].include?(File.extname(f)) }

  oceanFiles.delete_if {|f| /mh-override/.match(f) } unless isLocal?(build)
  oceanFiles.delete_if {|f| /mh-config-use/.match(f) }
  oceanFiles.delete_if {|f| /set-up.info/.match(f) }

  # remove some bad candidates
  %w[mo_async_latbc.f90 mo_async_latbc_utils.f90 mo_sync_latbc.f90].each {|item|
    oceanFiles.delete_if {|file| /#{item}$/.match(file) }
  }

  oceanFiles
end
task :checkOceanFilelist, [:build] do |t,args|

  args.build = :localGcc if args.build.nil?
  puts getOceanOnlyfilelist(args.build)
end

# synchronization call for given build
def doSync(build)
  dir      = $targetDir[build]
  rsyncOpts = "--delete-excluded --delete"
  rsyncOpts = "-L"

  file = Tempfile.new("rsyncIconTempfiles2Transfer")
  $_temps << file

  # if OceanOnly
  if OCEAN_ONLY then
    getOceanOnlyfilelist(build).each {|f| file.write(f + "\n") }
    file.close
    ENV['FILELIST'] = file.path
    system("cp #{file.path} tempList") if ENV.has_key?('DEBUG')
  end
  if ENV.has_key?('FILELIST')
    myFile = ENV['FILELIST']
  else
    myFile = file.path
    if File.exists?('.svn') then
      file.write(`svn list -R | grep -E -v '\/$'`)
      # add svn revision info
      revInfoFile = '.rev.info'
      File.open(revInfoFile,'w') {|f| f << `svn info` }
      file.write(revInfoFile << "\n")
    else
      file.write(`git ls-files | grep -v -E '^(data|doc)'`)
      # add svn revision info
      revInfoFile = '.rev.info'
      File.open(revInfoFile,'w') {|f|
        f << `git log -4`
        f << "# locally modified files:\n"
        f << `git status -s | grep '^ M'| cut -d ' ' -f 3`
      }
      file.write(revInfoFile << "\n")
    end
    file.write(`find src/lnd_phy_jsbach -name "*.f90"`)
    # add non-added runscripts
    file.write(Dir.glob("run/{exp,post}*").delete_if {|v| /\.run$/.match(v)}.join("\n"))
    file.write("\n")
    # add grids directory for local external data
    #file.write(Dir.glob("grids/*.nc").join("\n") + "\n") unless isLocal?(build)
    file.write("config/mh-override\n") if isLocal?(build)
    file.close

    system("cat #{file.path}") if  ENV.has_key?('SHOWRSYNC')
    system("cp #{file.path} tempList") if ENV.has_key?('DEBUG')
  end

  if isLocal?(build)
    syncCmd = "rsync #{rsyncOpts} -avz --files-from=#{myFile} . #{dir}"
  else
    syncCmd = "rsync #{rsyncOpts} -avz --files-from=#{myFile}  -e '#{$remoteConnection[build]}' . #{dir}"
  end
  call(syncCmd)
end
def doCheckoutRev(build,rev,url)
  dir  = $targetDir[build,false,rev]
  revTag = rev == 'HEAD' ? '' : "-r#{rev}"
  call(connect(build,<<-EOS
      [[ ! -d r#{rev} ]] && mkdir -p r#{rev};
      cd r#{rev};
      if [[ -d .svn ]];then
        #svn revert -R .;
        svn update #{revTag};
      else
        svn checkout #{url} #{revTag} .;
      fi;
               EOS
              ))
end
def addUserConfig(build,rev=nil)
  # local inplementation ONLY
  userConfigFile = './config/mh-override'
  unless File.exists?("#{$targetDir[build,false,rev]}/config" )
   sh  "mkdir #{$targetDir[build,false,rev]}/config"
  end
  sh "cp #{userConfigFile} #{$targetDir[build,false,rev]}/config/"
end
def doConfig(build,rev=nil)
  call(connect(build,$configureCall[build],$targetDir[build,false,rev]))
end
def doBuild(build,rev=nil)
  call(connect(build,$buildCall[build],$targetDir[build,false,rev]))
end
def doCreateRunscripts(build,exp='',rev=nil)
  call(connect(build,$runscriptCreateCall[build,exp],$targetDir[build,false,rev]))
end
def doSubmitRunscripts(build,exp='',rev=nil)
  call(connect(build,$runscriptSubmitCall[build,exp],$targetDir[build,false,rev]))
end
def doClean(build)
  call(connect(build,"make clean"))
end
def doCleanSync(build)
  if OCEAN_ONLY then
    call(connect(build,"rm -rf src support schedulers vertical_coord_tables lapack blas data"))
  else
    call(connect(build,"rm -rf src support build"))
  end
end
def doRunExp(build,exp,post=false)
  nprocs = ENV.has_key?('PROCS') ? ENV['PROCS'] : 4
  runScript = (post) ? 'post' : 'exp'
  runScript << '.' + exp +'.run'

  # clean up old experiment folder
  call(connect(build,"cd experiments; rm -rf ./#{exp}"))
  call(connect(build,<<-EOS
      ./make_runscripts #{exp};
      cd run;
      perl -pi.bak -e 's/submit=.*/submit=""/' #{runScript} ;
      perl -pi.bak -e 's/mpi_total_procs=.*/mpi_total_procs=#{nprocs}/' #{runScript} ;
      ./exp.#{exp}.run;
                           EOS
                          ),true)
end
def doSubmit(build,exp,post=false)
  runScript = (post) ? 'post' : 'exp'
  runScript << '.' + exp +'.run'
  call(connect(build,<<-EOS
      cd run;
      perl -pi.bak -e 's/partition=mpi-compute/partition=mpi-develop/' #{runScript} ;
      sbatch ./#{runScript}
      EOS
              ))
end
def doSubmitExp(build,exp)
  doSubmit(build,exp)
end
def doSubmitPost(build,exp)
  doSubmit(build,exp,true)
end
def doRunPost(build,exp)
  call(connect(build,"./make_runscripts ; cd run;./post.#{exp}.run;"))
end
def doCmpExps(build,expA,expB)
  var,ts,lev,show = setSelection

  call(connect(build,<<-EOS
    debug="#{ENV.has_key?('DEBUG')}"
    showDiffSelection="#{show}"
    [[ $debug = 'true' ]] && set -x
    #{joinModelOutput(exp: expA)}
    #{joinModelOutput(exp: expB)}
    #{diffThese("${joinModelOutput_#{expA}}","${joinModelOutput_#{expB}}")}
    [[ $showDiffSelection = 'true' ]] && (
        #{selectBy(var,ts,lev)}
        cdo infov ${selectBy} -sub ${joinModelOutput_#{expA}} ${joinModelOutput_#{expB}}
      )
    #{rmJoinedModelOutput(expA,expB)}
             EOS
            ),true)
end
def doCmpDirs(build,dirA: dirA,dirB: dirB,expA: expA,expB: expB)
  expA = File.basename(dirA) if expA.nil?
  expB = File.basename(dirB) if expB.nil?

  var,ts,lev,show = setSelection

  checkOutput = lambda {|dirA,dirB|
    expTagA, expTagB = 'A', 'B'
    call(connect(build,<<-EOS
      debug="#{ENV.has_key?('DEBUG')}"
      showDiffSelection="#{show}"
      [[ $debug = 'true' ]] && set -x
      #{joinModelOutput(exp: expA,ifilePattern: "#{dirA}/#{expA}*nc*",expTag: expTagA)}
      #{joinModelOutput(exp: expB,ifilePattern: "#{dirB}/#{expB}*nc*",expTag: expTagB)}
      #{diffThese("${joinModelOutput_#{expB}_#{expTagA}}","${joinModelOutput_#{expB}_#{expTagB}}")}
      [[ $showDiffSelection = 'true' ]] && (
          #{selectBy(var,ts,lev)}
          cdo infov ${selectBy} -sub ${joinModelOutput_#{expA}} ${joinModelOutput_#{expB}}
        )
      #{rmJoinedModelOutput(expA,expB)}
    EOS
    ),true)
  }

  checkRestart = lambda{|dirA,dirB|
    call(connect(build,<<-EOS
      debug="#{ENV.has_key?('DEBUG')}"
      showDiffSelection="#{show}"
      [[ $debug = 'true' ]] && set -x
      fileA=#{dirA}/restart_oce_DOM01.nc
      fileB=#{dirB}/restart_oce_DOM01.nc
      if test -f "$fileA" -a -f "$fileB" ; then
        [[ -f diffOut ]]  && rm diffOut
        cdo diffv $fileA $fileB > diffOut;
        if test -s diffOut
        then
          echo "DIFF found in restart files!!"
          cat diffOut
        else
          echo "no DIFF found in restart files"
          echo "FILE:  $fileA ======================"
          echo "FILE:  $fileB ======================"
        fi
      else
        echo "No restart available for #{dirA} of #{dirB}"
      fi;
               EOS
              ),true)
  }

  if ENV.has_key?('CHECK') then
    checks = ENV['CHECK'].split(',')
    checkOutput[dirA,dirB]  if checks.include?('output')
    checkRestart[dirA,dirB] if checks.include?('restart')
  else
    checkOutput[dirA,dirB]
    checkRestart[dirA,dirB]
  end
end
def doCheckExp(build,exp)
  var,ts,lev,show,min,max,mod,showgrid = setSelection

  if ENV.has_key?('CHECK') then
    checks = ENV['CHECK'].split(',')
    checks.each {|check|
      case check
      when 'acc' then
        checkAcc(build,exp,var)
      when 'plot' then
        var = 't' if '__ALL' == var
        showPlot(build,exp,var: var,timestep: ts,level: lev,min: min,max: max,mod: mod, showGrid: showgrid)
      when 'diag'
        showOnlineDiag(build,exp)
      when 'var'
	checkVar(build,exp,var,ts,lev)
      else
        checkNames(build,exp)
      end
    }
  else
    checkNames(build,exp)
    showOnlineDiag(build,exp) if ENV.has_key?('SHOW')
    checkAcc(build,exp,var)
  end
end
def doCheckPost(build,exp,ext)
  pattern = "*_#{exp}_*.#{ext}"
  displayWith = 'eps' == ext ? 'evince' : 'display'
  call(connect(build,<<-EOS
      cd experiments/#{exp}/plots;
      if test -f $(ls -1 #{pattern} | head -1) ; then
               #{displayWith} #{pattern};
      else
        echo 'No postprocessing avialable!';
      fi
               EOS
              ))
end

$builds.each_key {|build|
  # regular tasks with all dependencies {
  desc "sync for build #{build.to_s}"
  task "#{build.to_s}_sync".to_sym do
    doSync(build) unless ENV.has_key?('NOSYNC')
  end
  desc "Configure for build #{build.to_s}"
  task "#{build.to_s}_conf".to_sym => ["#{build.to_s}_sync".to_sym] do
    doConfig(build)
  end
  desc "Build for build #{build.to_s}"
  task "#{build.to_s}_build".to_sym => ["#{build.to_s}_conf".to_sym] do
    doBuild(build)
  end
  desc "Clean repository for build #{build.to_s}"
  task "#{build.to_s}_clean".to_sym do
    doClean(build)
  end
  desc "Run for build #{build.to_s}"
  task "#{build.to_s}_run".to_sym , [:exp] => ["#{build.to_s}_build".to_sym] do |t, args|
    doRunExp(build,args.exp || DEFAULT_EXP)
  end

  #special thunder submits
  if /thunder/.match($builds[build][:host].to_s) then
    desc "Submit experiment on thunder for build #{build.to_s} (incl. preq.)"
    task "#{build.to_s}_submit".to_sym , [:exp] => ["#{build.to_s}_build".to_sym] do |t, args|
      doSubmitExp(build,args.exp || DEFAULT_EXP)
    end
    desc "Submit experiment on thunder for build #{build.to_s}"
    task "#{build.to_s}_submitOnly".to_sym , :exp, :post do |t, args|
      performPostproc = args.post.to_s.to_b || false
      if performPostproc then
        doSubmitPost(build,args.exp || DEFAULT_EXP)
      else
        doSubmitExp(build,args.exp || DEFAULT_EXP)
      end
    end
  end
  desc "Check output files"
  task "#{build.to_s}_check".to_sym , [:exp] => ["#{build.to_s}_run".to_sym] do |t, args|
    doCheckExp(build,args.exp || DEFAULT_EXP)
  end
  desc "Compare outputs of two experiments"
  task "#{build.to_s}_cmp".to_sym, [:expA,:expB] => ["#{build.to_s}_run".to_sym] do |t, args|
    doCmpExps(build,args.expA,args.expB)
  end
  desc "Compare model output of 2 experiment directories"
  task "#{build.to_s}_cmpDirs".to_sym, [:dirA,:dirB,:expA,:expB] do |t, args|
    doCmpDirs(build,dirA: args.dirA,dirB: args.dirB,expA: args.expA, expB: args.expB)
  end

  # post processing
  desc "Run post-processing on given exp (build:#{build.to_s})"
  task "#{build.to_s}_post".to_sym, [:exp] => ["#{build.to_s}_sync".to_sym] do |t, args|
    doRunPost(build,args.exp || DEFAULT_EXP)
  end
  # check post processing
  desc "Display post-processing output for given exp (build:#{build.to_s})"
  task "#{build.to_s}_checkPost".to_sym, :exp, :ext  do |t, args|
    doCheckPost(build,
                args.exp || DEFAULT_EXP,
                args.ext || 'eps')
  end
  # diff two text files with (option) tool
  desc "diff two text files with (option) tool"
  task "#{build.to_s}_diff".to_sym, :fA, :fB, :tool  do |t, args|
    tool =  'meld' if args.tool.nil?
    doDiffThis(build,args.fA, args.fB, tool)
  end

  # }
  # TASKS FOR ONLY DO ONE SINGLE THING {
  desc "Configure for build #{build.to_s}"
  task "#{build.to_s}_confOnly".to_sym do
    doConfig(build)
  end
  desc "Build for build #{build.to_s}"
  task "#{build.to_s}_buildOnly".to_sym => ["#{build.to_s}_sync".to_sym] do
    doBuild(build)
  end
  desc "Run for build #{build.to_s}"
  task "#{build.to_s}_runOnly".to_sym, [:exp] => ["#{build.to_s}_sync".to_sym] do |t,args|
    doRunExp(build,args.exp || DEFAULT_EXP)
  end
  desc "Check output files"
  task "#{build.to_s}_checkOnly".to_sym , [:exp] => ["#{build.to_s}_sync".to_sym] do |t, args|
    doCheckExp(build,args.exp || DEFAULT_EXP)
  end
  desc "Compare outputs of two experiments"
  task "#{build.to_s}_cmpOnly".to_sym, [:expA,:expB] => ["#{build.to_s}_sync".to_sym] do |t, args|
    doCmpExps(build,args.expA,args.expB)
  end
  # diff two text files with (option) tool
  desc "diff two text files with (option) tool"
  task "#{build.to_s}_diff".to_sym, :fA, :fB, :tool  do |t, args|
    tool =  'meld' if args.tool.nil?
    doDiffThis(build,args.fA, args.fB, tool)
  end

  # }

  desc "remove old source trees and sync"
  task "#{build.to_s}_cleanSync".to_sym do
    doCleanSync(build)
    doSync(build)
  end

  desc "build special revision"
  task "#{build.to_s}_buildRev".to_sym, [:rev,:url] do |t,args|
    doCheckoutRev(build,args.rev,args.url)
    addUserConfig(build,args.rev) if isLocal?(build)
    doConfig(build,args.rev)
    doBuild(build,args.rev)
  end

  desc "build one/mutiple runscript(s)"
  task "#{build.to_s}_buildRun".to_sym, [:exp,:rev] do |t,args|
    doCreateRunscripts(build,args.exp,args.rev)
  end
  desc "submit one/mutiple runscript(s)"
  task "#{build.to_s}_submitRun".to_sym, [:exp,:rev] do |t,args|
    doSubmitRunscripts(build,args.exp,args.rev)
  end
}
desc "Sync source codes for all builds"
task :allSync do
  q = JobQueue.new
  $builds.each_key {|build| q.push { doSync(build) } }
  q.run
end
desc "Remove and Sync source codes for all builds"
task :allCleanSync do
  q = JobQueue.new
  $builds.each_key {|build| q.push { doCleanSync(build) } }
  q.run
end

# tasks for parallel run of other tasks in an Xtra terminal
[
  :par,      # execute everything in the calling terminal
  :parX,     # shut the terminal when finished
  :parXWait  # keep the terminal open when finished (shut with ENTER)
].each {|parTaskName|
  terminalEmulators = ['gnome-terminal','xterm']
  help = case parTaskName
         when :par then
           "execute all tasks in parallel"
         when :parX then
           "parallel run of other tasks in an Xtra terminal - shut when finished"
         when :parXWait then
           "parallel run of other tasks in an Xtra terminal - keep open when finished"
         else
           warn "Unknown task"
           exit 1
         end
  desc help
  task parTaskName do
    dbg(Rake.application.top_level_tasks)

    # remote the task from the internal task list because they will be started in a sub-thread
    Rake.application.top_level_tasks.clear

    # create a task list from the command line
    ARGV.shift
    taskList = ARGV
    dbg(taskList)

    # parallel execution queue
    q = JobQueue.new

    # execute everything from the cmldline, which has a corresponding task
    taskList.each {|taskName|
      dbg(taskName)

      case parTaskName
      when :par then
        task = Rake::Task.tasks.find {|t| taskName == t.name }
        if task.nil? then
          warn "Unkown task: #{taskName}"
          exit 2
        else
          q.push { task.invoke }
        end

      when :parX,:parXWait
        cmd     = "#{terminalEmulators[1]} -T 'rake #{taskName}' -e 'rake #{taskName};'"
        cmd[-1] = "read'" if :parXWait == parTaskName
        q.push { call(cmd) }

      end
    }
    q.run
  end
}

desc "Check variables an user input"
task :check ,[:p] do |t,args|
  pp args
  {
    'SRCDIR' => SRCDIR,
    'REMOTE_USER' => REMOTE_USER,
    'WORKDIR' => WORKDIR,
    'RAKEFILE' => __FILE__,
    'OCEAN_ONLY' => OCEAN_ONLY.to_s,
  }.each {|k,v|
    puts [k.rjust(16,' '),v.ljust(20,' ')].join(': ')
  }
end

desc "(Re)Build ctags db"
task :tags do
  sh "ctags  --fortran-kinds=efikmpstv #{@fileList.join(' ')}"
end

# vim:ft=ruby fdm=syntax
