#! /bin/ksh
###############################################################################
#                              esm-ssp585-ocn-alk_EXP3.run
#
# Generated by Make Experiments! (mkexp) 1.1.5rc2
#
# $Id: DEFAULT.run.tmpl 1 2021-02-01 14:22:42Z m221078 $
#
# $Id: DEFAULT.config 1 2021-02-03 20:53:47Z m221078 $
# $Id$
# $Id$
# $Id: CDRSYNTRA_OUTPUT.config 9899 2019-07-01 09:57:06Z m221078 $
# $Id: levante.config $
#
###############################################################################

#
# Setup for levante (SLURM)
#
# $Id: levante.tmpl $
#
#SBATCH --job-name=esm-ssp585-ocn-alk_EXP3_run
#SBATCH --partition=compute
#SBATCH --exclusive
#SBATCH --ntasks=960
#SBATCH --time=00:59:00
#SBATCH --output=%x_%j.log
#SBATCH --mail-type=FAIL,ARRAY_TASKS
#SBATCH --account=bm1241
#SBATCH --propagate=STACK,CORE

ulimit -s 390625 # * 1024 B = 400 MB

# OpenMPI
export HDF5_USE_FILE_LOCKING=FALSE
export MALLOC_TRIM_THRESHOLD_="-1"
export KMP_AFFINITY="granularity=fine,scatter"
export KMP_LIBRARY="turnaround"

export MKL_DEBUG_CPU_TYPE=5
export MKL_ENABLE_INSTRUCTIONS=AVX2

export OMPI_MCA_btl=self
export OMPI_MCA_coll="^ml"
export OMPI_MCA_coll_hcoll_enable="1"
export OMPI_MCA_io="romio321"
export OMPI_MCA_osc="pt2pt"
export OMPI_MCA_pml="ucx"

export HCOLL_ENABLE_MCAST_ALL="1"
export HCOLL_MAIN_IB=mlx5_0:1

export UCX_IB_ADDR_TYPE=ib_global
export UCX_NET_DEVICES=mlx5_0:1
export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self
export UCX_UNIFIED_MODE=y

export SLURM_CPU_FREQ_REQ=High
# Intel OpenMP
export KMP_STACKSIZE=128M
# OpenMP
export OMP_NUM_THREADS=1
# Workaround for SLURM bug in chained jobs
SLURM_JOB_NAME=esm-ssp585-ocn-alk_EXP3_run
sbatch () {
    unset SLURM_MEM_PER_CPU SLURM_NTASKS_PER_NODE
    command sbatch "$@"
}

###############################################################################

DEBUG_LEVEL=${DEBUG_LEVEL:-0}

# Support log style output
export LANG=C
print () { command print "$(date +'%F %T'):" "$@"; }
print_re='^[0-9]+-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}'

warn () { print 'Hey:' "$@" >&2; }
die () { print 'Oops:' "$@" >&2; return 1; }

# Bail out on error
trap 'print Error at line $LINENO >&2' ERR
set -eu
# Print command info
[[ $DEBUG_LEVEL -ge 2 ]] && set -x

###############################################################################
#
#     USER INTERFACE
#
###############################################################################

#------------------------------------------------------------------------------
#   Configuration of component(s)
#------------------------------------------------------------------------------

_mlre=${_mlre:-}
_mlv=${_mlv:-}
_sp_flags=${_sp_flags:-}
ATM_LEVELS=${ATM_LEVELS:-47}
ATM_RES=${ATM_RES:-T63}
ATMO_EXE=${ATMO_EXE:-echam6}
ATMO_RESTART_DIR=${ATMO_RESTART_DIR:-/work/bm1241/m300966/CDRSynTra-mpiesm-1.2.01p7/experiments/esm-ssp585-ocn-alk_EXP3/restart/echam6}
COUPLER_RESTART_DIR=${COUPLER_RESTART_DIR:-/work/bm1241/m300966/CDRSynTra-mpiesm-1.2.01p7/experiments/esm-ssp585-ocn-alk_EXP3/restart/oasis3mct}
delete=${delete:-}
EXP_ID=${EXP_ID:-esm-ssp585-ocn-alk_EXP3}
hist_or_scn=${hist_or_scn:-}
historical_or_scn=${historical_or_scn:-}
label=${label:-}
LAND_RESTART_DIR=${LAND_RESTART_DIR:-/work/bm1241/m300966/CDRSynTra-mpiesm-1.2.01p7/experiments/esm-ssp585-ocn-alk_EXP3/restart/jsbach}
OBGC_RESTART_DIR=${OBGC_RESTART_DIR:-/work/bm1241/m300966/CDRSynTra-mpiesm-1.2.01p7/experiments/esm-ssp585-ocn-alk_EXP3/restart/hamocc}
OCE_RES=${OCE_RES:-GR15}
OCEAN_EXE=${OCEAN_EXE:-mpiom.x}
OCEAN_RESTART_DIR=${OCEAN_RESTART_DIR:-/work/bm1241/m300966/CDRSynTra-mpiesm-1.2.01p7/experiments/esm-ssp585-ocn-alk_EXP3/restart/mpiom}
prevdate=${prevdate:-}
startmonth=${startmonth:-}
var=${var:-}
which_declare=${which_declare:-}
yr0=${yr0:-}
yrN=${yrN:-}


#------------------------------------------------------------------------------
# End of component configuration
#------------------------------------------------------------------------------

#------------------------------------------------------------------------------
#   1.3 TIME CONTROL
#------------------------------------------------------------------------------

#-- calendar type: Available calendar options:
#     0   : No leap year (365 days per year)
#     1   : Gregorian (365/366 days per year)
#     n   : Equal months of "n" days (30 for 30 day months)

CALENDAR=1

#
#-- initial and final date of the experiment
#   Format: YearMMDD[_hh[mm[ss]]], Year-MM-DD[_hh[:mm[:ss]]] or 
#           Year-MM-DD[Thh[:mm[:ss]]]
#   Note: The experiment will not stop within a run/chunk even if the
#         final date is reached.

initial_date=2020-01-01 # initial exp. date
final_date=2100-12-31 # final date of the experiment

#
#-- duration of a run/chunk
#      Specify the length of each run in one of the below units.

integer nyear nmonth nday nhour nminute nsecond

nyear=0          # number of years per run
nmonth=12 # number of months per run
nday=0 # number of days per run
nhour=0          # number of hours per run
nminute=0        # number of minutes per run
nsecond=0        # number of seconds per run
nstep_atm=0      # number of atmosphere model time steps per run
nstep_oce=0      # number of ocean model time steps per run


#------------------------------------------------------------------------------
#   1.4 INITIAL SETTINGS
#------------------------------------------------------------------------------

ATMO_DISTURBED_YEARS=''
ATMO_DISTURBANCE=1.00001

#------------------------------------------------------------------------------
#   1.6 FILE SYSTEMS
#------------------------------------------------------------------------------

#
#-- BIN_DIR, SCRIPT_DIR:
#          Permanent file system for the SCRIPTS/BINARIES on the COMPUTING HOST
#          (only needs to be specified if the tasks are NOT generated on the 
#          computing host)

BIN_DIR=/home/m/m300966/CDRSynTra-mpiesm-1.2.01p7/bin
SCRIPT_DIR=/home/m/m300966/CDRSynTra-mpiesm-1.2.01p7/experiments/esm-ssp585-ocn-alk_EXP3/scripts

#
#-- *_INPUT_DIR: Directories of the LONG TERM INPUT data archive. It needs
#                to reside on the same machine as the output archive. This 
#                archive is intended for input data that is needed with 
#                several experiments, e.g. initial , forcing or restart files.
#                The parent-directory needs to exist before job submission.

ATMO_INPUT_DIR=/pool/data/ECHAM6/input/r0008
LAND_INPUT_DIR=/pool/data/JSBACH/input/r0010
HYDRO_INPUT_DIR=/pool/data/JSBACH/input/r0010/HD
OCEAN_INPUT_DIR=/pool/data/MPIOM/input/r0010
OBGC_INPUT_DIR=/pool/data/MPIOM/input/r0010
COUPLER_INPUT_DIR=/pool/data/MPIESM/input/r0001

#
#-- DATA_DIR, RESTART_DIR, LOG_DIR, MON_DIR: 
#          Directories of the SHORT TERM data server.
#          Model INPUT and OUTPUT will be read from/written to 
#          this file system of the computing host
#
#  - The parent-directory needs to exist before submission of the job 

DATA_DIR=/work/bm1241/m300966/CDRSynTra-mpiesm-1.2.01p7/experiments/esm-ssp585-ocn-alk_EXP3/outdata
RESTART_DIR=/work/bm1241/m300966/CDRSynTra-mpiesm-1.2.01p7/experiments/esm-ssp585-ocn-alk_EXP3/restart
LOG_DIR=/work/bm1241/m300966/CDRSynTra-mpiesm-1.2.01p7/experiments/esm-ssp585-ocn-alk_EXP3/log
MON_DIR=/work/bm1241/m300966/CDRSynTra-mpiesm-1.2.01p7/experiments/monitoring

#
#-- WORK_DIR: Root directory for the temporary working directory
#             (for production runs use $TMPDIR on NEC)
#

WORK_DIR=/scratch/m/m300966/CDRSynTra-mpiesm-1.2.01p7/experiments/esm-ssp585-ocn-alk_EXP3/work

#
# Command line settings
#

FORCE_RUN=false

while [[ "${1-}" ]]
do
    case "$1" in
       --force|-f) FORCE_RUN=true;;
       -*) die "invalid option '$1'";;
       *) die "unexpected parameter '$1'";;
    esac
    shift
done

#
# Helpers wrt options above
#

die_unforced () { die "$*. It seems that the run for $startdate-$enddate failed or is still running. If this is expected, consider using '--force' to continue"; }

#
#-- Path to the IMDI function directory
#

export PATH=/home/m/m300966/CDRSynTra-mpiesm-1.2.01p7/util/running/functions:/home/m/m300966/CDRSynTra-mpiesm-1.2.01p7/util:$PATH

#------------------------------------------------------------------------------
#   UNIX COMMANDS
#------------------------------------------------------------------------------

mkdir () { command mkdir -vp "$@"; }
rm () { command rm -vf "$@"; }

time_diff () { command time_diff -c$CALENDAR "$@"; }
function calc_date  {
    typeset command="$1"
    shift
    command calc_date $command -c$CALENDAR "$@"
}

###############################################################################
#
#      END OF THE USER INTERFACE
#
###############################################################################

#------------------------------------------------------------------------------
#  Complete setup 
#------------------------------------------------------------------------------

#
#-- MPIOM
#
nodt=2700

#
#-- ECHAM
#
ATM_TIME_STEP=450

#------------------------------------------------------------------------------
#   Job specification
#------------------------------------------------------------------------------

job=${SLURM_JOB_NAME:-${EXP_ID}_run}
jobid=${SLURM_JOBID:-$$}
jobdir=${SCRIPT_DIR} # script directory

#------------------------------------------------------------------------------
#
#     3. CALENDAR
#
#------------------------------------------------------------------------------
#
#-- calculate length of the run in seconds for the case that (optionally)
#   the length of run is given in number of model steps of any of the models.
#
if   [ "${nstep_atm}" -ne 0 ] && [ "${nstep_atm}" -ne "" ]; then
  (( nsecond = nstep_atm * ATM_TIME_STEP ))
elif [ "${nstep_oce}" -ne 0 ] && [ "${nstep_oce}" -ne "" ]; then
  (( nsecond = nstep_oce * nodt ))
fi

#
#-- find out smallest time unit in inidate and job length
#

format=
if [ ${nsecond} -ne 0 ]; then
  format=-s
elif [ ${nminute} -ne 0 ]; then
  format=-m
elif [ ${nhour} -ne 0 ]; then
  format=-h
fi
# transform to format (YearMMDD_hhmmss)
inidate=`format_date $format -- ${initial_date}`
findate=`format_date $format -- ${final_date}`

#
# Read date and register of this run
#

cd ${jobdir}

datefmt='%a %b %d %H:%M:%S %Z %Y'  # date format for EXP_ID.log file

read startdate jobnum < ${EXP_ID}.date

# Compute dependent date variables

nextdate=$(calc_date plus -Y${nyear} -M${nmonth} -D${nday} -h${nhour} -m${nminute} -s${nsecond} -- ${startdate}_)
enddate=$(calc_date minus -D1 -- ${nextdate}_)
endtime=$(calc_date minus -s$((ATM_TIME_STEP * 2)) -- ${nextdate}_)
endtime=${endtime//_/}

logstamp=$(date +"${datefmt}")
cp ${EXP_ID}.log ${EXP_ID}.log.new &&
echo "$logstamp :  ${jobnum} ${startdate} ${jobid}  - start" >> ${EXP_ID}.log.new &&
mv ${EXP_ID}.log.new ${EXP_ID}.log ||
    die "error saving experiment status, check disk space and quota for $jobdir"

# Check available disk space on data directory

mkdir $DATA_DIR
(( $(stat -f -c'100*%a/%b' $DATA_DIR) > 0 )) ||
    die "< 1% free on data file system ($DATA_DIR), bailing out"

# Move previous log file to log directory

prevstartdate=$(calc_date minus -M12 -- ${startdate}_)
prevstartdate=$(calc_date minus -D0 -- ${prevstartdate}_)

mkdir $LOG_DIR
for log_path in $(grep -El "$print_re: model run finished for $prevstartdate" $SCRIPT_DIR/${EXP_ID}_run*_[0-9]*.log || :)
do
    log_file=${log_path##*/}
    # The * after run might contain underscores or numbers,
    # so make sure we only take the latter part as previous job's id
    log_prefix=${log_file%_[0-9]*.log}
    log_suffix=${log_file#$log_prefix}
    mv $log_path $LOG_DIR/${EXP_ID}_run_${prevstartdate}$log_suffix
done

#------------------------------------------------------------------------------
#   PRE - PROCESSING : check working directory, clean, and cd
#------------------------------------------------------------------------------

# Temporary directory

run_dir=$WORK_DIR/run_$startdate-$enddate

print "model run started for $startdate-$enddate"
print "in $run_dir"

# Show progress
sh -c 'scontrol update JobId=$SLURM_JOB_ID Comment="$*"' scomment $startdate

if [[ -d $run_dir ]]
then
    if $FORCE_RUN
    then
        rm -r $run_dir
    else
        die_unforced "directory '$run_dir' found"
    fi
fi

mkdir $run_dir
cd $run_dir

#------------------------------------------------------------------------------
#     PRE - PROCESSING : provide the input data 
#------------------------------------------------------------------------------

print "preparing input data for $startdate started"

# Set date variables for input files

prevdate=$(calc_date minus -D1 -- ${startdate}_)

startvals=($(format_date -f4 ${startdate}_))
endvals=($(format_date -f4 ${enddate}_))

yr0=$(printf %04d ${startvals[0]})
yrN=$(printf %04d ${endvals[0]})

startmonth=${startvals[1]}

# Format input files using date variables

# $Id: format.tmpl 9264 2021-06-21 21:24:57Z m221078 $
#
# [files]

#
#-- input files for OASIS
#
# [files.oasis]
oasis_dir=/pool/data/MPIESM/input/r0001/${ATM_RES}_${OCE_RES}
ln -sfv $oasis_dir/rmp_oces_to_atmo_CONSERV_FRACAREA_${ATM_RES}_${OCE_RES}.nc .
ln -sfv $oasis_dir/rmp_atmo_to_oces_CONSERV_FRACAREA_${ATM_RES}_${OCE_RES}.nc .
ln -sfv $oasis_dir/rmp_atml_to_oces_CONSERV_FRACAREA_${ATM_RES}_${OCE_RES}.nc .
ln -sfv $oasis_dir/rmp_atmo_to_oces_BICUBIC_D_${ATM_RES}_${OCE_RES}.nc .

# [files.oasis.grids]
grids_dir=$oasis_dir
install -pvm 644 $grids_dir/grids_${ATM_RES}_${OCE_RES}_frac.nc grids.nc
install -pvm 644 $grids_dir/areas_${ATM_RES}_${OCE_RES}_frac.nc areas.nc
install -pvm 644 $grids_dir/masks_${ATM_RES}_${OCE_RES}_frac.nc masks.nc

# [files.oasis.resources]
resources_dir=/home/m/m300966/CDRSynTra-mpiesm-1.2.01p7/util/running/adjunct_files/oasis3mct
ln -sfv $resources_dir/cf_name_table.txt .

# [files.oasis.restart]
restart_dir=/work/bm1241/m300966/CDRSynTra-mpiesm-1.2.01p7/experiments/esm-ssp585-ocn-alk_EXP3/restart/oasis3mct
tar -xf $restart_dir/flxatmos_${EXP_ID}_${prevdate}.tar
tar -xf $restart_dir/sstocean_${EXP_ID}_${prevdate}.tar

#
#-- input files for ECHAM
#
# [files.echam]
echam_dir=/pool/data/ECHAM6/input/r0008
ln -sfv $echam_dir/rrtmg_lw.nc .
ln -sfv $echam_dir/rrtmg_sw.nc .
ln -sfv $echam_dir/ECHAM6_CldOptProps.nc .
# not needed for CO2 but for the other trace gases
ln -sfv $echam_dir/greenhouse_ssp585.nc greenhouse_gases.nc
ln -sfv $echam_dir/MAC-SP_ssp585.nc MAC-SP.nc

# [files.echam.mapped]
mapped_dir=$echam_dir/${ATM_RES}
ln -sfv $mapped_dir/${ATM_RES}${OCE_RES}_VGRATCLIM.nc unit.91
ln -sfv $mapped_dir/${ATM_RES}${OCE_RES}_VLTCLIM.nc unit.90
ln -sfv $mapped_dir/${ATM_RES}_TSLCLIM.nc unit.92
ln -sfv $mapped_dir/${ATM_RES}L${ATM_LEVELS}_jan_spec.nc unit.23
ln -sfv $mapped_dir/${ATM_RES}${OCE_RES}_jan_surf.nc unit.24
ln -sfv $mapped_dir/${ATM_RES}_carbon_emissions_ssp585.nc carbon_emissions.nc

# [files.echam.mapped.aero]
aero_dir=$mapped_dir/aero
# echam.radctl.iaero = 8 needs 1850 aerosols
ln -sfv $aero_dir/${ATM_RES}_aeropt_kinne_sw_b14_fin_1865.nc aero_fine_1849.nc
ln -sfv $aero_dir/${ATM_RES}_aeropt_kinne_sw_b14_coa.nc aero_coarse_1849.nc
ln -sfv $aero_dir/${ATM_RES}_aeropt_kinne_lw_b16_coa.nc aero_farir_1849.nc
ln -sfv $aero_dir/${ATM_RES}_aeropt_kinne_sw_b14_fin_1865.nc aero_fine_1850.nc
ln -sfv $aero_dir/${ATM_RES}_aeropt_kinne_sw_b14_coa.nc aero_coarse_1850.nc
ln -sfv $aero_dir/${ATM_RES}_aeropt_kinne_lw_b16_coa.nc aero_farir_1850.nc
ln -sfv $aero_dir/${ATM_RES}_aeropt_kinne_sw_b14_fin_1865.nc aero_fine_1851.nc
ln -sfv $aero_dir/${ATM_RES}_aeropt_kinne_sw_b14_coa.nc aero_coarse_1851.nc
ln -sfv $aero_dir/${ATM_RES}_aeropt_kinne_lw_b16_coa.nc aero_farir_1851.nc

# [files.echam.varying]
for((yr = yr0 + -2; yr <= yrN + 1; ++yr)) 
do
    varying_dir=$echam_dir

    # [files.echam.varying.mapped]
    mapped_dir=$varying_dir/${ATM_RES}

    # [files.echam.varying.mapped.ozone]
    historical_or_scn=historical
        ((yr >= 2015)) && historical_or_scn=ssp585
    label=1850
        ((yr >= 1850)) && label=${yr}
        ((yr >= 2100)) && label=2099
    ozone_dir=$mapped_dir/ozone
    ln -sfv $ozone_dir/${ATM_RES}_ozone_${historical_or_scn}_${label}.nc \
        ozon${yr}

    # [files.echam.varying.mapped.aero]
    aero_dir=$mapped_dir/aero

    # [files.echam.varying.mapped.volcano_aerosols]
    label=1850
        ((yr >= 1850)) && label=${yr}
        ((yr >= 2025)) && label=2024
    volcano_aerosols_dir=$mapped_dir/volcano_aerosols
    ln -sfv $volcano_aerosols_dir/strat_aerosol_ir_${ATM_RES}_${label}.nc \
        strat_aerosol_ir_${yr}.nc
    ln -sfv $volcano_aerosols_dir/strat_aerosol_sw_${ATM_RES}_${label}.nc \
        strat_aerosol_sw_${yr}.nc

    # [files.echam.varying.solar_irradiance]
    label=1850
        ((yr >= 1850)) && label=${yr}
    solar_irradiance_dir=$varying_dir/solar_irradiance
    ln -sfv $solar_irradiance_dir/swflux_14band_${label}.nc swflux_${yr}.nc
done # offsets

#
#-- restart files for ECHAM
#
# [files.echam.restart]
restart_dir=/work/bm1241/m300966/CDRSynTra-mpiesm-1.2.01p7/experiments/esm-ssp585-ocn-alk_EXP3/restart/echam6
install -pvm 644 $restart_dir/restart_${EXP_ID}_echam_${prevdate}.nc \
    restart_${EXP_ID}_echam.nc
install -pvm 644 $restart_dir/restart_${EXP_ID}_co2_${prevdate}.nc \
    restart_${EXP_ID}_co2.nc
install -pvm 644 $restart_dir/restart_${EXP_ID}_accw_${prevdate}.nc \
    restart_${EXP_ID}_accw.nc
install -pvm 644 $restart_dir/restart_${EXP_ID}_tracer_${prevdate}.nc \
    restart_${EXP_ID}_tracer.nc

#
#-- input files for JSBACH/HD
#
# [files.jsbach]
jsbach_dir=/pool/data/JSBACH/input/r0010

# [files.jsbach.resources]
ln -sfv \
    /home/m/m300966/CDRSynTra-mpiesm-1.2.01p7/util/running/adjunct_files/jsbach/lctlib_nlct21.def \
    lctlib.def

# [files.jsbach.mapped]
mapped_dir=$jsbach_dir/${ATM_RES}
ln -sfv $mapped_dir/jsbach_${ATM_RES}${OCE_RES}_11tiles_5layers_1850_dynveg.nc \
    jsbach.nc

# [files.jsbach.mapped.landuse]
for((yr = yr0 + -1; yr <= yrN + 0; ++yr)) 
do
    hist_or_scn=hist
        ((yr >= 2015)) && hist_or_scn=ssp585
    landuse_dir=$mapped_dir/New_Hampshire_LCC
    ln -sfv \
        $landuse_dir/${hist_or_scn}_dynveg/LUH_scaled_transitions_${ATM_RES}_${yr}.nc \
        landuseTransitions.${yr}.nc
    ln -sfv $landuse_dir/${hist_or_scn}_harvest/LUH_harvest_${ATM_RES}_${yr}.nc \
        landuseHarvest.${yr}.nc
done # offsets

# [files.jsbach.mapped.nitrogen]
for((yr = yr0 + -1; yr <= yrN + 0; ++yr)) 
do
    label=1850
        ((yr >= 1850)) && label=${yr}
        ((yr >= 2100)) && label=2099
    nitrogen_dir=$mapped_dir
    cdo selyear,${label} \
        $nitrogen_dir/T63_ndepo.input4MIPs_surfaceFluxes_ScenarioMIP-NCAR-CCMI-ssp585.nc \
        Ndepo.${yr}.nc
done # offsets

# [files.jsbach.mapped.spitfire]
spitfire_dir=$mapped_dir/spitfire
ln -sfv $spitfire_dir/a_nd_${ATM_RES}.nc a_nd_file.nc
ln -sfv $spitfire_dir/LISOTD_HRMC_V2.2_${ATM_RES}.nc lightning.nc
ln -sfv $spitfire_dir/population_density_ssp5_${ATM_RES}.nc \
    population_density.nc

# [files.jsbach.hydro]
hydro_dir=/pool/data/JSBACH/input/r0010/HD
ln -sfv $hydro_dir/hdpara.nc hdpara.nc
ln -sfv $hydro_dir/hdstart.nc hdstart.nc
ln -sfv $hydro_dir/rmp_${ATM_RES}_to_hd.nc rmp_hd.nc

#
#-- restart files for JSBACH/HD
#
# [files.jsbach.restart]
restart_dir=/work/bm1241/m300966/CDRSynTra-mpiesm-1.2.01p7/experiments/esm-ssp585-ocn-alk_EXP3/restart/jsbach
install -pvm 644 $restart_dir/restart_${EXP_ID}_jsbach_${prevdate}.nc \
    restart_${EXP_ID}_jsbach.nc
install -pvm 644 $restart_dir/restart_${EXP_ID}_veg_${prevdate}.nc \
    restart_${EXP_ID}_veg.nc
install -pvm 644 $restart_dir/restart_${EXP_ID}_surf_${prevdate}.nc \
    restart_${EXP_ID}_surf.nc
install -pvm 644 $restart_dir/restart_${EXP_ID}_yasso_${prevdate}.nc \
    restart_${EXP_ID}_yasso.nc
install -pvm 644 $restart_dir/restart_${EXP_ID}_hd_${prevdate}.nc hdrestart.nc
install -pvm 644 $restart_dir/restart_${EXP_ID}_nitro_${prevdate}.nc \
    restart_${EXP_ID}_nitro.nc

#
#-- input files for MPIOM
#
# [files.mpiom]
mpiom_dir=/pool/data/MPIOM/input/r0010/${OCE_RES}
ln -sfv $mpiom_dir/${OCE_RES}_anta anta
ln -sfv $mpiom_dir/${OCE_RES}_arcgri.nc arcgri.nc
ln -sfv $mpiom_dir/${OCE_RES}_topo topo

# [files.mpiom.rewrite]
rewrite_dir=$mpiom_dir
install -pvm 644 $rewrite_dir/${OCE_RES}_BEK BEK

#
#-- restart files for MPIOM
#
# [files.mpiom.restart]
restart_dir=/work/bm1241/m300966/CDRSynTra-mpiesm-1.2.01p7/experiments/esm-ssp585-ocn-alk_EXP3/restart/mpiom
install -pvm 644 $restart_dir/rerun_${EXP_ID}_mpiom_${prevdate}.nc \
    rerun_${EXP_ID}_mpiom.nc

#
#-- input files for HAMOCC
#
# [files.hamocc]
hamocc_dir=/pool/data/MPIOM/input/r0010

# [files.hamocc.mapped]
mapped_dir=$hamocc_dir/${OCE_RES}
cdo_rotate_clim $startmonth $mapped_dir/${OCE_RES}_MAHOWALDDUST_2005.nc \
    dust_deposition.nc

cdo_rotate_clim $startmonth /work/bm1241/m300966/OAE_files/${EXP_ID}/${EXP_ID}_alkalinity_addition_${startdate:0:4}.nc  \
    alkalinity_input.nc

# [files.hamocc.mapped.first_year]
yr=${yr0}
label=1850
    ((yr >= 1850)) && label=${yr}
    ((yr >= 2100)) && label=2099
 yr=${startmonth}
    ((yr >= 1)) && delete=
    ((yr >= 2)) && delete=delete,timestep=1
    ((yr >= 3)) && delete=delete,timestep=1/$((yr - 1))
first_year_dir=$mapped_dir
cdo ${delete} -selyear,${label} \
    $first_year_dir/${OCE_RES}_ndepo.input4MIPs_surfaceFluxes_ScenarioMIP-NCAR-CCMI-ssp585.nc \
    nitrogen_deposition.nc

# [files.hamocc.mapped.other_years]
for((yr = yr0 + 1; yr <= yrN + 1; ++yr)) 
do
    label=1850
        ((yr >= 1850)) && label=${yr}
        ((yr >= 2100)) && label=2099
    other_years_dir=$mapped_dir
    cdo cat -selyear,${label} \
        $other_years_dir/${OCE_RES}_ndepo.input4MIPs_surfaceFluxes_ScenarioMIP-NCAR-CCMI-ssp585.nc \
        nitrogen_deposition.nc
done # offsets

#
#-- restart files for HAMOCC
#
# [files.hamocc.restart]
restart_dir=/work/bm1241/m300966/CDRSynTra-mpiesm-1.2.01p7/experiments/esm-ssp585-ocn-alk_EXP3/restart/hamocc
install -pvm 644 $restart_dir/rerun_${EXP_ID}_hamocc_${prevdate}.nc \
    rerun_${EXP_ID}_hamocc.nc

# [files.hamocc.annual]
annual_dir=$hamocc_dir


print "preparing input data for $startdate finished"

#------------------------------------------------------------------------------
#    PRE - PROCESSING : provide and update configuration files (namelists etc.)
#------------------------------------------------------------------------------

print "preparing configuration files started"

#------------------------------------------------------------------------------
#-- Namelist OASIS3 (namcouple)
#

# runtime: duration of the experiment (seconds)
runtime=$(time_between -- ${startdate} ${nextdate} seconds)

# Runtime needs to be adjusted.
# 'namcouple' keywords use a $ prefix so the here doc needs to be quoted.
sed "
    s/%runtime/$runtime/
" > namcouple << 'EOF'
###############################################################################
#
#      Input file for OASIS3-MCT
#
#      $Id$
#
#      This version is for use with ECHAM6 fluxes and
#      surface conditions computed in MPIOM.
#
#      The file will be edited in the run-script to update it for the
#      actual integration period and grid dimensions.
#
###############################################################################
#
#      Input delimiters have to occupy position 1 to 9 !
#      No blank lines allowed !
#      Length of input lines <= 80 !
#
###############################################################################
#
# NFIELDS : total number of fields being exchanged.
#
 $NFIELDS
            21
 $END
#
###############################################################################
#
# NBMODEL : number of models and their names (CHAR*6).
#
 $NBMODEL
            2 mpiom echam6
 $END
#
###############################################################################
#
# RUNTIME (<I10)
#         total simulated time for the actual run in seconds
#         the value will be set before the actual integration
#
 $RUNTIME
           %runtime
 $END
#
###############################################################################
#
 $NLOGPRT
           0
 $END
#
###############################################################################
#
###############################################################################
#
 $STRINGS
#
###############################################################################
#
# Field 1 Sea_surface_temperature [K;K]
#
SSTOCEAN SSTATMOS 1 86400 3 sstoce1 EXPORTED
254 220 192 96 oces atmo SEQ=1 LAG=2700
P  0  P  0
LOCTRANS SCRIPR MAPPING
INSTANT
CONSERV LR SCALAR LATITUDE 40 FRACAREA FIRST
rmp_oces_to_atmo_CONSERV_FRACAREA_T63_GR15.nc dst
#
###############################################################################
#
# Field 2 Sea_ice_thickness  [m;m]
#
SITOCEAN SITATMOS 45 86400  3 sstoce2  EXPORTED
254 220 192 96 oces atmo SEQ=1  LAG=2700
P  0  P  0
LOCTRANS SCRIPR MAPPING
INSTANT
CONSERV LR SCALAR LATITUDE 40 FRACAREA FIRST
rmp_oces_to_atmo_CONSERV_FRACAREA_T63_GR15.nc dst
#
###############################################################################
#
# Field 3 Sea_ice_area_fraction  [;]
#
SICOCEAN SICATMOS 31 86400  3 sstoce3  EXPORTED
254 220 192 96 oces atmo SEQ=1  LAG=2700
P  0  P  0
LOCTRANS SCRIPR MAPPING
INSTANT
CONSERV LR SCALAR LATITUDE 40 FRACAREA FIRST
rmp_oces_to_atmo_CONSERV_FRACAREA_T63_GR15.nc dst
#
###############################################################################
#
# Field 4 lwe_surface_snow_thickness_where_sea_ice  [m;m]
#
SNTOCEAN SNTATMOS 46 86400  3 sstoce4  EXPORTED
254 220 192 96 oces atmo SEQ=1 LAG=2700  
P  0  P  0
LOCTRANS SCRIPR MAPPING
INSTANT
CONSERV LR SCALAR LATITUDE 40 FRACAREA FIRST
rmp_oces_to_atmo_CONSERV_FRACAREA_T63_GR15.nc dst
#
###############################################################################
#
# Field 5 ocean eastw. velocity  [m/s]
#
OCUOCEAN OCUATMOS 47 86400  3 sstoce5  EXPORTED
254 220 192 96 oces atmo SEQ=1 LAG=2700
P  0  P  0
LOCTRANS SCRIPR MAPPING
INSTANT
CONSERV LR VECTOR LATITUDE 40 FRACAREA FIRST
rmp_oces_to_atmo_CONSERV_FRACAREA_T63_GR15.nc dst
#
###############################################################################
#
# Field 6 ocean northw. velocity [m/s]
#
OCVOCEAN OCVATMOS 48 86400  3 sstoce6  EXPORTED
254 220 192 96 oces atmo SEQ=1 LAG=2700
P  0  P  0
LOCTRANS SCRIPR MAPPING
INSTANT
CONSERV LR VECTOR LATITUDE 40 FRACAREA FIRST
rmp_oces_to_atmo_CONSERV_FRACAREA_T63_GR15.nc dst
#
###############################################################################
#
# Field 7 CO2 transfer coefficient
#
CO2TRAOC CO2TRAAT 99 86400  3 sstoce7  EXPORTED
254 220 192 96 oces atmo SEQ=1 LAG=2700
P  0  P  0
LOCTRANS SCRIPR MAPPING
INSTANT
CONSERV LR SCALAR LATITUDE 40 FRACAREA FIRST
rmp_oces_to_atmo_CONSERV_FRACAREA_T63_GR15.nc dst
#
###############################################################################
#
# Field 8 pCO2 in uppermost ocean layer [ppm CO2]
#
CO2OCEAN CO2ATMOS 99 86400  3 sstoce8  EXPORTED
254 220 192 96 oces atmo SEQ=1 LAG=2700
P  0  P  0
LOCTRANS SCRIPR MAPPING
INSTANT
CONSERV LR SCALAR LATITUDE 40 FRACAREA FIRST
rmp_oces_to_atmo_CONSERV_FRACAREA_T63_GR15.nc dst
#
###############################################################################
#
# Field 17 lwe_surface_downward_snow_flux_where_sea_ice [m/s;m/s]
#
FRIATMOS FRIOCEAN 55 86400  4 flxatm17   EXPORTED
192 96 254 220 atmo oces SEQ=1 LAG=450
P  0  P  0
LOCTRANS SCRIPR MAPPING CONSERV
INSTANT
CONSERV LR SCALAR LATITUDE 40 FRACAREA FIRST 
rmp_atmo_to_oces_CONSERV_FRACAREA_T63_GR15.nc src
GLOBAL
#
###############################################################################
#
# Field 18 water_flux_into_ocean [m/s;m/s]
#
FRWATMOS FRWOCEAN 29 86400  4 flxatm18   EXPORTED
192 96 254 220 atmo oces SEQ=1 LAG=450
P  0  P  0
LOCTRANS SCRIPR MAPPING CONSERV
INSTANT
CONSERV LR SCALAR LATITUDE 40 FRACAREA FIRST
rmp_atmo_to_oces_CONSERV_FRACAREA_T63_GR15.nc src
GLOBAL
#
###############################################################################
#
# Field 9 surface_downward_eastward_stress_where_open_sea [pa;pa]
#
TXWATMOU TXWOCEAS 50 86400  3 flxatm9   EXPORTED
192 96 254 220 atmo oces SEQ=1 LAG=450
P  0  P  0
LOCTRANS SCRIPR MAPPING
INSTANT
BICUBIC D SCALAR LATITUDE 40
rmp_atmo_to_oces_BICUBIC_D_T63_GR15.nc src
#
###############################################################################
#
# Field 11 surface_downward_northward_stress_where_open_sea [pa;pa]
#
TYWATMOU TYWOCEAS 51 86400  3 flxatm11   EXPORTED
192 96 254 220 atmo oces SEQ=1 LAG=450
P  0  P  0
LOCTRANS SCRIPR MAPPING
INSTANT
BICUBIC D SCALAR LATITUDE 40
rmp_atmo_to_oces_BICUBIC_D_T63_GR15.nc src
#
###############################################################################
#
# Field 13 surface_downward_eastward_stress_where_sea_ice [Pa;Pa]
#
TXIATMOU TXIOCEAS 52  86400  3 flxatm13   EXPORTED
192 96 254 220 atmo oces SEQ=1 LAG=450
P  0  P  0
LOCTRANS SCRIPR MAPPING
INSTANT
BICUBIC D SCALAR LATITUDE 40
rmp_atmo_to_oces_BICUBIC_D_T63_GR15.nc src
#
###############################################################################
#
# Field 15 surface_downward_northward_stress_where_sea_ice [pa;pa]
#
TYIATMOU TYIOCEAS 53 86400  3 flxatm15   EXPORTED
192 96 254 220 atmo oces SEQ=1 LAG=450
P  0  P  0
LOCTRANS SCRIPR MAPPING
INSTANT
BICUBIC D SCALAR LATITUDE 40
rmp_atmo_to_oces_BICUBIC_D_T63_GR15.nc src
#
###############################################################################
#
# Field 19 Residual heat flux (sea-ice topmelt heat flux)  [W/M^2;W/m^2]
#
RHIATMOS RHIOCEAN 43 86400  3 flxatm19   EXPORTED
192 96 254 220 atmo oces SEQ=1 LAG=450
P  0  P  0
LOCTRANS SCRIPR MAPPING
INSTANT
CONSERV LR SCALAR LATITUDE 40 FRACAREA FIRST
rmp_atmo_to_oces_CONSERV_FRACAREA_T63_GR15.nc src
#
###############################################################################
#
# Field 20 downward_heat_flux_in_sea_ice [W/M^2;W/m^2]
#
CHIATMOS CHIOCEAN 42 86400  3 flxatm20   EXPORTED
192 96 254 220 atmo oces SEQ=1 LAG=450
P  0  P  0
LOCTRANS SCRIPR MAPPING
INSTANT
CONSERV LR SCALAR LATITUDE 40 FRACAREA FIRST
rmp_atmo_to_oces_CONSERV_FRACAREA_T63_GR15.nc src
#
###############################################################################
#
# Field 21 surface_downward_heat_flux_in_air [W/m**2;W/m**2]
#
NHWATMOS NHWOCEAN 5 86400  3 flxatm21  EXPORTED
192 96 254 220 atmo oces SEQ=1 LAG=450
P  0  P  0
LOCTRANS SCRIPR MAPPING
INSTANT
CONSERV LR SCALAR LATITUDE 40 FRACAREA FIRST
rmp_atmo_to_oces_CONSERV_FRACAREA_T63_GR15.nc src
#
###############################################################################
#
# Field 22 surface_net_downward_shortwave_flux [W/m**2;W/m**2]
#
SHWATMOS SHWOCEAN 7  86400  3 flxatm22  EXPORTED
192 96 254 220 atmo oces SEQ=1 LAG=450
P  0  P  0
LOCTRANS SCRIPR MAPPING
INSTANT
CONSERV LR SCALAR LATITUDE 40 FRACAREA FIRST
rmp_atmo_to_oces_CONSERV_FRACAREA_T63_GR15.nc src
#
###############################################################################
#
# Field 23 wind_speed_at_10m                  [m /s ]
#
WSVATMOS WSVOCEAN 37  86400 3 flxatm23  EXPORTED
192 96 254 220 atmo oces SEQ=1 LAG=450
P  0  P  0
LOCTRANS SCRIPR MAPPING
INSTANT
CONSERV LR SCALAR LATITUDE 40 FRACAREA FIRST
rmp_atmo_to_oces_CONSERV_FRACAREA_T63_GR15.nc src
#
###############################################################################
#
# Field 24 atmospheric CO2 concentration [kg/kg] (only for diagnoses)
#
CO2CONAT CO2CONOC 99  86400 3 flxatm24  EXPORTED
192 96 254 220 atmo oces SEQ=1 LAG=450
P  0  P  0
LOCTRANS SCRIPR MAPPING
INSTANT
CONSERV LR SCALAR LATITUDE 40 FRACAREA FIRST
rmp_atmo_to_oces_CONSERV_FRACAREA_T63_GR15.nc src
#
###############################################################################
#
# Field 25 CO2 flux from ocean to atmosphere (calculated in the atmosphere)
#
CO2FLXAT CO2FLXOC 99  86400 4 flxatm25  EXPORTED
192 96 254 220 atml oces SEQ=1 LAG=450
P  0  P  0
LOCTRANS SCRIPR MAPPING CONSERV
INSTANT
CONSERV LR SCALAR LATITUDE 40 FRACAREA FIRST
rmp_atml_to_oces_CONSERV_FRACAREA_T63_GR15.nc dst
GLOBAL
#
###############################################################################
 $END
EOF

echo "* ----------------------------------------------------------------------"
echo "* Namelist of OASIS3-MCT : namcouple"
echo "* ----------------------------------------------------------------------"
sed '/^ *#/d' namcouple
echo "* ----------------------------------------------------------------------"
echo "*    end of namcouple"
echo "* ----------------------------------------------------------------------"
echo ""

#------------------------------------------------------------------------------
#-- Namelist ECHAM
#

# Adjust start date to avoid delays due to trigger initialisation
year_offset=$((nmonth/12 + (nmonth%12?1:0)))
year_offset=$((year_offset?year_offset:1))

pseudo_date_start=$(calc_date minus -Y$year_offset ${startdate}_)
pseudo_date_start=$(calc_date minus -s$ATM_TIME_STEP ${pseudo_date_start}_)
pseudo_date_start="$(format_date -f4 $pseudo_date_start)"
pseudo_date_start=${pseudo_date_start// /, }

cat -> namelist.echam << EOF

&runctl
    dt_start = $pseudo_date_start
    dt_stop = 2100, 12, 31, 23, 52, 30
    putrerun = 12, 'months', 'last', 0
    lfractional_mask = .false.
    lresume = .true.
    out_datapath = './'
    out_expname = '${EXP_ID}'
    rerun_filetype = 4
    delta_time = 450
    putdata = 1, 'months', 'last', 0
    nproma = 8
    lcouple = .true.
    getocean = 192.0, 'steps', 'exact', 0
    putocean = 192.0, 'steps', 'exact', -450
    lcouple_co2 = .true.
    default_output = .false.
/

&parctl
    nproca = 24
    nprocb = 24
/

&submodelctl
    lmethox = .true.
    lco2 = .true.
/

&mvstreamctl ! 'default spm glm g3bm'
    filetag = 'echam'
    source = 'sp', 'gl', 'g3b'
/
&mvstreamctl ! 'default co2m'
    filetag = 'co2'
    source = 'co2'
/
&mvstreamctl ! 'default surfm'
    filetag = 'surf'
    source = 'surf'
/
&mvstreamctl ! 'default tracerm'
    filetag = 'tracer'
    source = 'tracer'
/

&mvstreamctl ! 'default jsbachm'
    filetag = 'jsbach'
    source = 'jsbach'
/
&mvstreamctl ! 'default landm'
    filetag = 'land'
    source = 'land'
/
&mvstreamctl ! 'default vegm'
    filetag = 'veg'
    source = 'veg'
/
&mvstreamctl ! 'default yassom'
    filetag = 'yasso'
    source = 'yasso'
/

&set_stream ! 'default jsbach'
    stream = 'jsbach'
    lpost = 0
/
&set_stream ! 'default land'
    stream = 'land'
    lpost = 0
/
&set_stream ! 'default veg'
    stream = 'veg'
    lpost = 0
/
&set_stream ! 'default yasso'
    stream = 'yasso'
    lpost = 0
/
&radctl
    iaero = 8
    ighg = 1
    io3 = 4
    isolrad = 1
    ico2 = 1
    ich4 = 4
    in2o = 4
    icfc = 4
/
&co2ctl
    lco2_emis = .true.
    lco2_2perc = .true.
/
&submdiagctl
    vphysc_lpost = .false.
/
&mvstreamctl ! 'cmip6 echamday'
    filetag = 'echamday'
    source = 'sp', 'gl', 'g3b'
    target = '*day'
    interval = 1, 'days', 'last', 0
    variables = 'ahfl_na>ahfl=147', 'ahfs_na>ahfs=146', 'aprc_na>aprc=143',
                'aprl_na>aprl=142', 'aprs_na>aprs=144', 'aps', 'dew2', 'geosp',
                'q2m', 'rh2m', 'srads_na>srads=176', 'temp2',
                'tsurf_na>tsurf=169', 'u10', 'v10', 'wind10_na>wind10=171',
                'temp2:max>t2max=201', 'temp2:min>t2min=202'
/
&mvstreamctl ! 'cmip6 jsbachday'
    filetag = 'jsbachday'
    source = 'jsbach'
    target = '*day'
    interval = 1, 'days', 'last', 0
    variables = 'layer_moisture',
                'evapotranspiration_inst>evapotranspiration=44',
                'transpiration_inst>transpiration=76', 'cover_fract',
                'veg_ratio_max'
/
&mvstreamctl ! 'cmip6 landday'
    filetag = 'landday'
    source = 'land'
    target = '*day'
    interval = 1, 'days', 'last', 0
    variables = 'bare_soil_evaporation', 'snow_evaporation', 'lai',
                'snow_depth_canopy', 'snow_fract', 'soil_temperature',
                'surface_radiative_temp', 'par_acc', 'apar_acc'
/
&mvstreamctl ! 'cmip6 vegday'
    filetag = 'vegday'
    source = 'veg'
    target = '*day'
    interval = 1, 'days', 'last', 0
    variables = 'LCC_flux_box_C2atmos', 'boxC_construction_LCC',
                'boxC_construction_harvest', 'boxC_crop_harvest',
                'boxC_flux_2_construction_LCC',
                'boxC_flux_2_construction_harvest', 'boxC_flux_2_onSite_LCC',
                'boxC_flux_2_onSite_harvest', 'boxC_flux_2_paper_LCC',
                'boxC_flux_2_paper_harvest',
                'boxC_flux_construction_2_atmos_LCC',
                'boxC_flux_construction_2_atmos_harvest',
                'boxC_flux_onSite_2_atmos_LCC',
                'boxC_flux_onsite_2_atmos_harvest',
                'boxC_flux_paper_2_atmos_LCC',
                'boxC_flux_paper_2_atmos_harvest', 'boxC_green',
                'boxC_litter_green_bg', 'boxC_onSite_LCC',
                'boxC_onSite_harvest', 'boxC_paper_LCC', 'boxC_paper_harvest',
                'boxC_reserve', 'boxC_woods', 'box_Cflux_herbivory_2_atm',
                'box_Cflx_2_crop_harvest', 'box_Cflx_crop_harvest_2_atm',
                'box_Cpools_total', 'box_GPP_yDayMean', 'box_NPP_act_yDayMean',
                'box_fire_CO2_flux_2_atmos', 'box_soil_respiration',
                'burned_YC_acid_ag_wood', 'dist_nitrogen_2_atmos'
/

EOF

# Selectively add disturbance to overcome model failure
if [[ " ${ATMO_DISTURBED_YEARS} " == *" ${yr0} "* ]]; then
    print "disturbing atmosphere model in year ${yr0}"
    patch_namelist dynctl enstdif ${ATMO_DISTURBANCE} namelist.echam
fi

echo "* ----------------------------------------------------------------------"
echo "* Namelist of ECHAM: namelist.echam"
echo "* ----------------------------------------------------------------------"
cat namelist.echam
echo "* ----------------------------------------------------------------------"
echo "*    end of namelist.echam"
echo "* ----------------------------------------------------------------------"
echo ""

#
#-- Namelist JSBACH
#

cat -> namelist.jsbach <<EOF

&jsbach_ctl
    standalone = .false.
    ntiles = 11
    use_bethy = .true.
    use_phenology = .true.
    use_albedo = .true.
    with_yasso = .true.
    with_hd = .true.
    use_roughness_lai = .true.
    use_roughness_oro = .false.
    veg_at_1200 = .false.
    use_dynveg = .true.
    use_disturbance = .true.
    with_nitrogen = .true.
    lcc_forcing_type = 'transitions'
    lcc_scheme = 2
    test_cconservation = .true.
    test_nconservation = .true.
    file_ztype = 1
/

&soil_ctl
    nsoil = 5
/

&hydrology_ctl
    gethd = 192.0, 'steps', 'exact', 0
    puthd = 192.0, 'steps', 'exact', -450
    diag_water_budget = .true.
/
&cbalance_ctl
    read_ndepo = .true.
/
&disturbance_ctl
    fire_name = 'thonicke'
    fuel_name = 'thonicke'
    lburn_pasture = .true.
/
&input_ctl ! 'lightning_frq'
    var_name = 'lightning_frq'
    var_file_name = 'HRMC_COM_FR'
    file_name = 'lightning.nc'
    action_cycle = 'all'
/
&input_ctl ! 'population_density'
    var_name = 'population_density'
    var_file_name = 'popd'
/

EOF

echo "* ----------------------------------------------------------------------"
echo "* Namelist of JSBACH: namelist.jsbach"
echo "* ----------------------------------------------------------------------"
cat namelist.jsbach
echo "* ----------------------------------------------------------------------"
echo "*    end of namelist.jsbach"
echo "* ----------------------------------------------------------------------"
echo ""

#
#-- Namelist MPIOM
#

cat -> OCECTL << EOF
! Needs empty sections to guarantee order!

&proctl
/

&ocedim
    ie_g = 256
    je_g = 220
    ke = 40
    lbounds_exch_tp = .false.
/

&nprocs
    nprocx = 16
    nprocy = 24
/

&ocectl
    dt = 2700
    nmonts = 12
    ndays = 0
    nfixyearlen = -1
    lnonblock = .true.
    luse_sp_diag = .true.
    lundelayed_momentum_advection = .true.
    ! Base tuning
    aus = 0.
    dv0 = 0.2e-2
    av0 = 0.2e-2
    cwa = 0.75e-3
    cstabeps = 0.03
    dback = 1.05e-5
    cdvocon = 0.1
    ! Jerlov type IA
    jerlov_atten = 0.06
    jerlov_bluefrac = 0.41
    lsaoclose = .true.
    lzo_correct = .false.
    lakes = .false.
    caulapuv = 0.006
    ibolk = 250
    rleadclose = 0.5, 5.0, 4.0
    h0 = 0.3
    ! no hamocc feedback on absorbtion
    lfb_bgc_oce = .false.
/

&ocedzw
    cdzw = 12., 10., 10., 10., 10., 10., 13., 15., 20., 25., 30., 35., 40., 45.,
           50., 55., 60., 70., 80., 90., 100., 110., 120., 130., 140., 150.,
           170., 180., 190., 200., 220., 250., 270., 300., 350., 400., 450.,
           500., 500., 600.
/

&ioctl
    
    ! Use CFC fields (42, 45)
    iolist(100) = 99, 'rerun_${EXP_ID}_mpiom.nc', 'nc4', 1, 2, 3, 4, 5, 7, 9,
                  10, 13, 15, 35, 36, 39, 82, 99, 110, 111, 141, 501, 502, 503,
                  504
    iolist(99) = 2, '${EXP_ID}_mpiom_data_2d_mm.nc', 'nc4', 1, 11, 12, 13, 14,
                 15, 16, 17, 19, 20, 27, 35, 36, 66, 70, 78, 79, 104, 109, 123,
                 124, 125, 126, 127, 128, 134, 136, 138, 139, 140, 141, 149,
                 150, 151, 153, 165, 168, 169, 170, 181, 183, 215, 216, 217,
                 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229,
                 275, 276, 277, 278, 285, 286, 287, 288, 548, 549, 521, 1013,
                 1015, 1016, 1018, 1019, 1020, 1021, 1022, 1023, 1024, 1025,
                 1026, 1027, 1028, 1036, 1037, 1038, 1039, 1040, 1041, 1042,
                 1043, 1044, 1045, 1046, 1047, 1048
    iolist(98) = 2, '${EXP_ID}_mpiom_timeser_mm.nc', 'nc4', 508, 509, 520, 513,
                 514, 515, 516, 517, 518, 519, 592, 644, 645, 724, 725, 684,
                 685, 809, 613, 614, 615, 623, 624, 625, 633, 634, 635, 703,
                 704, 705, 612, 622, 632, 642, 652, 662, 672, 682, 692, 702,
                 712, 722, 732, 742, 752, 762, 770, 771, 772, 774, 775, 780,
                 781, 782, 784, 785, 790, 791, 792, 794, 795, 800, 801, 802,
                 804, 805, 810, 811, 812, 814, 815, 820, 821, 822, 823, 824,
                 825, 826, 827, 828, 829, 830, 831, 832, 833, 834, 835, 836,
                 837, 838, 839, 840, 841, 842, 843, 844, 845, 846, 847, 848,
                 849, 850, 851, 852, 853, 854, 855, 856, 857, 858, 859, 860,
                 861, 862, 863, 864, 865, 866, 867, 868, 869, 870, 871, 872,
                 873, 874, 875, 876, 877, 878, 879, 880, 881, 882, 883, 884,
                 885, 886, 887, 888, 889, 890, 891, 892, 893, 894, 895, 896,
                 897, 898, 899, 900, 901, 902, 903, 904, 905, 906, 907, 908,
                 909, 910, 911, 912, 913, 914, 915, 916, 917, 918, 919, 920,
                 921, 922, 923, 924, 925, 926, 927, 928, 929, 930, 931, 932,
                 933, 934, 935, 936, 937, 938, 939, 940, 941, 942, 943, 944,
                 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956,
                 957, 958, 959, 960, 961, 962, 963, 1001, 1050, 1051, 1052,
                 1053, 1054, 1055
    iolist(97) = 2, '${EXP_ID}_mpiom_data_moc_mm.nc', 'nc4', 93, 94, 95, 96, 97,
                 98, 100, 101, 102, 106, 107, 108, 1093, 1094, 1095, 1096, 1097,
                 1098, 1099, 1100, 1101, 1112, 1113, 1114
    iolist(96) = 3, '${EXP_ID}_mpiom_monitoring_ym.nc', 'nc4', 515, 516, 518,
                 519, 520, 622, 652, 672, 703, 800, 836, 837, 900, 901, 936,
                 950, 951, 952, 953, 955, 956, 958, 959, 961, 962, 1001
    iolist(95) = 2, '${EXP_ID}_mpiom_data_3d_mm.nc', 'nc4', 2, 3, 4, 5, 6, 7,
                 18, 21, 22, 23, 24, 25, 26, 30, 39, 54, 88, 90, 110, 111, 112,
                 135, 152, 197, 214, 279, 1029, 1030, 1031, 1032
    iolist(94) = 6402, '${EXP_ID}_mpiom_data_2d_mmin.nc', 'nc4', 276
    iolist(93) = 12802, '${EXP_ID}_mpiom_data_2d_mmax.nc', 'nc4', 276
    iolist(92) = 12801, '${EXP_ID}_mpiom_data_2d_dmax.nc', 'nc4', 276
    iolist(91) = 3, '${EXP_ID}_mpiom_data_2d_ym.nc', 'nc4', 277, 278
    iolist(80) = 107, '${EXP_ID}_mpiom_data_2d_3h.nc', 'nc4', 12
    iolist(90) = 1, '${EXP_ID}_mpiom_data_2d_dm.nc', 'nc4', 12, 13, 14, 15, 16,
                 104, 109, 140, 141, 276, 1016, 1022, 1023, 1024, 1025, 1026
    iolist(87) = 3, '${EXP_ID}_mpiom_data_3d_ym.nc', 'nc4', 25, 26, 28, 30, 88,
                 90, 111, 152, 282, 1029, 1030, 1031, 1032
/


&cplctl
/

EOF

echo "* ----------------------------------------------------------------------"
echo "* Namelist of MPIOM: OCECTL"
echo "* ----------------------------------------------------------------------"
cat OCECTL
echo "* ----------------------------------------------------------------------"
echo "*    end of OCECTL"
echo "* ----------------------------------------------------------------------"
echo ""



#
#-- Namelist for HAMOCC
#

cat > NAMELIST_BGC  << EOF

&bgcctl
    l_cpl_co2 = .true.
    lndep = .true.
    deltaorg = 4.40
    deltacalc = 463.18
    deltasil = 100.0
    sinkspeed_opal = 25.0
    lalkalinity_input = .true.
/

&ioctl
    iolist(100) = 99, 'rerun_${EXP_ID}_hamocc.nc', 'nc4', 7, 10, 11, 12, 13, 14,
                  15, 16, 17, 20, 21, 22, 23, 24, 27, 28, 29, 30, 31, 36, 37,
                  38, 41, 44, 45, 46, 47, 48, 49, 50, 51, 54, 55, 56, 57, 58,
                  59, 60, 61, 62, 63, 64, 203, 204, 205, 145
    iolist(99) = 4002, 'dust_deposition.nc', 'nc4', 34
    iolist(98) = 4002, 'nitrogen_deposition.nc', 'nc4', 35
    iolist(97) = 3, '${EXP_ID}_hamocc_data_3d_ym.nc', 'nc4', 36, 704, 731, 764,
                 787, 781, 739, 762, 783, 782, 747, 752, 701, 745, 20, 700, 765,
                 22, 754, 712, 724, 709, 766, 743, 719, 28, 707, 750, 735, 770,
                 789, 142, 727, 729, 166, 769, 741, 771, 772, 758, 773, 774,
                 775, 776, 777, 778, 779, 780, 784, 711, 785, 786, 788, 141,
                 100
    iolist(96) = 2, '${EXP_ID}_hamocc_data_3d_mm.nc', 'nc4', 69, 70, 71, 84, 85,
                 86, 100, 711, 141, 143, 292, 293, 294, 295, 296, 143, 510, 700,
                 701, 704, 707, 709, 712, 719, 724, 727, 729, 731, 735, 736,
                 739, 741, 743, 745, 147, 747, 750, 751, 752, 754, 758, 759,
                 762, 764, 765, 766, 236, 780, 778, 598, 599, 600, 601, 602,
                 603, 604, 605, 606, 607, 776, 785, 772, 782, 187, 188, 774,
                 788
    iolist(95) = 2, '${EXP_ID}_hamocc_data_2d_mm.nc', 'nc4', 61, 62, 64, 93,
                 755, 721, 742, 737, 733, 723, 716, 93, 738, 763, 107, 110, 111,
                 112, 114, 115, 116, 117, 120, 122, 123, 124, 127, 129, 131,
                 137, 157, 159, 714, 715, 749, 710, 746, 708, 730, 705, 791,
                 207, 210, 211, 214, 215, 703, 174, 201, 713, 206, 717, 718,
                 720, 722, 725, 726, 728, 732, 734, 740, 744, 748, 753, 756,
                 757, 760, 207, 761, 215, 790, 792, 608, 609, 417, 413, 419,
                 767, 796, 768, 797, 798, 799, 706, 94, 121, 144, 160, 174, 185,
                 186
    iolist(94) = 1, '${EXP_ID}_hamocc_data_2d_dm.nc', 'nc4', 767, 768
    iolist(93) = 101, '${EXP_ID}_hamocc_co2.nc', 'nc4', 301, 303, 304, 305, 306,
                 307, 308, 309, 310, 311, 312, 313, 314, 203, 204, 205
    iolist(92) = 3, '${EXP_ID}_hamocc_monitoring_ym.nc', 'nc4', 500, 501, 502,
                 503, 504, 505, 506, 507, 508, 509, 517, 519, 520, 523, 529,
                 530, 531, 532, 533, 534, 535
    iolist(91) = 3, '${EXP_ID}_hamocc_data_sedi_ym.nc', 'nc4', 38, 41, 44, 45,
                 46, 47, 48, 49, 50, 51, 54, 55, 56, 57, 58, 59, 60, 63, 94, 95,
                 96, 97, 167, 168, 280, 281, 282, 283, 284, 285, 286, 287
    iolist(90) = 3, '${EXP_ID}_hamocc_data_2d_ym.nc', 'nc4', 718
    iolist(89) = 4002, 'alkalinity_input.nc', 'nc4', 411
/

EOF

echo "* ----------------------------------------------------------------------"
echo "* Namelist of HAMOCC: NAMELIST_BGC"
echo "* ----------------------------------------------------------------------"
cat NAMELIST_BGC
echo "* ----------------------------------------------------------------------"
echo "*    end of NAMELIST_BGC"
echo "* ----------------------------------------------------------------------"
echo ""

print "preparing configuration files finished"

#------------------------------------------------------------------------------
#
#     5. LAUNCHING THE MODEL
#
#------------------------------------------------------------------------------

print "listing files in working directory before model execution"
echo --
ls -altr
echo --

ulimit -c 0   # size of core dumps, in number of 512-byte blocks

#
###############################################################################

export ECHAM6_THREADS=1
export MPIOM_THREADS=1

# Submit concurrent job scripts

for job in 
do
    print "starting  job for $jobid run"
    (cd $jobdir && 
     sbatch -t $(squeue -o '%L' -P -h -j $jobid) ${EXP_ID}.${job} $jobid run)
done

print "execution of model started"

status=0
/home/m/m300966/CDRSynTra-mpiesm-1.2.01p7/util/mpiexec-srun --native '-l --kill-on-bad-exit=0 --cpu_bind=cores --distribution=plane=64 --propagate=STACK,CORE' -n 384 $BIN_DIR/$OCEAN_EXE : -n 576 $BIN_DIR/$ATMO_EXE || status=$?

print "execution of model finished"

print "listing files in working directory after model execution"
echo --
ls -altr
echo --

case $status in
    0)  ;;
    127)
        print "Hey: ignoring return code $status (valid ECHAM6 return code)" ;;
    *)  print "Sorry: model returned error code $status"
        exit $status ;;
esac

if grep ERROR ${job}_${jobid}.log > /dev/null 2>&1; then
    print "Sorry: an error occurred during model integration"
    exit 1
fi

#
# Save restart files
#

print "storing of restart data started"

: > status

function tar_bg {
    typeset restdst=$(basename $1)
    typeset restdir=$(dirname $1)
    shift
    typeset restsrc="$@"
    mkdir $restdir
    (
        trap 'echo $restsrc:$? >> status' ERR
        tar -cf $restdir/$restdst $restsrc
    ) &
}
function cp_bg {
    typeset restsrc=$1
    typeset restdst=$(basename $2)
    typeset restdir=$(dirname $2)
    mkdir $restdir
    (
        trap 'echo $restsrc:$? >> status' ERR
        cp $restsrc $restdir/$restdst
    ) &
}

#  Save output of OASIS3

tar_bg $COUPLER_RESTART_DIR/sstocean_${EXP_ID}_${enddate}.tar sstoce[1-9]*
tar_bg $COUPLER_RESTART_DIR/flxatmos_${EXP_ID}_${enddate}.tar flxatm[1-9]*

#  Save output of ECHAM

for substream in echam co2 tracer accw
do
    cp_bg restart_${EXP_ID}_${endtime}_${substream}.nc \
          $ATMO_RESTART_DIR/restart_${EXP_ID}_${substream}_${enddate}.nc
done # substreams

#  Save output of JSBACH

for substream in jsbach veg surf yasso nitro
do
    cp_bg restart_${EXP_ID}_${endtime}_${substream}.nc \
          $LAND_RESTART_DIR/restart_${EXP_ID}_${substream}_${enddate}.nc
done # substreams

cp_bg hdrestart.nc $LAND_RESTART_DIR/restart_${EXP_ID}_hd_${enddate}.nc

#  Save output of MPIOM

cp_bg rerun_${EXP_ID}_mpiom.nc \
      $OCEAN_RESTART_DIR/rerun_${EXP_ID}_mpiom_${enddate}.nc

# Save output of HAMOCC

cp_bg rerun_${EXP_ID}_hamocc.nc \
      $OBGC_RESTART_DIR/rerun_${EXP_ID}_hamocc_${enddate}.nc



# Wait for storing to complete and evaluate error messages before continuing

wait
[[ -s status ]] &&
    die "storing of restart data failed ("$(<status)")"

print "storing of restart data finished"

#
#     8. SUBMISSION OF THE NEXT JOBS
#

print 'saving experiment status'
cd ${jobdir}

# Number of the next job
(( nextjob = ${jobnum} + 1 ))

# Edit .date and .log file

space_error="no"

echo "${nextdate} ${nextjob}" > ${EXP_ID}.date.new || { 
    space_error="yes"; print "Sorry: cannot create ${EXP_ID}.date"; 
}
cp ${EXP_ID}.log ${EXP_ID}.log.new || { 
    space_error="yes"; print "Sorry: cannot save ${EXP_ID}.log"; 
}
echo "$(date +"${datefmt}") :  ${jobnum} ${nextdate} ${jobid}  - done" >> ${EXP_ID}.log.new || {
    space_error="yes"; print "Sorry: cannot append to ${EXP_ID}.log"; 
}

if [ "${space_error}" = "no" ]; then
    mv ${EXP_ID}.date.new ${EXP_ID}.date
    mv ${EXP_ID}.log.new ${EXP_ID}.log
else
    print "Sorry: error saving experiment status, check disk space and quota for"
    print "Sorry: $jobdir"
    exit 1
fi

# Update run dates and submit job scripts
# $Id$

# Check for specialized scripts
function get_script {
    typeset job=$1
    typeset startdate=$2
    typeset script=esm-ssp585-ocn-alk_EXP3.$job
    typeset special_script dates firstdate lastdate
    for special_script in \
        esm-ssp585-ocn-alk_EXP3.$job.$startdate \
        esm-ssp585-ocn-alk_EXP3.$job.*[0-9][0-9][0-9][0-9][0-9]-*[0-9][0-9][0-9][0-9][0-9]
    do
        if [ -f $special_script ]
        then
            dates=${special_script#esm-ssp585-ocn-alk_EXP3.$job.}
            firstdate=${dates%-*}
            lastdate=${dates#*-}
            if [[ $(later_date -- $startdate $firstdate) == $startdate &&
                  $(later_date -- $startdate $lastdate) == $lastdate ]]
            then
                print "Hey: using specialized $job script '$special_script'" >&2
                script=$special_script
                break
            fi
        fi
    done
    echo $script
}

print "starting store job for $startdate"
sbatch $(get_script store $startdate) $startdate

# Check whether final date is reached; otherwise submit follow-up job
if [[ $(later_date -- $nextdate $findate) == $nextdate ]]
then
    print "experiment finished"
    logstamp=$(date +"$datefmt")
    echo "$logstamp :  Experiment over" >> esm-ssp585-ocn-alk_EXP3.log
else
    print "starting run job for $nextdate"
    sbatch $(get_script run $nextdate)
fi

#------------------------------------------------------------------------------
#   EPILOGUE
#------------------------------------------------------------------------------

print "model run finished for $startdate-$enddate"