#! /bin/ksh # # esm-ssp585-ocn-alk_EXP2.store # # Generated by Make Experiments! (mkexp) 1.1.5rc2 # # $Id: DEFAULT.store.tmpl 3 2021-02-08 13:12:54Z m221078 $ # # $Id: DEFAULT.config 1 2021-02-03 20:53:47Z m221078 $ # $Id$ # $Id$ # $Id: CDRSYNTRA_OUTPUT.config 9899 2019-07-01 09:57:06Z m221078 $ # $Id: levante.config $ # # # Setup for levante (SLURM) # # $Id: levante.tmpl $ # #SBATCH --job-name=esm-ssp585-ocn-alk_EXP2_store #SBATCH --partition=shared #SBATCH --ntasks=4 #SBATCH --ntasks-per-core=1 #SBATCH --mem-per-cpu=2G #SBATCH --time=00:10:00 #SBATCH --output=%x_%j.log #SBATCH --mail-type=FAIL,ARRAY_TASKS #SBATCH --account=bm1241 #SBATCH --propagate=STACK,CORE ulimit -s 390625 # * 1024 B = 400 MB # OpenMPI export HDF5_USE_FILE_LOCKING=FALSE export MALLOC_TRIM_THRESHOLD_="-1" export KMP_AFFINITY="granularity=fine,scatter" export KMP_LIBRARY="turnaround" export MKL_DEBUG_CPU_TYPE=5 export MKL_ENABLE_INSTRUCTIONS=AVX2 export OMPI_MCA_btl=self export OMPI_MCA_coll="^ml" export OMPI_MCA_coll_hcoll_enable="1" export OMPI_MCA_io="romio321" export OMPI_MCA_osc="pt2pt" export OMPI_MCA_pml="ucx" export HCOLL_ENABLE_MCAST_ALL="1" export HCOLL_MAIN_IB=mlx5_0:1 export UCX_IB_ADDR_TYPE=ib_global export UCX_NET_DEVICES=mlx5_0:1 export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self export UCX_UNIFIED_MODE=y export SLURM_CPU_FREQ_REQ=HighM1 # Intel OpenMP export KMP_STACKSIZE=128M # OpenMP export OMP_NUM_THREADS=1 # Workaround for SLURM bug in chained jobs SLURM_JOB_NAME=esm-ssp585-ocn-alk_EXP2_store sbatch () { unset SLURM_MEM_PER_CPU SLURM_NTASKS_PER_NODE command sbatch "$@" } # DEBUG_LEVEL=${DEBUG_LEVEL:-0} # Support log style output export LANG=C print () { command print "$(date +'%F %T'):" "$@"; } print_re='^[0-9]+-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}' warn () { print 'Hey:' "$@" >&2; } die () { print 'Oops:' "$@" >&2; return 1; } # Bail out on error trap 'print Error at line $LINENO >&2' ERR set -eu # Print command info [[ $DEBUG_LEVEL -ge 2 ]] && set -x # Format log file lines as warnings show () { while read show_line_ do warn "$show_line_" done < "$1" } ############################################################################### # # SETUP OF EXPERIMENT # ############################################################################### EXP_ID=esm-ssp585-ocn-alk_EXP2 atmmod=echam6 srfmod=jsbach ocemod=mpiom bgcmod=hamocc coupler=oasis3mct ############################################################################### # # USER INTERFACE # ############################################################################### #------------------------------------------------------------------------------ # 1.3 TIME CONTROL #------------------------------------------------------------------------------ # #-- calendar type: Available calendar options: # 0 : No leap year (365 days per year) # 1 : Gregorian (365/366 days per year) # n : Equal months of "n" days (30 for 30 day months) CALENDAR=1 # #-- initial and final date of the experiment # Format: YearMMDD[_hh[mm[ss]]], Year-MM-DD[_hh[:mm[:ss]]] or # Year-MM-DD[Thh[:mm[:ss]]] # Note: The experiment will not stop within a run/chunk even if the # final date is reached. inidate=20200101 findate=21001231 # #-- duration of a run/chunk # Specify the length of each run in one of the below units. integer nyear nmonth nday nhour nminute nsecond nyear=0 # number of years per run nmonth=12 # number of months per run nday=0 # number of days per run nhour=0 # number of hours per run nminute=0 # number of minutes per run nsecond=0 # number of seconds per run nstep_atm=0 # number of atmosphere model time steps per run nstep_oce=0 # number of ocean model time steps per run #------------------------------------------------------------------------------ # 1.6 FILE SYSTEMS #------------------------------------------------------------------------------ # #-- SCRIPT_DIR: # Permanent file system for the SCRIPTS/BINARIES on the COMPUTING HOST # (only needs to be specified if the tasks are NOT generated on the # computing host) SCRIPT_DIR=/home/m/m300966/CDRSynTra-mpiesm-1.2.01p7/experiments/esm-ssp585-ocn-alk_EXP2/scripts # #-- DATA_DIR, RESTART_DIR, LOG_DIR: # Directories of the SHORT TERM data server. # Model INPUT and OUTPUT will be read from/written to # this file system of the computing host # # - The parent-directory needs to exist before submission of the job DATA_DIR=/work/bm1241/m300966/CDRSynTra-mpiesm-1.2.01p7/experiments/esm-ssp585-ocn-alk_EXP2/outdata RESTART_DIR=/work/bm1241/m300966/CDRSynTra-mpiesm-1.2.01p7/experiments/esm-ssp585-ocn-alk_EXP2/restart LOG_DIR=/work/bm1241/m300966/CDRSynTra-mpiesm-1.2.01p7/experiments/esm-ssp585-ocn-alk_EXP2/log # #-- WORK_DIR: Root directory for the temporary working directory # (for production runs use $TMPDIR on NEC) # WORK_DIR=/scratch/m/m300966/CDRSynTra-mpiesm-1.2.01p7/experiments/esm-ssp585-ocn-alk_EXP2/work # #-- Path to the IMDI function directory # export PATH=/home/m/m300966/CDRSynTra-mpiesm-1.2.01p7/util/running/functions:$PATH #------------------------------------------------------------------------------ # UNIX COMMANDS #------------------------------------------------------------------------------ mkdir () { command mkdir -pv "$@"; } cp () { command cp -pv "$@"; } rm () { command rm -vf "$@"; } function calc_date { typeset command="$1" shift command calc_date $command -c$CALENDAR "$@" } ############################################################################### # # END OF THE USER INTERFACE # ############################################################################### #------------------------------------------------------------------------------ # Read dates for this run from command line #------------------------------------------------------------------------------ [[ -z "$1" ]] && die 'invalid number of parameters; need startdate as YYYYMMDD' startdate=$1 # Compute dependent date variables nextdate=$(calc_date plus -M$nmonth -- ${startdate}_) nextdate=$(calc_date plus -D$nday -- ${nextdate}_) enddate=$(calc_date minus -D1 -- ${nextdate}) prevstartdate=$(calc_date minus -M$nmonth -- ${startdate}_) prevstartdate=$(calc_date minus -D$nday -- ${prevstartdate}_) prevdate=$(calc_date minus -D1 -- ${startdate}_) #------------------------------------------------------------------------------ # Directory definitions #------------------------------------------------------------------------------ run_subdir=run_$startdate-$enddate mkdir ${DATA_DIR} mkdir ${LOG_DIR} ############################################################################### # # Start raw output saving ... # ############################################################################### print "storing of model output started for $startdate-$enddate" print "from $WORK_DIR/$run_subdir" print "to $DATA_DIR" print "and $LOG_DIR" # Show progress sh -c 'scontrol update JobId=$SLURM_JOB_ID Comment="$*"' scomment $startdate # Move previous log file to log directory for log_path in $(grep -El "$print_re: storing of model output finished for $prevstartdate" $SCRIPT_DIR/${EXP_ID}_store*_[0-9]*.log || :) do log_file=${log_path##*/} # The * after store might contain underscores or numbers, # so make sure we only take the latter part as previous job's id log_prefix=${log_file%_[0-9]*.log} log_suffix=${log_file#$log_prefix} mv $log_path $LOG_DIR/${EXP_ID}_store_${prevstartdate}$log_suffix done # Prepare saving of output cd $WORK_DIR/$run_subdir function cp_echam { typeset src_file=$1 typeset src_base=$(basename $src_file) typeset dst_file=$2 typeset dst_code_file=$3 ( trap 'echo $src_base:$? >> status' ERR if [[ -r $src_file ]] then cp $src_file $dst_file.grb if ! cmp $src_file.codes $dst_code_file.codes > /dev/null 2>&1 then trap 'echo $src_base.codes:$? >> warnings' ERR cp $src_file.codes $dst_code_file.codes fi else cp $src_file.nc $dst_file.nc fi ) & } ############################################################################### # # Save output of OASIS3 ( do not save input ! ) # ############################################################################### print "storing of ${coupler} data started" : > status : > warnings outdir=${DATA_DIR}/${coupler} mkdir ${outdir} #-- output files of OASIS if files=$(ls ????????_out.????-??-??T??:??:??.nc 2>/dev/null) then ( trap 'echo outfiles:$? >> warnings' ERR cp ${files} ${outdir} ) & fi #-- log files of OASIS if files=$(ls cplout* Oasis.prt* 2>/dev/null) then ( trap 'echo cplout:$? >> warnings' ERR cp ${files} ${LOG_DIR} ) & fi wait show warnings [[ -s status ]] && die "storing of ${coupler} data failed ("$( status : > warnings outdir=${DATA_DIR}/${atmmod} mkdir ${outdir} #-- raw output and restart files substreams="echam co2 tracer accw echamday" [[ $startdate == $inidate ]] && substreams="$substreams " for substream in ${substreams} do # Loop over all output timesteps date=${startdate} while [[ $(later_date -- ${date} ${enddate}) == ${enddate} ]] do read year month day ignore << --- $(format_date -f4 -- ${date}) --- cp_echam ${EXP_ID}_${year}${month}.${day}_${substream} \ $outdir/${EXP_ID}_${atmmod}_${substream}_${year}${month} \ $LOG_DIR/${EXP_ID}_${atmmod}_${substream} # TODO: increment currently fixed to 1 month date=$(calc_date plus -M1 -- ${date}) done done # substreams #-- std output if [[ -r atmout ]] then ( trap 'echo atmout:$? >> warnings' ERR cp atmout ${LOG_DIR}/${EXP_ID}_${atmmod}_atmout_${startdate}_${enddate} ) & fi wait show warnings [[ -s status ]] && die "storing of ${atmmod} data failed ("$( status : > warnings outdir=${DATA_DIR}/${srfmod} mkdir ${outdir} #-- raw output and restart files substreams="jsbach veg surf yasso nitro land jsbachday landday vegday" [[ $startdate == $inidate ]] && substreams="$substreams " for substream in ${substreams} do # Loop over all output timesteps date=${startdate} while [[ $(later_date -- ${date} ${enddate}) = ${enddate} ]] do read year month day ignore << --- $(format_date -f4 -- ${date}) --- cp_echam ${EXP_ID}_${year}${month}.${day}_${substream} \ $outdir/${EXP_ID}_${srfmod}_${substream}_${year}${month} \ $LOG_DIR/${EXP_ID}_${srfmod}_${substream} # TODO: increment currently fixed to 1 month date=$(calc_date plus -M1 -- ${date}) done done # substreams #-- hydrology log files for file in $(ls hd_outflow_??.log 2> /dev/null) do ( trap 'echo hd:log:$? >> warnings' ERR cp $file ${LOG_DIR}/${EXP_ID}_${srfmod}_hd_outflow_${enddate}.log ) & done wait show warnings [[ -s status ]] && die "storing of ${srfmod} data failed ("$( status : > warnings suff=.nc outdir=${DATA_DIR}/${ocemod} mkdir ${outdir} #-- raw diagnostic output #-- for first run, also fixed fields substreams="data_2d_3h data_2d_dm data_2d_dmax data_2d_mm data_2d_ym data_2d_mmin data_2d_mmax data_3d_mm data_3d_ym data_moc_mm timeser_mm monitoring_ym" [[ $startdate == $inidate ]] && substreams="$substreams map fx" for substream in ${substreams} do outfile=${EXP_ID}_mpiom_${substream}${suff} outfile_save=${EXP_ID}_mpiom_${substream}_${startdate}_${enddate}${suff} ( trap 'echo $substream:$? >> status' ERR cp ${outfile} ${outdir}/${outfile_save} ) & done #-- std output if [[ -r oceout ]] then ( trap 'echo oceout:$? >> warnings' ERR cp oceout ${LOG_DIR}/${EXP_ID}_mpiom_oceout_${startdate}_${enddate} ) & fi #-- output parameter table if [[ -r mpiom.partab ]] && ! diff mpiom.partab ${LOG_DIR}/mpiom.partab > /dev/null 2>&1 then ( trap 'echo partab:$? >> warnings' ERR cp mpiom.partab ${LOG_DIR}/mpiom.partab ) & fi wait show warnings [[ -s status ]] && die "storing of ${ocemod} data failed ("$( status : > warnings suff=.nc outdir=${DATA_DIR}/${bgcmod} mkdir ${outdir} #-- raw diagnostic output substreams="data_3d_ym data_3d_mm data_2d_ym data_2d_mm data_2d_dm data_sedi_ym co2 monitoring_ym" [[ $startdate == $inidate ]] && substreams="$substreams " for substream in ${substreams} do outfile=${EXP_ID}_hamocc_${substream}${suff} outfile_save=${EXP_ID}_hamocc_${substream}_${startdate}_${enddate}${suff} ( trap 'echo $substream:$? >> status' ERR cp ${outfile} ${outdir}/${outfile_save} ) & done #-- std output if [[ -r bgcout ]] then ( trap 'echo bgcout:$? >> warnings' ERR cp bgcout ${LOG_DIR}/${EXP_ID}_hamocc_bgcout_${startdate}_${enddate} ) & fi #-- output parameter table if [[ -r hamocc.partab ]] && ! diff hamocc.partab ${LOG_DIR}/hamocc.partab > /dev/null 2>&1 then ( trap 'echo partab:$? >> warnings' ERR cp hamocc.partab ${LOG_DIR}/hamocc.partab ) & fi wait show warnings [[ -s status ]] && die "storing of ${bgcmod} data failed ("$(