[INFO] JOBID=18944395

job_name_ptrn='/gpfs/scratch/bsc32/bsc32627/a3tb/LOG_a3tb/a3tb_19930101_fc01_1_SIM'
+ job_name_ptrn=/gpfs/scratch/bsc32/bsc32627/a3tb/LOG_a3tb/a3tb_19930101_fc01_1_SIM
echo $(date +%s) > ${job_name_ptrn}_STAT
date +%s
++ date +%s
+ echo 1638380885

###################
# Autosubmit job
###################

#!/usr/bin/env bash

set -xuve
+ set -xuve

cd /gpfs/scratch/bsc32/bsc32627/a3tb
+ cd /gpfs/scratch/bsc32/bsc32627/a3tb

# librunscript defines some helper functions
source ./librunscript.sh
+ source ./librunscript.sh
# librunscript.sh is a library of shell script functions that can be used in
# EC-Earth run scripts.
#
# Usage: source ./librunscript.sh

# Function info writes information to standard out.
#
# Usage: info MESSAGE ...
#
function info()
{
    echo "*II* $@"
}

# Function error writes information to standard out and exits the script with
# error code 1.
#
# Usage: error MESSAGE ...
#
function error()
{
    echo "*EE* $@"
    exit 1
}

# Function has_config checks it's arguments for matches in the $config variable
# and returns true (0) or false (1) accordingly. Optionally, the first argument
# can be either "all" (all arguments must match) or "any" (at least one
# argument must match). If the first argument is neither "all" nor "any", the
# function behaves like "all" was given as the first argument.
#
# Usage: has_config [all|any] ARGS ...
#
# Syntax rules:
#
# The $config variable takes a list of names (typically software components),
# separated by white spaces:
#
# config="foo bar baz"  # Specifies three components: 'foo', 'bar', and 'baz'
#
# It is possible to add comma-separated lists of options to components. The
# list is separated from the component by a colon:
#
# config="foo bar:qux,fred baz:plugh"  # Adds the options 'qux' and 'fred' to
#                                      # component 'bar' as well as option
#                                      # 'plugh' to  component 'baz'
#
# When using the has_config function to check the $config variable, it is
# important to list every component-option pair separately. To check for both
# the 'qux' and 'fred' options of component 'bar' in the above example, use:
#
# has_config bar:qux bar:fred && echo "Got it!"
#
function has_config()
{
    # If called without arguments, return false
    (( $# )) || return 1

    # If $config unset or empty, return false
    [[ -z "${config:-}" ]] && return 1

    local __c
    local __m

    # If first argument is "any" then only one of the arguments needs to match
    # to return true. Return false otherwise
    if [ "$1" == "any" ]
    then
        shift
        for __c in "$@"
        do
            for __m in $config
            do
                [[ "$__m" =~ "${__c%:*}" ]] && [[ "$__m" =~ "${__c#*:}" ]] && return 0
            done
        done
        return 1
    fi

    # If first argument is "all", or neither "any" nor "all", all arguments
    # must match to return true. Return false otherwise.
    [[ "$1" == "all" ]] && shift

    local __f
    for __c in "$@"
    do
        __f=0
        for __m in $config
        do
            [[ "$__m" =~ "${__c%:*}" ]] && [[ "$__m" =~ "${__c#*:}" ]] && __f=1
        done
        (( __f )) || return 1
    done
    return 0
}

# Function leap days calculates the number of leap days (29th of Februrary) in
# a time intervall between two dates.
#
# Usage leap_days START_DATE END_DATE
function leap_days()
{
    local ld=0
    local frstYYYY=$(date -ud "$1" +%Y)
    local lastYYYY=$(date -ud "$2" +%Y)

    set +e

    # Check first year for leap day between start and end date
    $(date -ud "${frstYYYY}-02-29" > /dev/null 2>&1) \
    && (( $(date -ud "$1" +%s) < $(date -ud "${frstYYYY}-03-01" +%s) )) \
    && (( $(date -ud "$2" +%s) > $(date -ud "${frstYYYY}-02-28" +%s) )) \
    && (( ld++ ))

    # Check intermediate years for leap day
    for (( y=(( ${frstYYYY}+1 )); y<=(( ${lastYYYY}-1 )); y++ ))
    do
        $(date -ud "$y-02-29" > /dev/null 2>&1) && (( ld++ ))
    done

    # Check last year (if different from first year) for leap day between start
    # and end date
    (( $lastYYYY > $frstYYYY )) \
    && $(date -ud "${lastYYYY}-02-29" > /dev/null 2>&1) \
    && (( $(date -ud "$1" +%s) < $(date -ud "${lastYYYY}-03-01" +%s) )) \
    && (( $(date -ud "$2" +%s) > $(date -ud "${lastYYYY}-02-28" +%s) )) \
    && (( ld++ ))

    set -e

    echo "$ld"
}

# Helper function for absolute_date_noleap().
#
# Recursively 'fixes' a relative date into an absolute date referenced to
# the "noleap" calendar instead of Julian/Gregorian.
#
# Usage: __noleap_fixer base_date modifier
#

function __noleap_fixer ()
{
    local base_date="$1"
    local modifier="$2"
    # Evaluate the base and the modified date.
    local t0=$(date -u -d "${base_date}" +'%Y%m%d %H')
    local t1=$(date -u -d "${base_date} ${modifier}" +'%Y%m%d %H')
    local nleap
    local op
    # Compute the number of leap days between the base date and the modified
    # date.
    if [[ $(date -u -d "${t0}" +%Y%j) -lt $(date -u -d "${t1}" +%Y%j) ]]; then
        nleap=$(leap_days "${t0}" "${t1}")
    else
        nleap=$(leap_days "${t1}" "${t0}")
    fi
    if [[ ${nleap} -eq 0 ]]; then
        # Return the modified date if there are no leap days to account for.
        echo "${t1}"
    else
        # Obtain the modifier symbol (either + or -), this is used to determine
        # the direction to apply corrections in.
        if echo "${modifier}" | grep '+' 2>&1 > /dev/null; then
            op='+'
        else
            op='-'
        fi
        if [[ $(date -ud "${t1}" +%m%d) == 0229 ]]; then
            # If the modified day is the leap day we shift it by one day in the
            # appropriate direction and subtract 1 from the number of leap days
            # we need to account for.
            t1=$(date -ud "${t1} ${op} 1 day" +'%Y%m%d %H')
            nleap=$(( nleap - 1 ))
        fi
        if [[ ${nleap}  -eq 0 ]]; then
            # After correction there maybe no leap days left to account for, if
            # so we just return the corrected modified date.
            echo "${t1}"
        else
            # Otherwise we recurse, to apply our modification safely accounting
            # for leap days.
            __noleap_fixer "${t1}" "${op} ${nleap} days"
        fi
    fi
}

# Convert a relative date to an absolute date assuming the "noleap" calendar.
#
# A relative date is a base date with some modifier, such as
# "2012-01-01 + 2 months".
#
# Usage: absolute_date_noleap date_specifier.
function absolute_date_noleap ()
{
    local input_date="$1"
    # Split the input into a date part and a modifier part.
    local regex='([a-z|A-Z|0-9|-|+|,|:|\s]*)\s*([+|-]\s*[0-9]+\s*(second|minute|hour|day)s{0,1}).*'
    local base_date=$(echo "$1" | sed -r "s/${regex}/\1^\2/" | cut -d "^" -f 1)
    local modifier=$(echo "$1" | sed -r "s/${regex}/\1^\2/" | cut -d "^" -f 2)
    if [ -z "${modifier}" ] || [ "${modifier}" = "${input_date}" ] || \
            ! echo "${modifier}" | egrep -i -e 'second|minute|hour|day' 2>&1 >/dev/null; then
        # Either there was no modifier or it performs calendar independent
        # modifications, and thus we can safely use the normal `date` tool.
        date -uRd "${input_date}"
    else
        # Special care is needed to make sure the modifier is applied within
        # the noleap calendar.
        date -uRd "$(__noleap_fixer "${base_date}" "${modifier}")"
    fi
}

# Functions to compute computationa performance according to CPMIP metrics
#
# cpmip_sypd: Computes Simulated Years Per Day (SYPD)
#             Needs two arguments:
#             $1 - length of leg in seconds (use $leg_length_sec)
#             $2 - run time of leg in seconds (use $(( t2 - t1 )))
function cpmip_sypd ()
{
    bc <<< "scale=2 ; $1 / ( $2 * 365 )"
}

# cpmip_chpsy: Computes Core Hours Per Simulated Years (CHPSY)
#              Needs three arguments:
#              $1 - length of leg in seconds (use $leg_length_sec)
#              $2 - run time of leg in seconds (use $(( t2 - t1 )))
#              $3 - overall number of cores
function cpmip_chpsy ()
{
    bc <<< "scale=0 ; 365 * 24 * $2 * $3 / $1"
}

# =============================================================================
# *** BEGIN User configuration
# =============================================================================

# -----------------------------------------------------------------------------
# *** General configuration
# -----------------------------------------------------------------------------
# Component configuration (for syntax of the $config variable, see librunscript.sh)
#
# Currently maintained:
#     config="ifs amip oasis"                            # "GCM forced-SST" : IFS + AMIP
#     config="ifs amip oasis lpjg:fdbck"                 # "Veg"            : forced-GCM + LPJ-Guess
#     config="ifs amip oasis tm5:chem,o3fb,ch4fb,aerfb"  # "AerChem"        : forced-GCM + TM5
#
#     config="ifs nemo lim3 rnfmapper xios:detached oasis"                                 # "GCM"     : IFS+NEMO
#     config="ifs nemo lim3 rnfmapper xios:detached oasis lpjg:fdbck"                      # "Veg"     : GCM+LPJ-Guess
#     config="ifs nemo lim3 rnfmapper xios:detached oasis pisces lpjg:fdbck tm5:co2,co2fb" # "C-cycle" : GCM+LPJG+TM5
#     config="ifs nemo lim3 rnfmapper xios:detached oasis tm5:chem,o3fb,ch4fb,aerfb"       # "AerChem" : GCM+TM5
#
#     config="ifs nemo pisces lim3 rnfmapper xios:detached oasis"     # "GCM"     : IFS+NEMO+PISCES
#

#config="ifs nemo lim3 rnfmapper xios:detached oasis lpjg:fdbck tm5:co2"

# get config from autosubmit variables

#ifs
ifs="ifs"
+ ifs=ifs
TEMPLATE_NAME=ecearth3
+ TEMPLATE_NAME=ecearth3
[[ "$TEMPLATE_NAME" = ifs3* ]] && amip="amip" || amip=""
+ [[ ecearth3 = ifs3* ]]
+ amip=
[ "FALSE" = TRUE ] && atmnudg="atmnudg" || atmnudg=""
+ '[' FALSE = TRUE ']'
+ atmnudg=
[ "FALSE" = TRUE ] && sppt="sppt" || sppt=""
+ '[' FALSE = TRUE ']'
+ sppt=

#nemo
nemo=""; pisces=""; lim3=""; rnfmapper=""; xios=""
+ nemo=
+ pisces=
+ lim3=
+ rnfmapper=
+ xios=
[[ "$TEMPLATE_NAME" = nemo3* ]] && error "ece-esm.sh runscript does not support nemo only!"
+ [[ ecearth3 = nemo3* ]]
if [[ "$TEMPLATE_NAME" = ecearth3* ]];
then
  [[ "a2kv" = *[!\ ]* ]] && start_nemo_from_restart=":start_from_restart" || start_nemo_from_restart=""
  [[ "" = *[!\ ]* ]] && start_pisces_from_restart=":start_from_restart" || start_pisces_from_restart=""
  [ "FALSE" = TRUE ] && ocenudg=":ocenudg" || ocenudg=""
  [ "FALSE" = TRUE ] && surfresto=":surfresto" || surfresto=""
  [ "TRUE" = TRUE ] && elpin=":elpin" || elpin=""
  nemo="nemo"${start_nemo_from_restart}${ocenudg}${surfresto}${elpin}
  [ "FALSE" = TRUE ] && pisces="pisces"${start_pisces_from_restart} || pisces=""
  lim3="lim3"; xios="xios:detached"; rnfmapper="rnfmapper"
fi
+ [[ ecearth3 = ecearth3* ]]
+ [[ a2kv = *[!\ ]* ]]
+ start_nemo_from_restart=:start_from_restart
+ [[ '' = *[!\ ]* ]]
+ start_pisces_from_restart=
+ '[' FALSE = TRUE ']'
+ ocenudg=
+ '[' FALSE = TRUE ']'
+ surfresto=
+ '[' TRUE = TRUE ']'
+ elpin=:elpin
+ nemo=nemo:start_from_restart:elpin
+ '[' FALSE = TRUE ']'
+ pisces=
+ lim3=lim3
+ xios=xios:detached
+ rnfmapper=rnfmapper

#others
CMIP5_RCP=0
+ CMIP5_RCP=0
[[ "FALSE" = TRUE ]] && lpjg=lpjg:fdbck || lpjg=""
+ [[ FALSE = TRUE ]]
+ lpjg=
[[ "FALSE" = TRUE ]] && tm5=tm5:chem,o3fb,ch4fb,aerfb || tm5=""
+ [[ FALSE = TRUE ]]
+ tm5=
[ "TRUE" = TRUE ] && elpin=":elpin" || elpin=""
+ '[' TRUE = TRUE ']'
+ elpin=:elpin
[ "cmip6" == "" ]  && ifs_veg_source="era20c" || ifs_veg_source="cmip6"
+ '[' cmip6 == '' ']'
+ ifs_veg_source=cmip6
if [[ "end_leg" = FALSE ]] || [[ "end_leg" = "" ]] ; then save_ic="" ; else save_ic="save_ic:end_leg" ; fi
+ [[ end_leg = FALSE ]]
+ [[ end_leg = '' ]]
+ save_ic=save_ic:end_leg

config="${ifs} ${amip} ${atmnudg} ${sppt} ${nemo} ${pisces} ${lim3} ${rnfmapper} ${xios} oasis ${lpjg} ${tm5} ${save_ic}"
+ config='ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg'

# minimum sanity
has_config amip nemo && error "Cannot have both nemo and amip in config!!"
+ has_config amip nemo
+ ((  2  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' amip == any ']'
+ [[ amip == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ amip ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ amip ]]
+ for __m in '$config'
+ [[ lim3 =~ amip ]]
+ for __m in '$config'
+ [[ rnfmapper =~ amip ]]
+ for __m in '$config'
+ [[ xios:detached =~ amip ]]
+ for __m in '$config'
+ [[ oasis =~ amip ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ amip ]]
+ ((  __f  ))
+ return 1
! has_config ifs && error "The ESM script requires ifs in config"
+ has_config ifs
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' ifs == any ']'
+ [[ ifs == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ ifs ]]
+ [[ ifs =~ ifs ]]
+ __f=1
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ ifs ]]
+ for __m in '$config'
+ [[ lim3 =~ ifs ]]
+ for __m in '$config'
+ [[ rnfmapper =~ ifs ]]
+ for __m in '$config'
+ [[ xios:detached =~ ifs ]]
+ for __m in '$config'
+ [[ oasis =~ ifs ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ ifs ]]
+ ((  __f  ))
+ return 0

# Experiment name (exactly 4 letters!)
export exp_name=a3tb
+ export exp_name=a3tb
+ exp_name=a3tb

# Simulation start and end date. Use any (reasonable) syntax you want.
run_start_date="19930101"
+ run_start_date=19930101
# define run_start_date_as with proper format e.g. 19900101
export run_start_date_as=$(date -u -d "${run_start_date}" +%Y%m%d)
date -u -d "${run_start_date}" +%Y%m%d
++ date -u -d 19930101 +%Y%m%d
+ export run_start_date_as=19930101
+ run_start_date_as=19930101
run_end_date="19930101 + 1 month"
+ run_end_date='19930101 + 1 month'

# Simulation member. Use any (reasonable) syntax you want.
export member="fc01"
+ export member=fc01
+ member=fc01

# Set $force_run_from_scratch to 'true' if you want to force this run to start
# from scratch, possibly ignoring any restart files present in the run
# directory. Leave set to 'false' otherwise.
# NOTE: If set to 'true' the run directory $run_dir is cleaned!
CHUNK=1
+ CHUNK=1
force_run_from_scratch=TRUE
+ force_run_from_scratch=TRUE
force_run_from_scratch=${force_run_from_scratch:-false}
+ force_run_from_scratch=TRUE
force_run_from_scratch=$(echo ${force_run_from_scratch} | tr '[:upper:]' '[:lower:]')
echo ${force_run_from_scratch} | tr '[:upper:]' '[:lower:]'
++ echo TRUE
++ tr '[:upper:]' '[:lower:]'
+ force_run_from_scratch=true
# we only apply this for the first chunk
if ${force_run_from_scratch} && [[ "${CHUNK}" != "1" ]] ; then
  force_run_from_scratch=false
fi
+ true
+ [[ 1 != \1 ]]

# Resolution (TM5 resolution is set at compilation)
ifs_grid=T255L91
+ ifs_grid=T255L91
nem_grid=ORCA1L75
+ nem_grid=ORCA1L75

# Restart frequency. Use any (reasonable) number and time unit you want.
# For runs without restart, leave this variable empty
rst_freq="1 month"
+ rst_freq='1 month'

# Number of restart legs to be run in one go
run_num_legs=1
+ run_num_legs=1

# Coupling frequencies
has_config ifs tm5  && cpl_freq_atm_ctm_hrs=6
+ has_config ifs tm5
+ ((  2  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' ifs == any ']'
+ [[ ifs == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ ifs ]]
+ [[ ifs =~ ifs ]]
+ __f=1
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ ifs ]]
+ for __m in '$config'
+ [[ lim3 =~ ifs ]]
+ for __m in '$config'
+ [[ rnfmapper =~ ifs ]]
+ for __m in '$config'
+ [[ xios:detached =~ ifs ]]
+ for __m in '$config'
+ [[ oasis =~ ifs ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ ifs ]]
+ ((  __f  ))
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ tm5 ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ tm5 ]]
+ for __m in '$config'
+ [[ lim3 =~ tm5 ]]
+ for __m in '$config'
+ [[ rnfmapper =~ tm5 ]]
+ for __m in '$config'
+ [[ xios:detached =~ tm5 ]]
+ for __m in '$config'
+ [[ oasis =~ tm5 ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ tm5 ]]
+ ((  __f  ))
+ return 1
has_config ifs lpjg && cpl_freq_atm_lpjg_hrs=24
+ has_config ifs lpjg
+ ((  2  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' ifs == any ']'
+ [[ ifs == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ ifs ]]
+ [[ ifs =~ ifs ]]
+ __f=1
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ ifs ]]
+ for __m in '$config'
+ [[ lim3 =~ ifs ]]
+ for __m in '$config'
+ [[ rnfmapper =~ ifs ]]
+ for __m in '$config'
+ [[ xios:detached =~ ifs ]]
+ for __m in '$config'
+ [[ oasis =~ ifs ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ ifs ]]
+ ((  __f  ))
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ lpjg ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ lpjg ]]
+ for __m in '$config'
+ [[ lim3 =~ lpjg ]]
+ for __m in '$config'
+ [[ rnfmapper =~ lpjg ]]
+ for __m in '$config'
+ [[ xios:detached =~ lpjg ]]
+ for __m in '$config'
+ [[ oasis =~ lpjg ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ lpjg ]]
+ ((  __f  ))
+ return 1

# Don't change the coupling frequency because UPDCLIE (where SST and SIC
# are updated) is called every 24 hours (hardcoded in ifs-36r4/src/ifs/utility/updtim.F90)
has_config amip && cpl_freq_amip_sec=86400
+ has_config amip
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' amip == any ']'
+ [[ amip == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ amip ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ amip ]]
+ for __m in '$config'
+ [[ lim3 =~ amip ]]
+ for __m in '$config'
+ [[ rnfmapper =~ amip ]]
+ for __m in '$config'
+ [[ xios:detached =~ amip ]]
+ for __m in '$config'
+ [[ oasis =~ amip ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ amip ]]
+ ((  __f  ))
+ return 1

# Directories
start_dir=${PWD}
+ start_dir=/gpfs/scratch/bsc32/bsc32627/a3tb
ctrl_file_dir=${start_dir}/ctrl
+ ctrl_file_dir=/gpfs/scratch/bsc32/bsc32627/a3tb/ctrl
output_control_files_dir=${start_dir}/ctrl/output-control-files/cmip6/ScenarioMIP/EC-EARTH-CC/cmip6-experiment-ScenarioMIP-ssp245
+ output_control_files_dir=/gpfs/scratch/bsc32/bsc32627/a3tb/ctrl/output-control-files/cmip6/ScenarioMIP/EC-EARTH-CC/cmip6-experiment-ScenarioMIP-ssp245

# Architecture
build_arch=ecconf
+ build_arch=ecconf
use_machinefile=TRUE
+ use_machinefile=TRUE

# This file is used to store information about restarts
ece_info_file="ece.info"
+ ece_info_file=ece.info

# -----------------------------------------------------------------------------
# *** Read platform dependent configuration
# -----------------------------------------------------------------------------
source ./ecconf.cfg
+ source ./ecconf.cfg
# Platform dependent configuration functions for MareNostrum
#(mnX.bsc.es)

function configure()
{
    # This function should configure all settings/modules needed to
    # later prepare the EC-Earth run directory and set variables used
    # in the run script

    # SCRATCH is not defined in MN3, define it here
    # and also make sure it is defined when compiling
    export SCRATCH=/gpfs/scratch/`id -gn`/${USER}

    # Configure paths for building/running EC-Earth
    ecearth_src_dir=${SCRATCH}/${exp_name}/auto-ecearth3/sources/sources
    run_dir=${SCRATCH}/${exp_name}/${run_start_date_as}/${member}/runtime
    ini_data_dir=${SCRATCH}/${exp_name}/${run_start_date_as}/${member}/inidata

    # File for standard output.
    # NOTE: This will be modified for restart jobs!
    stdout_file=${start_dir}/out/$(basename ${SLURM_JOB_NAME}).out

    # Resubmit this job for automatic restarts? [true/false]
    # Also, add options for the resubmit command here.
    resubmit_job=false
    resubmit_opt="-W 4:00"

    # Configure grib api paths
    export GRIB_DEFINITION_PATH=${SCRATCH}/${exp_name}/auto-ecearth3/sources/sources/util/grib_table_126:/apps/GRIB/1.14.0/INTEL//share/grib_api/definitions
    export GRIB_SAMPLES_PATH=/apps/GRIB/1.14.0/INTEL//share/grib_api/ifs_samples/grib1
    export GRIB_BIN_PATH=/apps/GRIB/1.14.0/INTEL//bin

    # Configure number of processors per node
    proc_per_node=48

    # Configure and load modules
    pre_load_modules_cmd="module purge"
    module_list="
               intel/2017.4 impi/2018.4 netcdf/4.2 hdf5/1.8.19 perl/5.26 mkl/2018.4 CDO/1.8.2 szip/2.1.1
            "

    if [ -n "${module_list}" ]
    then
        set +eu
        if [ -n "${pre_load_modules_cmd}" ]
        then
            ${pre_load_modules_cmd}
        fi
        for m in "${module_list}"
        do
            module add $m
        done
        set -eu
    fi

    # Add directories to the shared library search path
    if [ -n "/apps/GRIB/1.14.0/INTEL/lib" ]
    then
        export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"/apps/GRIB/1.14.0/INTEL/lib"
    fi

    # Use machinefiles or not
    [[ `echo "$use_machinefile" | tr '[:upper:]' '[:lower:]'` == true ]] && use_machinefile=true || use_machinefile=false

    ulimit -s unlimited

    # Load specific IMPI environment configuration
    configure_impi
}

function configure_python()
{
    # specific for python+eccodes setup - used for OSM pre/post-processing
    # it would be simple to do the following in configure
    # module load eccodes/2.8.0 python/2.7.13
    module load eccodes/2.8.0 python/2.7.13
    unset GRIB_DEFINITION_PATH
    unset GRIB_SAMPLES_PATH
    unset GRIB_BIN_PATH
    export GRIB_BIN_PATH=/apps/ECCODES/2.8.0/INTEL/bin
}

function configure_impi()
{
    [ -z "${OMP_NUM_THREADS-}" ] && export OMP_NUM_THREADS=1
    export I_MPI_DEBUG=5
}

function get_hosts()
{
    # This function uses a scheduler command to get the hosts allocated for the current job
    hosts=(`scontrol show hostname | paste -s`)
}

function machinefile_config()
{
    # User configuration starts here
    # hard-coded c4mip configurations, must use the proper _numproc settings
    if has_config ifs nemo pisces rnfmapper xios lpjg ; then
      if ! has_config tm5 ; then
            ifs_ppn=48 ; [[ ${ifs_numproc}  != 336 ]] && info "wrong numproc setting for ifs in machinefile_config" || true
            nem_ppn=43 ; [[ ${nem_numproc}  != 380 ]] && info "wrong numproc setting for nemo in machinefile_config" || true
            xio_ppn=5  ; [[ ${xio_numproc}  !=   5 ]] && info "wrong numproc setting for xios in machinefile_config" || true
            lpjg_ppn=5 ; [[ ${lpjg_numproc} !=  40 ]] && info "wrong numproc setting for lpjg in machinefile_config" || true
        else
            ifs_ppn=48 ; [[ ${ifs_numproc}  != 256 ]] && info "wrong numproc setting for ifs in machinefile_config" || true
            nem_ppn=46 ; [[ ${nem_numproc}  != 192 ]] && info "wrong numproc setting for nemo in machinefile_config" || true
            xio_ppn=2  ; [[ ${xio_numproc}  !=   2 ]] && info "wrong numproc setting for xios in machinefile_config" || true
            lpjg_ppn=2 ; [[ ${lpjg_numproc} !=   8 ]] && info "wrong numproc setting for lpjg in machinefile_config" || true
            tm5_ppn=4  ; [[ ${tm5_numproc}  !=   4 ]] && info "wrong numproc setting for tm5 in machinefile_config" || true
        fi
      else
        # Add any new exclusive binary here
        ifs_exc=TRUE
        nem_exc=TRUE
        xio_exc=TRUE
        lpjg_exc=TRUE
        tm5_exc=TRUE

        # Modify the allocation to each binary using more than one process here
        ifs_ppn=48
        if has_config nemo && [ "$nem_grid" == "ORCA025L75" ] ; then
          nem_ppn=47
        else
          nem_ppn=48
        fi
        xio_ppn=48
        lpjg_ppn=48
        tm5_ppn=45
    fi
}

function machinefile_init()
{
    # Get max processes per node from the platform variable
    max_ppn=$proc_per_node

    components=( ifs nem xio rnf amip lpjg )
    if $(has_config tm5)
    then
      components=( "${components[@]}" "tm5" )
    fi
    
    for component in ${components[@]}
    do
      eval ${component}_exc=FALSE
      eval ${component}_ppn=1
    done
  
    # Call user configuration and get_host functions
    machinefile_config
    get_hosts
 
    # Declare array to store the processes as they are assigned
    declare -a -g processes_hosts
    for n in `seq 0 ${#hosts[@]}`
    do
      processes_hosts[$n]=0
    done

    > machinefile
    
    current_hostid=0
}

machinefile_find_available_node()
{
  while [ $((${processes_hosts[$current_hostid]} + ${!ppn})) -gt $max_ppn ]
  do
    let "current_hostid += 1"
  done
}

machinefile_add()
{
  total_proc=$2
  # Iterate through all the possible binaries
  for component in ${components[@]} 
  do
    binary="${component}_exe_file"
    exclusive="${component}_exc"
    # Check if the current binary matches the input executable
    if [ ./$(basename ${!binary}) = "$1" ]
    then
       ppn="${component}_ppn"
       # Exclusive mode: start allocation at the first empty node
       if [[ ${!exclusive} == "TRUE" ]]
       then
         while [ ${processes_hosts[$current_hostid]} -gt 0 ]
         do
           let "current_hostid += 1"
         done
       # Shared mode: start allocation in the first node with enough free cores
       # Notice that only the first node is checked
       # Then, if a previous binary had "exc=TRUE", allocation space is not ensure in subsequent nodes
       else
         current_hostid=0
         machinefile_find_available_node
       fi

       # Allocate ppn cores in each of the subsequent nodes till there are no more processes to assign
       count=0
       while [ ${total_proc} -gt 0 ]
       do
         if [ ${current_hostid} -ge ${#hosts[@]} ]
         then
           echo "Not enough computing nodes"
           exit 1
         fi
         current_hostname=${hosts[$current_hostid]}
         while [[ ${total_proc} -gt 0 && ${count} -lt ${!ppn} ]] 
         do 
           echo ${hosts[$current_hostid]} >> machinefile
           let "count += 1"
           let "processes_hosts[$current_hostid] += 1"
           let "total_proc -= 1" || true 
         done 
         if [ ${count} -eq ${!ppn} ]
         then
           let "current_hostid += 1"
           machinefile_find_available_node
           count=0
         fi
       done
    fi
  done
}

function launch()
{

    cmd="mpirun"

    if [ "$use_machinefile" = "true" ]
    then
      cmd="mpirun -machinefile machinefile"
      machinefile_init
    fi

    while (( "$#" ))
    do
        # Get number of MPI ranks and executable name
        nranks=$1
        executable=./$(basename $2)
        
        if [ "$use_machinefile" = "true" ]
        then
          machinefile_add $executable $nranks
        fi

        shift
        shift

        cmd+=" -np $nranks $executable"

        # Add any arguments to executable
        while (( "$#" )) && [ "$1" != "--" ]
        do
            cmd+=" $1"
            shift
        done
        shift || true

        # Add colon of more executables follow
        (( "$#" )) && cmd+=" :"
    done

    $cmd
}

function finalise()
{
    # This function should execute of any post run functionality, e.g.
    # platform dependent cleaning or a resubmit

    if ${resubmit_job} && [ $(date -d "${leg_end_date}" +%s) -lt $(date -d "${run_end_date}" +%s) ]
    then
        info "Resubmitting job for leg $((leg_number+1))"
        # Need to go to start_dir to find the run script
        cd ${start_dir}
        # Submit command
        # Note: This does not work if you explicitely specify a job name!
#        bsub -n ${SLURM_JOB_NUM_NODES} \
#             -w ${SLURM_JOB_ID} \
#             -oo ${run_dir}/$(basename ${stdout_file}).$(printf %03d $((leg_number+1)))  \
#             ${resubmit_opt} \
#             ${SLURM_JOB_NAME}
    fi
}

function run_node()
{
    # This function launches a command once on each node used in the job, currently only used in ece-lsm.sh runcript
    mpirun -n $SLURM_JOB_NUM_NODES -ppn 1 -hosts `scontrol show hostname | paste -d, -s` bash -c "$1"
}

configure
+ configure
id -gn
++ id -gn
+ export SCRATCH=/gpfs/scratch/bsc32/bsc32627
+ SCRATCH=/gpfs/scratch/bsc32/bsc32627
+ ecearth_src_dir=/gpfs/scratch/bsc32/bsc32627/a3tb/auto-ecearth3/sources/sources
+ run_dir=/gpfs/scratch/bsc32/bsc32627/a3tb/19930101/fc01/runtime
+ ini_data_dir=/gpfs/scratch/bsc32/bsc32627/a3tb/19930101/fc01/inidata
basename ${SLURM_JOB_NAME}
++ basename a3tb_19930101_fc01_1_SIM
+ stdout_file=/gpfs/scratch/bsc32/bsc32627/a3tb/out/a3tb_19930101_fc01_1_SIM.out
+ resubmit_job=false
+ resubmit_opt='-W 4:00'
+ export GRIB_DEFINITION_PATH=/gpfs/scratch/bsc32/bsc32627/a3tb/auto-ecearth3/sources/sources/util/grib_table_126:/apps/GRIB/1.14.0/INTEL//share/grib_api/definitions
+ GRIB_DEFINITION_PATH=/gpfs/scratch/bsc32/bsc32627/a3tb/auto-ecearth3/sources/sources/util/grib_table_126:/apps/GRIB/1.14.0/INTEL//share/grib_api/definitions
+ export GRIB_SAMPLES_PATH=/apps/GRIB/1.14.0/INTEL//share/grib_api/ifs_samples/grib1
+ GRIB_SAMPLES_PATH=/apps/GRIB/1.14.0/INTEL//share/grib_api/ifs_samples/grib1
+ export GRIB_BIN_PATH=/apps/GRIB/1.14.0/INTEL//bin
+ GRIB_BIN_PATH=/apps/GRIB/1.14.0/INTEL//bin
+ proc_per_node=48
+ pre_load_modules_cmd='module purge'
+ module_list='
               intel/2017.4 impi/2018.4 netcdf/4.2 hdf5/1.8.19 perl/5.26 mkl/2018.4 CDO/1.8.2 szip/2.1.1
            '
+ '[' -n '
               intel/2017.4 impi/2018.4 netcdf/4.2 hdf5/1.8.19 perl/5.26 mkl/2018.4 CDO/1.8.2 szip/2.1.1
            ' ']'
+ set +eu
+ '[' -n 'module purge' ']'
+ module purge
$LMOD_CMD bash "$@"
++ /apps/modules/LMOD/7.5/lmod/lmod/libexec/lmod bash purge
remove mkl/2017.4 (LD_LIBRARY_PATH) 
remove impi/2017.4 (PATH, MANPATH, LD_LIBRARY_PATH) 
+ eval unset 'CC;' unset 'CXX;' unset 'C_INCLUDE_PATH;' unset 'F77;' unset 'F90;' unset 'F95;' unset 'FC;' unset 'INTEL_HOME;' unset 'INTEL_INC;' unset 'INTEL_VERSION;' unset 'I_MPI_CC;' unset 'I_MPI_CXX;' unset 'I_MPI_F77;' unset 'I_MPI_F90;' unset 'I_MPI_ROOT;' unset 'LD_LIBRARY_PATH;' unset 'LOADEDMODULES;' 'MANPATH="/apps/modules/LMOD/7.5/lmod/lmod/share/man:/usr/local/man:/usr/share/man";' export 'MANPATH;' unset 'MKLROOT;' unset 'MKL_HOME;' 'MODULEPATH="/apps/modules/modulefiles/applications:/apps/modules/modulefiles/compilers:/apps/modules/modulefiles/environment:/apps/modules/modulefiles/libraries:/apps/modules/modulefiles/tools:/apps/modules/modulefiles/prace";' export 'MODULEPATH;' unset 'MPI;' unset 'MPICH_CC;' unset 'MPICH_CXX;' unset 'MPICH_F77;' unset 'MPICH_F90;' unset 'MPICH_FC;' unset 'MPI_ROOT;' unset 'MPI_V;' unset 'MP_COMPILER;' unset 'OMPI_CC;' unset 'OMPI_CXX;' unset 'OMPI_F77;' unset 'OMPI_F90;' unset 'OMPI_FC;' 'PATH="/usr/local/bin:/usr/bin:/bin:/usr/games:/usr/lpp/mmfs/bin";' export 'PATH;' unset '_LMFILES_;' '_ModuleTable001_="X01vZHVsZVRhYmxlXz17WyJNVHZlcnNpb24iXT0zLFsiY19yZWJ1aWxkVGltZSJdPWZhbHNlLFsiY19zaG9ydFRpbWUiXT1mYWxzZSxkZXB0aFQ9e30sZmFtaWx5PXt9LG1UPXt9LG1wYXRoQT17Ii9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvYXBwbGljYXRpb25zIiwiL2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9jb21waWxlcnMiLCIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2Vudmlyb25tZW50IiwiL2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9saWJyYXJpZXMiLCIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL3Rvb2xzIiwiL2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9wcmFjZSIsfSxbInN5c3RlbUJhc2VNUEFUSCJdPSIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVz";' export '_ModuleTable001_;' '_ModuleTable002_="L2FwcGxpY2F0aW9uczovYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2NvbXBpbGVyczovYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2Vudmlyb25tZW50Oi9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvbGlicmFyaWVzOi9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvdG9vbHM6L2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9wcmFjZSIsfQ==";' export '_ModuleTable002_;' '_ModuleTable_Sz_="2";' export '_ModuleTable_Sz_;' unset '_ModuleTable004_;'
unset CC; unset CXX; unset C_INCLUDE_PATH; unset F77; unset F90; unset F95; unset FC; unset INTEL_HOME; unset INTEL_INC; unset INTEL_VERSION; unset I_MPI_CC; unset I_MPI_CXX; unset I_MPI_F77; unset I_MPI_F90; unset I_MPI_ROOT; unset LD_LIBRARY_PATH; unset LOADEDMODULES; MANPATH="/apps/modules/LMOD/7.5/lmod/lmod/share/man:/usr/local/man:/usr/share/man"; export MANPATH; unset MKLROOT; unset MKL_HOME; MODULEPATH="/apps/modules/modulefiles/applications:/apps/modules/modulefiles/compilers:/apps/modules/modulefiles/environment:/apps/modules/modulefiles/libraries:/apps/modules/modulefiles/tools:/apps/modules/modulefiles/prace"; export MODULEPATH; unset MPI; unset MPICH_CC; unset MPICH_CXX; unset MPICH_F77; unset MPICH_F90; unset MPICH_FC; unset MPI_ROOT; unset MPI_V; unset MP_COMPILER; unset OMPI_CC; unset OMPI_CXX; unset OMPI_F77; unset OMPI_F90; unset OMPI_FC; PATH="/usr/local/bin:/usr/bin:/bin:/usr/games:/usr/lpp/mmfs/bin"; export PATH; unset _LMFILES_; _ModuleTable001_="X01vZHVsZVRhYmxlXz17WyJNVHZlcnNpb24iXT0zLFsiY19yZWJ1aWxkVGltZSJdPWZhbHNlLFsiY19zaG9ydFRpbWUiXT1mYWxzZSxkZXB0aFQ9e30sZmFtaWx5PXt9LG1UPXt9LG1wYXRoQT17Ii9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvYXBwbGljYXRpb25zIiwiL2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9jb21waWxlcnMiLCIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2Vudmlyb25tZW50IiwiL2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9saWJyYXJpZXMiLCIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL3Rvb2xzIiwiL2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9wcmFjZSIsfSxbInN5c3RlbUJhc2VNUEFUSCJdPSIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVz"; export _ModuleTable001_; _ModuleTable002_="L2FwcGxpY2F0aW9uczovYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2NvbXBpbGVyczovYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2Vudmlyb25tZW50Oi9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvbGlicmFyaWVzOi9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvdG9vbHM6L2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9wcmFjZSIsfQ=="; export _ModuleTable002_; _ModuleTable_Sz_="2"; export _ModuleTable_Sz_; unset _ModuleTable004_;
++ unset CC
++ unset CXX
++ unset C_INCLUDE_PATH
++ unset F77
++ unset F90
++ unset F95
++ unset FC
++ unset INTEL_HOME
++ unset INTEL_INC
++ unset INTEL_VERSION
++ unset I_MPI_CC
++ unset I_MPI_CXX
++ unset I_MPI_F77
++ unset I_MPI_F90
++ unset I_MPI_ROOT
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ MANPATH=/apps/modules/LMOD/7.5/lmod/lmod/share/man:/usr/local/man:/usr/share/man
++ export MANPATH
++ unset MKLROOT
++ unset MKL_HOME
++ MODULEPATH=/apps/modules/modulefiles/applications:/apps/modules/modulefiles/compilers:/apps/modules/modulefiles/environment:/apps/modules/modulefiles/libraries:/apps/modules/modulefiles/tools:/apps/modules/modulefiles/prace
++ export MODULEPATH
++ unset MPI
++ unset MPICH_CC
++ unset MPICH_CXX
++ unset MPICH_F77
++ unset MPICH_F90
++ unset MPICH_FC
++ unset MPI_ROOT
++ unset MPI_V
++ unset MP_COMPILER
++ unset OMPI_CC
++ unset OMPI_CXX
++ unset OMPI_F77
++ unset OMPI_F90
++ unset OMPI_FC
++ PATH=/usr/local/bin:/usr/bin:/bin:/usr/games:/usr/lpp/mmfs/bin
++ export PATH
++ unset _LMFILES_
++ _ModuleTable001_=X01vZHVsZVRhYmxlXz17WyJNVHZlcnNpb24iXT0zLFsiY19yZWJ1aWxkVGltZSJdPWZhbHNlLFsiY19zaG9ydFRpbWUiXT1mYWxzZSxkZXB0aFQ9e30sZmFtaWx5PXt9LG1UPXt9LG1wYXRoQT17Ii9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvYXBwbGljYXRpb25zIiwiL2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9jb21waWxlcnMiLCIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2Vudmlyb25tZW50IiwiL2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9saWJyYXJpZXMiLCIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL3Rvb2xzIiwiL2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9wcmFjZSIsfSxbInN5c3RlbUJhc2VNUEFUSCJdPSIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVz
++ export _ModuleTable001_
++ _ModuleTable002_=L2FwcGxpY2F0aW9uczovYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2NvbXBpbGVyczovYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2Vudmlyb25tZW50Oi9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvbGlicmFyaWVzOi9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvdG9vbHM6L2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9wcmFjZSIsfQ==
++ export _ModuleTable002_
++ _ModuleTable_Sz_=2
++ export _ModuleTable_Sz_
++ unset _ModuleTable004_
${LMOD_SETTARG_CMD:-:} -s sh
++ : -s sh
+ eval
+ for m in '"${module_list}"'
+ module add intel/2017.4 impi/2018.4 netcdf/4.2 hdf5/1.8.19 perl/5.26 mkl/2018.4 CDO/1.8.2 szip/2.1.1
$LMOD_CMD bash "$@"
++ /apps/modules/LMOD/7.5/lmod/lmod/libexec/lmod bash add intel/2017.4 impi/2018.4 netcdf/4.2 hdf5/1.8.19 perl/5.26 mkl/2018.4 CDO/1.8.2 szip/2.1.1
Set INTEL compilers as MPI wrappers backend 
load impi/2018.4 (PATH, MANPATH, LD_LIBRARY_PATH) 
load netcdf/4.2 (PATH, LD_LIBRARY_PATH, NETCDF, NETCDF_PREFIX, NETCDF_HOME,
NETCDF_INC, NETCDF_LIB, NETCDF_ROOT) 
load hdf5/1.8.19 (PATH, LD_LIBRARY_PATH, LIBRARY_PATH, C_INCLUDE_PATH,
CPLUS_INCLUDE_PATH) 
load perl/5.26 (PATH, LD_LIBRARY_PATH, MANPATH, PERL5LIB) 
load mkl/2018.4 (LD_LIBRARY_PATH) 
load CDO/1.8.2 (PATH) 
load szip/2.1.1 (C_INCLUDE_PATH, LD_LIBRARY_PATH) 
+ eval 'CC="icc";' export 'CC;' 'CFLAGS="-I/apps/SZIP/2.1.1/INTEL/include' '-I/apps/HDF5/1.8.19/INTEL/IMPI/include";' export 'CFLAGS;' 'CPLUS_INCLUDE_PATH="/apps/HDF5/1.8.19/INTEL/IMPI/include";' export 'CPLUS_INCLUDE_PATH;' 'CXX="icpc";' export 'CXX;' 'CXXFLAGS="-I/apps/SZIP/2.1.1/INTEL/include' '-I/apps/HDF5/1.8.19/INTEL/IMPI/include";' export 'CXXFLAGS;' 'C_INCLUDE_PATH="/apps/SZIP/2.1.1/INTEL/include:/apps/INTEL/2018.4.057/mkl/include:/apps/HDF5/1.8.19/INTEL/IMPI/include";' export 'C_INCLUDE_PATH;' 'F77="ifort";' export 'F77;' 'F90="ifort";' export 'F90;' 'F95="ifort";' export 'F95;' 'FC="ifort";' export 'FC;' 'HDF5_DIR="/apps/HDF5/1.8.19/INTEL/IMPI";' export 'HDF5_DIR;' 'INTEL_HOME="/apps/INTEL/2017.4";' export 'INTEL_HOME;' 'INTEL_INC="/apps/INTEL/2017.4/include";' export 'INTEL_INC;' 'INTEL_VERSION="2017.4";' export 'INTEL_VERSION;' 'I_MPI_CC="icc";' export 'I_MPI_CC;' 'I_MPI_CXX="icpc";' export 'I_MPI_CXX;' 'I_MPI_F77="ifort";' export 'I_MPI_F77;' 'I_MPI_F90="ifort";' export 'I_MPI_F90;' 'I_MPI_ROOT="/apps/INTEL/2018.4.057/impi/2018.4.274";' export 'I_MPI_ROOT;' 'LDFLAGS="-L/apps/SZIP/2.1.1/INTEL/lib' -lsz -L/apps/HDF5/1.8.19/INTEL/IMPI/lib -lhdf5 '-lhdf5_hl";' export 'LDFLAGS;' 'LD_LIBRARY_PATH="/apps/SZIP/2.1.1/INTEL/lib:/apps/INTEL/2018.4.057/mkl/lib/intel64:/apps/HDF5/1.8.19/INTEL/IMPI/lib:/apps/NETCDF/4.2/INTEL/IMPI/lib:/apps/INTEL/2018.4.057/impi/2018.4.274/lib64:/apps/INTEL/2017.4/lib/intel64";' export 'LD_LIBRARY_PATH;' 'LIBRARY_PATH="/apps/HDF5/1.8.19/INTEL/IMPI/lib";' export 'LIBRARY_PATH;' 'LOADEDMODULES="intel/2017.4:impi/2018.4:netcdf/4.2:hdf5/1.8.19:perl/5.26:mkl/2018.4:CDO/1.8.2:szip/2.1.1";' export 'LOADEDMODULES;' 'MANPATH="/apps/PERL/5.26/man:/apps/INTEL/2018.4.057/impi/2018.4.274/man:/apps/INTEL/2017.4/man/common:/apps/modules/LMOD/7.5/lmod/lmod/share/man:/usr/local/man:/usr/share/man";' export 'MANPATH;' 'MKLROOT="/apps/INTEL/2018.4.057/mkl";' export 'MKLROOT;' 'MKL_HOME="/apps/INTEL/2018.4.057/mkl";' export 'MKL_HOME;' 'MODULEPATH="/apps/modules/modulefiles/applications:/apps/modules/modulefiles/compilers:/apps/modules/modulefiles/environment:/apps/modules/modulefiles/libraries:/apps/modules/modulefiles/tools:/apps/modules/modulefiles/prace";' export 'MODULEPATH;' 'MPI="impi";' export 'MPI;' 'MPICH_CC="icc";' export 'MPICH_CC;' 'MPICH_CXX="icpc";' export 'MPICH_CXX;' 'MPICH_F77="ifort";' export 'MPICH_F77;' 'MPICH_F90="ifort";' export 'MPICH_F90;' 'MPICH_FC="ifort";' export 'MPICH_FC;' 'MPI_ROOT="/apps/INTEL/2018.4.057/impi/2018.4.274";' export 'MPI_ROOT;' 'MPI_V="2018.4.057";' export 'MPI_V;' 'MP_COMPILER="intel";' export 'MP_COMPILER;' 'NETCDF="/apps/NETCDF/4.2/INTEL/IMPI";' export 'NETCDF;' 'NETCDF_HOME="/apps/NETCDF/4.2/INTEL/IMPI";' export 'NETCDF_HOME;' 'NETCDF_INC="/apps/NETCDF/4.2/INTEL/IMPI/include";' export 'NETCDF_INC;' 'NETCDF_LIB="/apps/NETCDF/4.2/INTEL/IMPI/lib";' export 'NETCDF_LIB;' 'NETCDF_PREFIX="/apps/NETCDF/4.2/INTEL/IMPI";' export 'NETCDF_PREFIX;' 'NETCDF_ROOT="/apps/NETCDF/4.2/INTEL/IMPI";' export 'NETCDF_ROOT;' 'OMPI_CC="icc";' export 'OMPI_CC;' 'OMPI_CXX="icpc";' export 'OMPI_CXX;' 'OMPI_F77="ifort";' export 'OMPI_F77;' 'OMPI_F90="ifort";' export 'OMPI_F90;' 'OMPI_FC="ifort";' export 'OMPI_FC;' 'PATH="/apps/CDO/1.8.2/INTEL/IMPI/bin:/apps/PERL/5.26/bin:/apps/HDF5/1.8.19/INTEL/IMPI/bin:/apps/NETCDF/4.2/INTEL/IMPI/bin:/apps/INTEL/2018.4.057/impi/2018.4.274/bin64:/apps/INTEL/2017.4/bin:/usr/local/bin:/usr/bin:/bin:/usr/games:/usr/lpp/mmfs/bin";' export 'PATH;' 'PERL5LIB="/apps/PERL/5.26/lib/perl5/site_perl/5.26.0:/apps/PERL/5.26/lib/perl5/5.26.0:/apps/PERL/5.26/lib/perl5/site_perl/5.26.0/x86_64-linux";' export 'PERL5LIB;' '_LMFILES_="/apps/modules/modulefiles/compilers/intel/2017.4:/apps/modules/modulefiles/environment/impi/2018.4:/apps/modules/modulefiles/libraries/netcdf/4.2:/apps/modules/modulefiles/libraries/hdf5/1.8.19:/apps/modules/modulefiles/tools/perl/5.26:/apps/modules/modulefiles/libraries/mkl/2018.4:/apps/modules/modulefiles/applications/CDO/1.8.2:/apps/modules/modulefiles/libraries/szip/2.1.1";' export '_LMFILES_;' '_ModuleTable001_="X01vZHVsZVRhYmxlXz17WyJNVHZlcnNpb24iXT0zLFsiY19yZWJ1aWxkVGltZSJdPWZhbHNlLFsiY19zaG9ydFRpbWUiXT1mYWxzZSxkZXB0aFQ9e30sZmFtaWx5PXt9LG1UPXtDRE89e1siZm4iXT0iL2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9hcHBsaWNhdGlvbnMvQ0RPLzEuOC4yIixbImZ1bGxOYW1lIl09IkNETy8xLjguMiIsWyJsb2FkT3JkZXIiXT03LHByb3BUPXt9LFsic3RhdHVzIl09ImFjdGl2ZSIsWyJ1c2VyTmFtZSJdPSJDRE8vMS44LjIiLH0saGRmNT17WyJmbiJdPSIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2xpYnJhcmllcy9oZGY1LzEuOC4xOSIsWyJmdWxsTmFtZSJdPSJoZGY1LzEuOC4xOSIsWyJsb2FkT3JkZXIiXT00LHByb3BUPXt9LFsic3RhdHVz";' export '_ModuleTable001_;' '_ModuleTable002_="Il09ImFjdGl2ZSIsWyJ1c2VyTmFtZSJdPSJoZGY1LzEuOC4xOSIsfSxpbXBpPXtbImZuIl09Ii9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvZW52aXJvbm1lbnQvaW1waS8yMDE4LjQiLFsiZnVsbE5hbWUiXT0iaW1waS8yMDE4LjQiLFsibG9hZE9yZGVyIl09Mixwcm9wVD17fSxbInN0YXR1cyJdPSJhY3RpdmUiLFsidXNlck5hbWUiXT0iaW1waS8yMDE4LjQiLH0saW50ZWw9e1siZm4iXT0iL2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9jb21waWxlcnMvaW50ZWwvMjAxNy40IixbImZ1bGxOYW1lIl09ImludGVsLzIwMTcuNCIsWyJsb2FkT3JkZXIiXT0xLHByb3BUPXt9LFsic3RhdHVzIl09ImFjdGl2ZSIsWyJ1c2VyTmFtZSJdPSJpbnRlbC8yMDE3LjQiLH0sbWtsPXtbImZu";' export '_ModuleTable002_;' '_ModuleTable003_="Il09Ii9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvbGlicmFyaWVzL21rbC8yMDE4LjQiLFsiZnVsbE5hbWUiXT0ibWtsLzIwMTguNCIsWyJsb2FkT3JkZXIiXT02LHByb3BUPXt9LFsic3RhdHVzIl09ImFjdGl2ZSIsWyJ1c2VyTmFtZSJdPSJta2wvMjAxOC40Iix9LG5ldGNkZj17WyJmbiJdPSIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2xpYnJhcmllcy9uZXRjZGYvNC4yIixbImZ1bGxOYW1lIl09Im5ldGNkZi80LjIiLFsibG9hZE9yZGVyIl09Myxwcm9wVD17fSxbInN0YXR1cyJdPSJhY3RpdmUiLFsidXNlck5hbWUiXT0ibmV0Y2RmLzQuMiIsfSxwZXJsPXtbImZuIl09Ii9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvdG9vbHMvcGVybC81LjI2IixbImZ1bGxOYW1lIl09";' export '_ModuleTable003_;' '_ModuleTable004_="InBlcmwvNS4yNiIsWyJsb2FkT3JkZXIiXT01LHByb3BUPXt9LFsic3RhdHVzIl09ImFjdGl2ZSIsWyJ1c2VyTmFtZSJdPSJwZXJsLzUuMjYiLH0sc3ppcD17WyJmbiJdPSIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2xpYnJhcmllcy9zemlwLzIuMS4xIixbImZ1bGxOYW1lIl09InN6aXAvMi4xLjEiLFsibG9hZE9yZGVyIl09OCxwcm9wVD17fSxbInN0YXR1cyJdPSJhY3RpdmUiLFsidXNlck5hbWUiXT0ic3ppcC8yLjEuMSIsfSx9LG1wYXRoQT17Ii9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvYXBwbGljYXRpb25zIiwiL2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9jb21waWxlcnMiLCIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2Vudmlyb25tZW50IiwiL2FwcHMvbW9k";' export '_ModuleTable004_;' '_ModuleTable005_="dWxlcy9tb2R1bGVmaWxlcy9saWJyYXJpZXMiLCIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL3Rvb2xzIiwiL2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9wcmFjZSIsfSxbInN5c3RlbUJhc2VNUEFUSCJdPSIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2FwcGxpY2F0aW9uczovYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2NvbXBpbGVyczovYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2Vudmlyb25tZW50Oi9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvbGlicmFyaWVzOi9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvdG9vbHM6L2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9wcmFjZSIsfQ==";' export '_ModuleTable005_;' '_ModuleTable_Sz_="5";' export '_ModuleTable_Sz_;'
CC="icc"; export CC; CFLAGS="-I/apps/SZIP/2.1.1/INTEL/include -I/apps/HDF5/1.8.19/INTEL/IMPI/include"; export CFLAGS; CPLUS_INCLUDE_PATH="/apps/HDF5/1.8.19/INTEL/IMPI/include"; export CPLUS_INCLUDE_PATH; CXX="icpc"; export CXX; CXXFLAGS="-I/apps/SZIP/2.1.1/INTEL/include -I/apps/HDF5/1.8.19/INTEL/IMPI/include"; export CXXFLAGS; C_INCLUDE_PATH="/apps/SZIP/2.1.1/INTEL/include:/apps/INTEL/2018.4.057/mkl/include:/apps/HDF5/1.8.19/INTEL/IMPI/include"; export C_INCLUDE_PATH; F77="ifort"; export F77; F90="ifort"; export F90; F95="ifort"; export F95; FC="ifort"; export FC; HDF5_DIR="/apps/HDF5/1.8.19/INTEL/IMPI"; export HDF5_DIR; INTEL_HOME="/apps/INTEL/2017.4"; export INTEL_HOME; INTEL_INC="/apps/INTEL/2017.4/include"; export INTEL_INC; INTEL_VERSION="2017.4"; export INTEL_VERSION; I_MPI_CC="icc"; export I_MPI_CC; I_MPI_CXX="icpc"; export I_MPI_CXX; I_MPI_F77="ifort"; export I_MPI_F77; I_MPI_F90="ifort"; export I_MPI_F90; I_MPI_ROOT="/apps/INTEL/2018.4.057/impi/2018.4.274"; export I_MPI_ROOT; LDFLAGS="-L/apps/SZIP/2.1.1/INTEL/lib -lsz -L/apps/HDF5/1.8.19/INTEL/IMPI/lib -lhdf5 -lhdf5_hl"; export LDFLAGS; LD_LIBRARY_PATH="/apps/SZIP/2.1.1/INTEL/lib:/apps/INTEL/2018.4.057/mkl/lib/intel64:/apps/HDF5/1.8.19/INTEL/IMPI/lib:/apps/NETCDF/4.2/INTEL/IMPI/lib:/apps/INTEL/2018.4.057/impi/2018.4.274/lib64:/apps/INTEL/2017.4/lib/intel64"; export LD_LIBRARY_PATH; LIBRARY_PATH="/apps/HDF5/1.8.19/INTEL/IMPI/lib"; export LIBRARY_PATH; LOADEDMODULES="intel/2017.4:impi/2018.4:netcdf/4.2:hdf5/1.8.19:perl/5.26:mkl/2018.4:CDO/1.8.2:szip/2.1.1"; export LOADEDMODULES; MANPATH="/apps/PERL/5.26/man:/apps/INTEL/2018.4.057/impi/2018.4.274/man:/apps/INTEL/2017.4/man/common:/apps/modules/LMOD/7.5/lmod/lmod/share/man:/usr/local/man:/usr/share/man"; export MANPATH; MKLROOT="/apps/INTEL/2018.4.057/mkl"; export MKLROOT; MKL_HOME="/apps/INTEL/2018.4.057/mkl"; export MKL_HOME; MODULEPATH="/apps/modules/modulefiles/applications:/apps/modules/modulefiles/compilers:/apps/modules/modulefiles/environment:/apps/modules/modulefiles/libraries:/apps/modules/modulefiles/tools:/apps/modules/modulefiles/prace"; export MODULEPATH; MPI="impi"; export MPI; MPICH_CC="icc"; export MPICH_CC; MPICH_CXX="icpc"; export MPICH_CXX; MPICH_F77="ifort"; export MPICH_F77; MPICH_F90="ifort"; export MPICH_F90; MPICH_FC="ifort"; export MPICH_FC; MPI_ROOT="/apps/INTEL/2018.4.057/impi/2018.4.274"; export MPI_ROOT; MPI_V="2018.4.057"; export MPI_V; MP_COMPILER="intel"; export MP_COMPILER; NETCDF="/apps/NETCDF/4.2/INTEL/IMPI"; export NETCDF; NETCDF_HOME="/apps/NETCDF/4.2/INTEL/IMPI"; export NETCDF_HOME; NETCDF_INC="/apps/NETCDF/4.2/INTEL/IMPI/include"; export NETCDF_INC; NETCDF_LIB="/apps/NETCDF/4.2/INTEL/IMPI/lib"; export NETCDF_LIB; NETCDF_PREFIX="/apps/NETCDF/4.2/INTEL/IMPI"; export NETCDF_PREFIX; NETCDF_ROOT="/apps/NETCDF/4.2/INTEL/IMPI"; export NETCDF_ROOT; OMPI_CC="icc"; export OMPI_CC; OMPI_CXX="icpc"; export OMPI_CXX; OMPI_F77="ifort"; export OMPI_F77; OMPI_F90="ifort"; export OMPI_F90; OMPI_FC="ifort"; export OMPI_FC; PATH="/apps/CDO/1.8.2/INTEL/IMPI/bin:/apps/PERL/5.26/bin:/apps/HDF5/1.8.19/INTEL/IMPI/bin:/apps/NETCDF/4.2/INTEL/IMPI/bin:/apps/INTEL/2018.4.057/impi/2018.4.274/bin64:/apps/INTEL/2017.4/bin:/usr/local/bin:/usr/bin:/bin:/usr/games:/usr/lpp/mmfs/bin"; export PATH; PERL5LIB="/apps/PERL/5.26/lib/perl5/site_perl/5.26.0:/apps/PERL/5.26/lib/perl5/5.26.0:/apps/PERL/5.26/lib/perl5/site_perl/5.26.0/x86_64-linux"; export PERL5LIB; _LMFILES_="/apps/modules/modulefiles/compilers/intel/2017.4:/apps/modules/modulefiles/environment/impi/2018.4:/apps/modules/modulefiles/libraries/netcdf/4.2:/apps/modules/modulefiles/libraries/hdf5/1.8.19:/apps/modules/modulefiles/tools/perl/5.26:/apps/modules/modulefiles/libraries/mkl/2018.4:/apps/modules/modulefiles/applications/CDO/1.8.2:/apps/modules/modulefiles/libraries/szip/2.1.1"; export _LMFILES_; _ModuleTable001_="X01vZHVsZVRhYmxlXz17WyJNVHZlcnNpb24iXT0zLFsiY19yZWJ1aWxkVGltZSJdPWZhbHNlLFsiY19zaG9ydFRpbWUiXT1mYWxzZSxkZXB0aFQ9e30sZmFtaWx5PXt9LG1UPXtDRE89e1siZm4iXT0iL2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9hcHBsaWNhdGlvbnMvQ0RPLzEuOC4yIixbImZ1bGxOYW1lIl09IkNETy8xLjguMiIsWyJsb2FkT3JkZXIiXT03LHByb3BUPXt9LFsic3RhdHVzIl09ImFjdGl2ZSIsWyJ1c2VyTmFtZSJdPSJDRE8vMS44LjIiLH0saGRmNT17WyJmbiJdPSIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2xpYnJhcmllcy9oZGY1LzEuOC4xOSIsWyJmdWxsTmFtZSJdPSJoZGY1LzEuOC4xOSIsWyJsb2FkT3JkZXIiXT00LHByb3BUPXt9LFsic3RhdHVz"; export _ModuleTable001_; _ModuleTable002_="Il09ImFjdGl2ZSIsWyJ1c2VyTmFtZSJdPSJoZGY1LzEuOC4xOSIsfSxpbXBpPXtbImZuIl09Ii9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvZW52aXJvbm1lbnQvaW1waS8yMDE4LjQiLFsiZnVsbE5hbWUiXT0iaW1waS8yMDE4LjQiLFsibG9hZE9yZGVyIl09Mixwcm9wVD17fSxbInN0YXR1cyJdPSJhY3RpdmUiLFsidXNlck5hbWUiXT0iaW1waS8yMDE4LjQiLH0saW50ZWw9e1siZm4iXT0iL2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9jb21waWxlcnMvaW50ZWwvMjAxNy40IixbImZ1bGxOYW1lIl09ImludGVsLzIwMTcuNCIsWyJsb2FkT3JkZXIiXT0xLHByb3BUPXt9LFsic3RhdHVzIl09ImFjdGl2ZSIsWyJ1c2VyTmFtZSJdPSJpbnRlbC8yMDE3LjQiLH0sbWtsPXtbImZu"; export _ModuleTable002_; _ModuleTable003_="Il09Ii9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvbGlicmFyaWVzL21rbC8yMDE4LjQiLFsiZnVsbE5hbWUiXT0ibWtsLzIwMTguNCIsWyJsb2FkT3JkZXIiXT02LHByb3BUPXt9LFsic3RhdHVzIl09ImFjdGl2ZSIsWyJ1c2VyTmFtZSJdPSJta2wvMjAxOC40Iix9LG5ldGNkZj17WyJmbiJdPSIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2xpYnJhcmllcy9uZXRjZGYvNC4yIixbImZ1bGxOYW1lIl09Im5ldGNkZi80LjIiLFsibG9hZE9yZGVyIl09Myxwcm9wVD17fSxbInN0YXR1cyJdPSJhY3RpdmUiLFsidXNlck5hbWUiXT0ibmV0Y2RmLzQuMiIsfSxwZXJsPXtbImZuIl09Ii9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvdG9vbHMvcGVybC81LjI2IixbImZ1bGxOYW1lIl09"; export _ModuleTable003_; _ModuleTable004_="InBlcmwvNS4yNiIsWyJsb2FkT3JkZXIiXT01LHByb3BUPXt9LFsic3RhdHVzIl09ImFjdGl2ZSIsWyJ1c2VyTmFtZSJdPSJwZXJsLzUuMjYiLH0sc3ppcD17WyJmbiJdPSIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2xpYnJhcmllcy9zemlwLzIuMS4xIixbImZ1bGxOYW1lIl09InN6aXAvMi4xLjEiLFsibG9hZE9yZGVyIl09OCxwcm9wVD17fSxbInN0YXR1cyJdPSJhY3RpdmUiLFsidXNlck5hbWUiXT0ic3ppcC8yLjEuMSIsfSx9LG1wYXRoQT17Ii9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvYXBwbGljYXRpb25zIiwiL2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9jb21waWxlcnMiLCIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2Vudmlyb25tZW50IiwiL2FwcHMvbW9k"; export _ModuleTable004_; _ModuleTable005_="dWxlcy9tb2R1bGVmaWxlcy9saWJyYXJpZXMiLCIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL3Rvb2xzIiwiL2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9wcmFjZSIsfSxbInN5c3RlbUJhc2VNUEFUSCJdPSIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2FwcGxpY2F0aW9uczovYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2NvbXBpbGVyczovYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2Vudmlyb25tZW50Oi9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvbGlicmFyaWVzOi9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvdG9vbHM6L2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9wcmFjZSIsfQ=="; export _ModuleTable005_; _ModuleTable_Sz_="5"; export _ModuleTable_Sz_;
++ CC=icc
++ export CC
++ CFLAGS='-I/apps/SZIP/2.1.1/INTEL/include -I/apps/HDF5/1.8.19/INTEL/IMPI/include'
++ export CFLAGS
++ CPLUS_INCLUDE_PATH=/apps/HDF5/1.8.19/INTEL/IMPI/include
++ export CPLUS_INCLUDE_PATH
++ CXX=icpc
++ export CXX
++ CXXFLAGS='-I/apps/SZIP/2.1.1/INTEL/include -I/apps/HDF5/1.8.19/INTEL/IMPI/include'
++ export CXXFLAGS
++ C_INCLUDE_PATH=/apps/SZIP/2.1.1/INTEL/include:/apps/INTEL/2018.4.057/mkl/include:/apps/HDF5/1.8.19/INTEL/IMPI/include
++ export C_INCLUDE_PATH
++ F77=ifort
++ export F77
++ F90=ifort
++ export F90
++ F95=ifort
++ export F95
++ FC=ifort
++ export FC
++ HDF5_DIR=/apps/HDF5/1.8.19/INTEL/IMPI
++ export HDF5_DIR
++ INTEL_HOME=/apps/INTEL/2017.4
++ export INTEL_HOME
++ INTEL_INC=/apps/INTEL/2017.4/include
++ export INTEL_INC
++ INTEL_VERSION=2017.4
++ export INTEL_VERSION
++ I_MPI_CC=icc
++ export I_MPI_CC
++ I_MPI_CXX=icpc
++ export I_MPI_CXX
++ I_MPI_F77=ifort
++ export I_MPI_F77
++ I_MPI_F90=ifort
++ export I_MPI_F90
++ I_MPI_ROOT=/apps/INTEL/2018.4.057/impi/2018.4.274
++ export I_MPI_ROOT
++ LDFLAGS='-L/apps/SZIP/2.1.1/INTEL/lib -lsz -L/apps/HDF5/1.8.19/INTEL/IMPI/lib -lhdf5 -lhdf5_hl'
++ export LDFLAGS
++ LD_LIBRARY_PATH=/apps/SZIP/2.1.1/INTEL/lib:/apps/INTEL/2018.4.057/mkl/lib/intel64:/apps/HDF5/1.8.19/INTEL/IMPI/lib:/apps/NETCDF/4.2/INTEL/IMPI/lib:/apps/INTEL/2018.4.057/impi/2018.4.274/lib64:/apps/INTEL/2017.4/lib/intel64
++ export LD_LIBRARY_PATH
++ LIBRARY_PATH=/apps/HDF5/1.8.19/INTEL/IMPI/lib
++ export LIBRARY_PATH
++ LOADEDMODULES=intel/2017.4:impi/2018.4:netcdf/4.2:hdf5/1.8.19:perl/5.26:mkl/2018.4:CDO/1.8.2:szip/2.1.1
++ export LOADEDMODULES
++ MANPATH=/apps/PERL/5.26/man:/apps/INTEL/2018.4.057/impi/2018.4.274/man:/apps/INTEL/2017.4/man/common:/apps/modules/LMOD/7.5/lmod/lmod/share/man:/usr/local/man:/usr/share/man
++ export MANPATH
++ MKLROOT=/apps/INTEL/2018.4.057/mkl
++ export MKLROOT
++ MKL_HOME=/apps/INTEL/2018.4.057/mkl
++ export MKL_HOME
++ MODULEPATH=/apps/modules/modulefiles/applications:/apps/modules/modulefiles/compilers:/apps/modules/modulefiles/environment:/apps/modules/modulefiles/libraries:/apps/modules/modulefiles/tools:/apps/modules/modulefiles/prace
++ export MODULEPATH
++ MPI=impi
++ export MPI
++ MPICH_CC=icc
++ export MPICH_CC
++ MPICH_CXX=icpc
++ export MPICH_CXX
++ MPICH_F77=ifort
++ export MPICH_F77
++ MPICH_F90=ifort
++ export MPICH_F90
++ MPICH_FC=ifort
++ export MPICH_FC
++ MPI_ROOT=/apps/INTEL/2018.4.057/impi/2018.4.274
++ export MPI_ROOT
++ MPI_V=2018.4.057
++ export MPI_V
++ MP_COMPILER=intel
++ export MP_COMPILER
++ NETCDF=/apps/NETCDF/4.2/INTEL/IMPI
++ export NETCDF
++ NETCDF_HOME=/apps/NETCDF/4.2/INTEL/IMPI
++ export NETCDF_HOME
++ NETCDF_INC=/apps/NETCDF/4.2/INTEL/IMPI/include
++ export NETCDF_INC
++ NETCDF_LIB=/apps/NETCDF/4.2/INTEL/IMPI/lib
++ export NETCDF_LIB
++ NETCDF_PREFIX=/apps/NETCDF/4.2/INTEL/IMPI
++ export NETCDF_PREFIX
++ NETCDF_ROOT=/apps/NETCDF/4.2/INTEL/IMPI
++ export NETCDF_ROOT
++ OMPI_CC=icc
++ export OMPI_CC
++ OMPI_CXX=icpc
++ export OMPI_CXX
++ OMPI_F77=ifort
++ export OMPI_F77
++ OMPI_F90=ifort
++ export OMPI_F90
++ OMPI_FC=ifort
++ export OMPI_FC
++ PATH=/apps/CDO/1.8.2/INTEL/IMPI/bin:/apps/PERL/5.26/bin:/apps/HDF5/1.8.19/INTEL/IMPI/bin:/apps/NETCDF/4.2/INTEL/IMPI/bin:/apps/INTEL/2018.4.057/impi/2018.4.274/bin64:/apps/INTEL/2017.4/bin:/usr/local/bin:/usr/bin:/bin:/usr/games:/usr/lpp/mmfs/bin
++ export PATH
++ PERL5LIB=/apps/PERL/5.26/lib/perl5/site_perl/5.26.0:/apps/PERL/5.26/lib/perl5/5.26.0:/apps/PERL/5.26/lib/perl5/site_perl/5.26.0/x86_64-linux
++ export PERL5LIB
++ _LMFILES_=/apps/modules/modulefiles/compilers/intel/2017.4:/apps/modules/modulefiles/environment/impi/2018.4:/apps/modules/modulefiles/libraries/netcdf/4.2:/apps/modules/modulefiles/libraries/hdf5/1.8.19:/apps/modules/modulefiles/tools/perl/5.26:/apps/modules/modulefiles/libraries/mkl/2018.4:/apps/modules/modulefiles/applications/CDO/1.8.2:/apps/modules/modulefiles/libraries/szip/2.1.1
++ export _LMFILES_
++ _ModuleTable001_=X01vZHVsZVRhYmxlXz17WyJNVHZlcnNpb24iXT0zLFsiY19yZWJ1aWxkVGltZSJdPWZhbHNlLFsiY19zaG9ydFRpbWUiXT1mYWxzZSxkZXB0aFQ9e30sZmFtaWx5PXt9LG1UPXtDRE89e1siZm4iXT0iL2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9hcHBsaWNhdGlvbnMvQ0RPLzEuOC4yIixbImZ1bGxOYW1lIl09IkNETy8xLjguMiIsWyJsb2FkT3JkZXIiXT03LHByb3BUPXt9LFsic3RhdHVzIl09ImFjdGl2ZSIsWyJ1c2VyTmFtZSJdPSJDRE8vMS44LjIiLH0saGRmNT17WyJmbiJdPSIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2xpYnJhcmllcy9oZGY1LzEuOC4xOSIsWyJmdWxsTmFtZSJdPSJoZGY1LzEuOC4xOSIsWyJsb2FkT3JkZXIiXT00LHByb3BUPXt9LFsic3RhdHVz
++ export _ModuleTable001_
++ _ModuleTable002_=Il09ImFjdGl2ZSIsWyJ1c2VyTmFtZSJdPSJoZGY1LzEuOC4xOSIsfSxpbXBpPXtbImZuIl09Ii9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvZW52aXJvbm1lbnQvaW1waS8yMDE4LjQiLFsiZnVsbE5hbWUiXT0iaW1waS8yMDE4LjQiLFsibG9hZE9yZGVyIl09Mixwcm9wVD17fSxbInN0YXR1cyJdPSJhY3RpdmUiLFsidXNlck5hbWUiXT0iaW1waS8yMDE4LjQiLH0saW50ZWw9e1siZm4iXT0iL2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9jb21waWxlcnMvaW50ZWwvMjAxNy40IixbImZ1bGxOYW1lIl09ImludGVsLzIwMTcuNCIsWyJsb2FkT3JkZXIiXT0xLHByb3BUPXt9LFsic3RhdHVzIl09ImFjdGl2ZSIsWyJ1c2VyTmFtZSJdPSJpbnRlbC8yMDE3LjQiLH0sbWtsPXtbImZu
++ export _ModuleTable002_
++ _ModuleTable003_=Il09Ii9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvbGlicmFyaWVzL21rbC8yMDE4LjQiLFsiZnVsbE5hbWUiXT0ibWtsLzIwMTguNCIsWyJsb2FkT3JkZXIiXT02LHByb3BUPXt9LFsic3RhdHVzIl09ImFjdGl2ZSIsWyJ1c2VyTmFtZSJdPSJta2wvMjAxOC40Iix9LG5ldGNkZj17WyJmbiJdPSIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2xpYnJhcmllcy9uZXRjZGYvNC4yIixbImZ1bGxOYW1lIl09Im5ldGNkZi80LjIiLFsibG9hZE9yZGVyIl09Myxwcm9wVD17fSxbInN0YXR1cyJdPSJhY3RpdmUiLFsidXNlck5hbWUiXT0ibmV0Y2RmLzQuMiIsfSxwZXJsPXtbImZuIl09Ii9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvdG9vbHMvcGVybC81LjI2IixbImZ1bGxOYW1lIl09
++ export _ModuleTable003_
++ _ModuleTable004_=InBlcmwvNS4yNiIsWyJsb2FkT3JkZXIiXT01LHByb3BUPXt9LFsic3RhdHVzIl09ImFjdGl2ZSIsWyJ1c2VyTmFtZSJdPSJwZXJsLzUuMjYiLH0sc3ppcD17WyJmbiJdPSIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2xpYnJhcmllcy9zemlwLzIuMS4xIixbImZ1bGxOYW1lIl09InN6aXAvMi4xLjEiLFsibG9hZE9yZGVyIl09OCxwcm9wVD17fSxbInN0YXR1cyJdPSJhY3RpdmUiLFsidXNlck5hbWUiXT0ic3ppcC8yLjEuMSIsfSx9LG1wYXRoQT17Ii9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvYXBwbGljYXRpb25zIiwiL2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9jb21waWxlcnMiLCIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2Vudmlyb25tZW50IiwiL2FwcHMvbW9k
++ export _ModuleTable004_
++ _ModuleTable005_=dWxlcy9tb2R1bGVmaWxlcy9saWJyYXJpZXMiLCIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL3Rvb2xzIiwiL2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9wcmFjZSIsfSxbInN5c3RlbUJhc2VNUEFUSCJdPSIvYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2FwcGxpY2F0aW9uczovYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2NvbXBpbGVyczovYXBwcy9tb2R1bGVzL21vZHVsZWZpbGVzL2Vudmlyb25tZW50Oi9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvbGlicmFyaWVzOi9hcHBzL21vZHVsZXMvbW9kdWxlZmlsZXMvdG9vbHM6L2FwcHMvbW9kdWxlcy9tb2R1bGVmaWxlcy9wcmFjZSIsfQ==
++ export _ModuleTable005_
++ _ModuleTable_Sz_=5
++ export _ModuleTable_Sz_
${LMOD_SETTARG_CMD:-:} -s sh
++ : -s sh
+ eval
+ set -eu
+ '[' -n /apps/GRIB/1.14.0/INTEL/lib ']'
+ export LD_LIBRARY_PATH=/apps/SZIP/2.1.1/INTEL/lib:/apps/INTEL/2018.4.057/mkl/lib/intel64:/apps/HDF5/1.8.19/INTEL/IMPI/lib:/apps/NETCDF/4.2/INTEL/IMPI/lib:/apps/INTEL/2018.4.057/impi/2018.4.274/lib64:/apps/INTEL/2017.4/lib/intel64:/apps/GRIB/1.14.0/INTEL/lib
+ LD_LIBRARY_PATH=/apps/SZIP/2.1.1/INTEL/lib:/apps/INTEL/2018.4.057/mkl/lib/intel64:/apps/HDF5/1.8.19/INTEL/IMPI/lib:/apps/NETCDF/4.2/INTEL/IMPI/lib:/apps/INTEL/2018.4.057/impi/2018.4.274/lib64:/apps/INTEL/2017.4/lib/intel64:/apps/GRIB/1.14.0/INTEL/lib
echo "$use_machinefile" | tr '[:upper:]' '[:lower:]'
++ echo TRUE
++ tr '[:upper:]' '[:lower:]'
+ [[ true == true ]]
+ use_machinefile=true
+ ulimit -s unlimited
+ configure_impi
+ '[' -z 1 ']'
+ export I_MPI_DEBUG=5
+ I_MPI_DEBUG=5

# -----------------------------------------------------------------------------
# *** Time step settings
# -----------------------------------------------------------------------------
if has_config ifs
then
    case "${ifs_grid}" in

        T159L*) ifs_time_step_sec=3600 ;;
        T255L*) ifs_time_step_sec=2700 ;;
        T511L*) ifs_time_step_sec=900  ;;

        *)  error "Can't set time steps for unknown horizontal grid: ${ifs_grid}"
            ;;
    esac
fi
+ has_config ifs
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' ifs == any ']'
+ [[ ifs == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ ifs ]]
+ [[ ifs =~ ifs ]]
+ __f=1
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ ifs ]]
+ for __m in '$config'
+ [[ lim3 =~ ifs ]]
+ for __m in '$config'
+ [[ rnfmapper =~ ifs ]]
+ for __m in '$config'
+ [[ xios:detached =~ ifs ]]
+ for __m in '$config'
+ [[ oasis =~ ifs ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ ifs ]]
+ ((  __f  ))
+ return 0
+ case "${ifs_grid}" in
+ ifs_time_step_sec=2700

if has_config nemo
then
    case "${nem_grid}" in

        ORCA1L*)   nem_time_step_sec=2700; lim_time_step_sec=2700 ;;
        ORCA025L*) nem_time_step_sec=900 ; lim_time_step_sec=900  ;;

        *)  error "Can't set time steps for unknown horizontal grid: ${nem_grid}"
            ;;
    esac
fi
+ has_config nemo
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' nemo == any ']'
+ [[ nemo == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ nemo ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ nemo ]]
+ [[ nemo:start_from_restart:elpin =~ nemo ]]
+ __f=1
+ for __m in '$config'
+ [[ lim3 =~ nemo ]]
+ for __m in '$config'
+ [[ rnfmapper =~ nemo ]]
+ for __m in '$config'
+ [[ xios:detached =~ nemo ]]
+ for __m in '$config'
+ [[ oasis =~ nemo ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ nemo ]]
+ ((  __f  ))
+ return 0
+ case "${nem_grid}" in
+ nem_time_step_sec=2700
+ lim_time_step_sec=2700

if has_config ifs nemo
then
    case "${ifs_grid}--${nem_grid}" in

        T159L*--ORCA1L*)
            ifs_time_step_sec=3600; nem_time_step_sec=2700; lim_time_step_sec=2700; cpl_freq_atm_oce_sec=10800
            ;;
        T255L*--ORCA1L*)
            ifs_time_step_sec=2700; nem_time_step_sec=2700; lim_time_step_sec=2700; cpl_freq_atm_oce_sec=2700
            ;;
        T511L*--ORCA025L*)
            ifs_time_step_sec=900 ; nem_time_step_sec=900 ; lim_time_step_sec=900 ; cpl_freq_atm_oce_sec=2700
            ;;

        *)  error "Can't set time steps for unknown combination of horizontal grids: ${ifs_grid}-${nem_grid}"
            ;;
    esac
fi
+ has_config ifs nemo
+ ((  2  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' ifs == any ']'
+ [[ ifs == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ ifs ]]
+ [[ ifs =~ ifs ]]
+ __f=1
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ ifs ]]
+ for __m in '$config'
+ [[ lim3 =~ ifs ]]
+ for __m in '$config'
+ [[ rnfmapper =~ ifs ]]
+ for __m in '$config'
+ [[ xios:detached =~ ifs ]]
+ for __m in '$config'
+ [[ oasis =~ ifs ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ ifs ]]
+ ((  __f  ))
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ nemo ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ nemo ]]
+ [[ nemo:start_from_restart:elpin =~ nemo ]]
+ __f=1
+ for __m in '$config'
+ [[ lim3 =~ nemo ]]
+ for __m in '$config'
+ [[ rnfmapper =~ nemo ]]
+ for __m in '$config'
+ [[ xios:detached =~ nemo ]]
+ for __m in '$config'
+ [[ oasis =~ nemo ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ nemo ]]
+ ((  __f  ))
+ return 0
+ case "${ifs_grid}--${nem_grid}" in
+ ifs_time_step_sec=2700
+ nem_time_step_sec=2700
+ lim_time_step_sec=2700
+ cpl_freq_atm_oce_sec=2700

# -----------------------------------------------------------------------------
# *** IFS configuration
# -----------------------------------------------------------------------------

ifs_version=36r4
+ ifs_version=36r4

ifs_di_freq=$(( 24 * 3600 / ifs_time_step_sec ))
+ ifs_di_freq=32
ifs_ddh_freq=$(( 120 * 3600 / ifs_time_step_sec ))
+ ifs_ddh_freq=160

export ifs_res_hor=$(echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\1:')
echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\1:'
++ sed 's:T\([0-9]\+\)L\([0-9]\+\):\1:'
++ echo T255L91
+ export ifs_res_hor=255
+ ifs_res_hor=255
ifs_res_ver=$(echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\2:')
echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\2:'
++ echo T255L91
++ sed 's:T\([0-9]\+\)L\([0-9]\+\):\2:'
+ ifs_res_ver=91

export ifs_numproc=480
+ export ifs_numproc=480
+ ifs_numproc=480

ifs_exe_file=${ecearth_src_dir}/ifs-${ifs_version}/bin/ifsmaster-${build_arch}
+ ifs_exe_file=/gpfs/scratch/bsc32/bsc32627/a3tb/auto-ecearth3/sources/sources/ifs-36r4/bin/ifsmaster-ecconf

ifs_lastout=false
+ ifs_lastout=false

ifs_cmip5=TRUE
+ ifs_cmip5=TRUE
ifs_cmip5_rcp=0
+ ifs_cmip5_rcp=0
export ifs_cmip_fixyear=0
+ export ifs_cmip_fixyear=0
+ ifs_cmip_fixyear=0
[ -z "${ifs_cmip_fixyear}" ] && ifs_cmip_fixyear=0
+ '[' -z 0 ']'
export ifs_cmip_fixyear_ch4=0
+ export ifs_cmip_fixyear_ch4=0
+ ifs_cmip_fixyear_ch4=0
[ -z "${ifs_cmip_fixyear_ch4}" ] && ifs_cmip_fixyear_ch4=0
+ '[' -z 0 ']'

# Repeat trap from ifs/suecrad.F90 for early catch
if ! has_config tm5:ch4fb && (( $ifs_cmip_fixyear != $ifs_cmip_fixyear_ch4 ))
then
    error 'CH4 in IFS is not provided by TM5, NCMIPFIXYR_CH4 should be set equal to NCMIPFIXYR'
fi
+ has_config tm5:ch4fb
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' tm5:ch4fb == any ']'
+ [[ tm5:ch4fb == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ tm5 ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ tm5 ]]
+ for __m in '$config'
+ [[ lim3 =~ tm5 ]]
+ for __m in '$config'
+ [[ rnfmapper =~ tm5 ]]
+ for __m in '$config'
+ [[ xios:detached =~ tm5 ]]
+ for __m in '$config'
+ [[ oasis =~ tm5 ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ tm5 ]]
+ ((  __f  ))
+ return 1
+ ((  0 != 0  ))

ifs_cmip6=TRUE
+ ifs_cmip6=TRUE
ifs_mac2sp=TRUE
+ ifs_mac2sp=TRUE
ifs_cmip6piaer=TRUE
+ ifs_cmip6piaer=TRUE
ifs_cmip6_scenario=historical
+ ifs_cmip6_scenario=historical

# enable optional COVID-19 scenarios, requires ifs_cmip6_scenario=SSP2-4.5
ifs_covid19=FALSE
+ ifs_covid19=FALSE
# choose one scenario : Base TwoYearBlip ModerateGreen StrongGreen FossilFuel
ifs_covid19scen=Base
+ ifs_covid19scen=Base
# basic sanity checks when covid19 is activated (SSP scenario and no support for LPJ-GUESS, PISCES nor TM5)
if [ ${ifs_covid19} == TRUE ] ; then
    [ ${ifs_cmip6_scenario} != SSP2-4.5 ] && info "with ifs_covid19=TRUE IFS uses ifs_cmip6_scenario=SSP2-4.5 not ${ifs_cmip6_scenario}"
    has_config any lpjg pisces tm5 && error "ifs_covid19=TRUE is not supported with LPJ-GUESS, PISCES nor TM5"
fi
+ '[' FALSE == TRUE ']'

lcmip6_strataer_simp=FALSE
+ lcmip6_strataer_simp=FALSE
lcmip6_strataer_full=TRUE
+ lcmip6_strataer_full=TRUE
lcmip6_strataer_bckgd=FALSE
+ lcmip6_strataer_bckgd=FALSE

export ifs_A4xCO2=FALSE
+ export ifs_A4xCO2=FALSE
+ ifs_A4xCO2=FALSE
export ifs_1PCTCO2=FALSE
+ export ifs_1PCTCO2=FALSE
+ ifs_1PCTCO2=FALSE
export bgc_1PCTCO2=FALSE
+ export bgc_1PCTCO2=FALSE
+ bgc_1PCTCO2=FALSE

# Time-varying orbital forcing (Qiong Zhang, SU-2013-09)
# https://dev.ec-earth.org/projects/ecearth3/wiki/Orbital_forcing_in_EC-Earth_3
#
#   ifs_orb_switch=false, no orbital calculations applied
#   ifs_orb_switch=true, use orbital calculations according to ifs_orb_mode
#   ifs_orb_mode="fixed_year", or "variable_year", or "fixed_parameters"
#     fixed_year: calculate the orbital parameters at ifs_orb_iyear, e.g.,1850
#     variable_year: calculate orbital parameters annually start from ifs_orb_iyear
#     fixed_parameters: prescribe orbital parameters for given year
case "${ifs_grid}" in
    T159*) ifs_orb_switch=true ;;
    *)     ifs_orb_switch=false ;;
esac
+ case "${ifs_grid}" in
+ ifs_orb_switch=false
ifs_orb_mode="variable_year"
+ ifs_orb_mode=variable_year
ifs_orb_iyear=$(date -u -d "${run_start_date}" +%Y)
date -u -d "${run_start_date}" +%Y
++ date -u -d 19930101 +%Y
+ ifs_orb_iyear=1993

# Relaxation of soil moisture (Wilhelm May, LU; October 2017)
#  
# LRXSM: Parameter indicating the levels to be nudged 
#
#   LRXSM =  0: no nudging 
#   LRXSM = 12: 4xdaily data and 3 levels (excluding level 1)      
#   LRXSM = 13: 4xdaily data and 4 levels
#
# LRXSMTx: time scale of the relaxation for level X (in hours)
#  
#   LRXSMTx =   0: actual values relpaced by external ones 
#   LRXSMTx =  24: 1 day
#   LRXSMTx = 120: 5 days
#
# LRXSMS: indicates when the relaxation is done 
#
#   LRXSMS = 0: before the time step
#   LRXSMS = 1:  after the time step
#
has_config soilnudg && ifs_lrxsm=13 || ifs_lrxsm=0
+ has_config soilnudg
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' soilnudg == any ']'
+ [[ soilnudg == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ soilnudg ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ soilnudg ]]
+ for __m in '$config'
+ [[ lim3 =~ soilnudg ]]
+ for __m in '$config'
+ [[ rnfmapper =~ soilnudg ]]
+ for __m in '$config'
+ [[ xios:detached =~ soilnudg ]]
+ for __m in '$config'
+ [[ oasis =~ soilnudg ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ soilnudg ]]
+ ((  __f  ))
+ return 1
+ ifs_lrxsm=0
ifs_lrxsmt1=96
+ ifs_lrxsmt1=96
ifs_lrxsmt2=72
+ ifs_lrxsmt2=72
ifs_lrxsmt3=48
+ ifs_lrxsmt3=48
ifs_lrxsmt4=24
+ ifs_lrxsmt4=24
ifs_lrxsms=1
+ ifs_lrxsms=1

# IFS tuning parameters
variant=
+ variant=
has_config tm5:chem && variant=-AerChem
+ has_config tm5:chem
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' tm5:chem == any ']'
+ [[ tm5:chem == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ tm5 ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ tm5 ]]
+ for __m in '$config'
+ [[ lim3 =~ tm5 ]]
+ for __m in '$config'
+ [[ rnfmapper =~ tm5 ]]
+ for __m in '$config'
+ [[ xios:detached =~ tm5 ]]
+ for __m in '$config'
+ [[ oasis =~ tm5 ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ tm5 ]]
+ ((  __f  ))
+ return 1
ifs_tuning_parameter_file=${ctrl_file_dir}/ifs-tuning-parameters-${ifs_grid}${variant}.sh
+ ifs_tuning_parameter_file=/gpfs/scratch/bsc32/bsc32627/a3tb/ctrl/ifs-tuning-parameters-T255L91.sh
if [ -f ${ifs_tuning_parameter_file} ]
then
    source ${ifs_tuning_parameter_file}
else
    error "Sorry, ${ifs_tuning_parameter_file} not found, exiting."
fi
+ '[' -f /gpfs/scratch/bsc32/bsc32627/a3tb/ctrl/ifs-tuning-parameters-T255L91.sh ']'
+ source /gpfs/scratch/bsc32/bsc32627/a3tb/ctrl/ifs-tuning-parameters-T255L91.sh
# *** IFS tuning (the commented values are EC-Earth 3.2beta and IFS cy36r4)
# *** for resolution T255L91
# Default based on the v16 LPJG vegetation dataset discussed in #449-197 (jcn4)
# The default values are WITH 2nd indirect effect NCLOUDACT=2, NAERCLD=9
# jvg6 is the best alternative configuration found in #449 with NCLOUDACT=0
#                    jvg6     ECE32b     IFS cy36r4
RPRCON=1.34E-3      # 1.52e-4  1.2E-3     1.4E-3
++ RPRCON=1.34E-3
RVICE=0.137         # 0.126    0.13       0.15
++ RVICE=0.137
RLCRITSNOW=4.0E-5   # 4.2e-5   3.0E-5     5.0E-5
++ RLCRITSNOW=4.0E-5
RSNOWLIN2=0.035     # 0.035    0.035      0.025
++ RSNOWLIN2=0.035
ENTRORG=1.70E-4     # 1.58e-4  1.5E-4     1.8E-4
++ ENTRORG=1.70E-4
DETRPEN=0.75E-4     # 0.75e-4  0.75E-4    0.75E-4
++ DETRPEN=0.75E-4
ENTRDD=3.0E-4       # 3.5e-4   3.0E-4     2.0E-4
++ ENTRDD=3.0E-4
RMFDEPS=0.3         # 0.27     0.3        0.35
++ RMFDEPS=0.3
RCLDIFF=3.E-6       # 3.6e-6   3.E-6
++ RCLDIFF=3.E-6
RCLDIFFC=5.0        # 5.0
++ RCLDIFFC=5.0
RLCRIT_UPHYS=0.875e-5
++ RLCRIT_UPHYS=0.875e-5


# Select source of vegetation data:
#  ifs       climatology from IFS
#  era20c    vegetation from an off-line LPJ-Guess run forced with ERA20C
#            (currently available only for T255 and T159)
#  cmip6     vegetation from an EC-Earth3-Veg (interactive LPJ-Guess) run 
#            (currently available only for T255)
#  custom_exp vegetation from any EC-Earth3-Veg run exp
#            (must contain same variables as era20c & cmip6 and located in icmcl_exp folder)
#  none      don't create an ICMCL file with vegetation data (this is set
#            automatically if LPJG is used with feedback)
#
# set above in AS runtime

has_config lpjg:fdbck && ifs_veg_source="none"
+ has_config lpjg:fdbck
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' lpjg:fdbck == any ']'
+ [[ lpjg:fdbck == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ lpjg ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ lpjg ]]
+ for __m in '$config'
+ [[ lim3 =~ lpjg ]]
+ for __m in '$config'
+ [[ rnfmapper =~ lpjg ]]
+ for __m in '$config'
+ [[ xios:detached =~ lpjg ]]
+ for __m in '$config'
+ [[ oasis =~ lpjg ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ lpjg ]]
+ ((  __f  ))
+ return 1

case ${ifs_veg_source} in
"ifs" )
    # Use Lambert-Beer to compute effective vegetation cover
    n_compute_eff_veg_fraction=2
    ;;
"era20c" )
    # LPJG vegetation is provided as effective cover
    # Don't use Lambert-Beer
    n_compute_eff_veg_fraction=0

    case "${ifs_grid}" in
        T159L*) veg_version=v29 ;;
        T255L*) veg_version=v16 ;;
        *)  error "Vegetation from off-line LPJ-Guess not available for ${ifs_grid}" ;;
    esac    
    ;;
"cmip6" )
    # LPJG vegetation is provided as effective cover
    # Don't use Lambert-Beer
    n_compute_eff_veg_fraction=0

    case "${ifs_grid}" in
        T255L*) veg_version=v32 ;;
        *)  error "Vegetation from CMIP6 EC-Earth3-Veg not available for ${ifs_grid}" ;;
    esac    
    ;;
"custom_"* )
    # LPJG vegetation is provided as effective cover
    # Don't use Lambert-Beer
    n_compute_eff_veg_fraction=0

    veg_version=${ifs_veg_source:7}
    if [ ! -d ${ini_data_dir}/ifs/${ifs_grid}/icmcl_${veg_version} ]
    then
        error "requested IFS_VEG_SOURCE = ${ifs_veg_source} but not found in ${ini_data_dir}/ifs/${ifs_grid}/icmcl_${veg_version}"
    fi
    ;;
"none" )
    # LPJG with feedback
    n_compute_eff_veg_fraction=0
    ! has_config lpjg:fdbck && error "IFS requires an offline source of vegetation"
    ;;
* )
    error "Vegetation from ${ifs_veg_source} not implemented"
    ;;
esac
+ case ${ifs_veg_source} in
+ n_compute_eff_veg_fraction=0
+ case "${ifs_grid}" in
+ veg_version=v32

# use DMI land ice physics and varying snow albedo
case "${ifs_grid}" in
    T159*) ifs_landice=true ;;
    *)     ifs_landice=false ;;
esac
+ case "${ifs_grid}" in
+ ifs_landice=false

# -----------------------------------------------------------------------------
# *** NEMO/LIM configuration
# -----------------------------------------------------------------------------

# This is only needed if the experiment is started from an existing set of NEMO
# restart files
nem_restart_file_path=${ini_data_dir}
+ nem_restart_file_path=/gpfs/scratch/bsc32/bsc32627/a3tb/19930101/fc01/inidata

nem_restart_offset=0
+ nem_restart_offset=0

nem_res_hor=$(echo ${nem_grid} | sed 's:ORCA\([0-9]\+\)L[0-9]\+:\1:')
echo ${nem_grid} | sed 's:ORCA\([0-9]\+\)L[0-9]\+:\1:'
++ echo ORCA1L75
++ sed 's:ORCA\([0-9]\+\)L[0-9]\+:\1:'
+ nem_res_hor=1

nem_config=${nem_grid}
+ nem_config=ORCA1L75
has_config lim3           && nem_config=${nem_config}_LIM3
+ has_config lim3
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' lim3 == any ']'
+ [[ lim3 == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ lim3 ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ lim3 ]]
+ for __m in '$config'
+ [[ lim3 =~ lim3 ]]
+ [[ lim3 =~ lim3 ]]
+ __f=1
+ for __m in '$config'
+ [[ rnfmapper =~ lim3 ]]
+ for __m in '$config'
+ [[ xios:detached =~ lim3 ]]
+ for __m in '$config'
+ [[ oasis =~ lim3 ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ lim3 ]]
+ ((  __f  ))
+ return 0
+ nem_config=ORCA1L75_LIM3

if has_config pisces tm5:co2
then
    nem_config=${nem_config}_CarbonCycle
elif has_config pisces
then
    nem_config=${nem_config}_PISCES
fi
+ has_config pisces tm5:co2
+ ((  2  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' pisces == any ']'
+ [[ pisces == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ pisces ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ pisces ]]
+ for __m in '$config'
+ [[ lim3 =~ pisces ]]
+ for __m in '$config'
+ [[ rnfmapper =~ pisces ]]
+ for __m in '$config'
+ [[ xios:detached =~ pisces ]]
+ for __m in '$config'
+ [[ oasis =~ pisces ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ pisces ]]
+ ((  __f  ))
+ return 1
+ has_config pisces
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' pisces == any ']'
+ [[ pisces == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ pisces ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ pisces ]]
+ for __m in '$config'
+ [[ lim3 =~ pisces ]]
+ for __m in '$config'
+ [[ rnfmapper =~ pisces ]]
+ for __m in '$config'
+ [[ xios:detached =~ pisces ]]
+ for __m in '$config'
+ [[ oasis =~ pisces ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ pisces ]]
+ ((  __f  ))
+ return 1

# TODO - nemo standalone configs are not accounted for in this script, but this would set the required nem_config
! has_config ifs && nem_config=${nem_config}_standalone
+ has_config ifs
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' ifs == any ']'
+ [[ ifs == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ ifs ]]
+ [[ ifs =~ ifs ]]
+ __f=1
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ ifs ]]
+ for __m in '$config'
+ [[ lim3 =~ ifs ]]
+ for __m in '$config'
+ [[ rnfmapper =~ ifs ]]
+ for __m in '$config'
+ [[ xios:detached =~ ifs ]]
+ for __m in '$config'
+ [[ oasis =~ ifs ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ ifs ]]
+ ((  __f  ))
+ return 0

nem_exe_file=${ecearth_src_dir}/nemo-3.6/CONFIG/${nem_config}/BLD/bin/nemo.exe
+ nem_exe_file=/gpfs/scratch/bsc32/bsc32627/a3tb/auto-ecearth3/sources/sources/nemo-3.6/CONFIG/ORCA1L75_LIM3/BLD/bin/nemo.exe

nem_numproc=240
+ nem_numproc=240

# Thermal conductivity of snow, see comment in ctrl/namelist.lim3.ref.sh
case "${ifs_grid}" in
    T159L* ) nem_rn_cdsn=0.25 ;;
    * )      nem_rn_cdsn=0.27 ;;
esac
+ case "${ifs_grid}" in
+ nem_rn_cdsn=0.27

# -----------------------------------------------------------------------------
# *** Runoff mapper configuration
# -----------------------------------------------------------------------------

rnf_exe_file=${ecearth_src_dir}/runoff-mapper/bin/runoff-mapper.exe
+ rnf_exe_file=/gpfs/scratch/bsc32/bsc32627/a3tb/auto-ecearth3/sources/sources/runoff-mapper/bin/runoff-mapper.exe
rnf_numproc=1
+ rnf_numproc=1

# -----------------------------------------------------------------------------
# *** LPJ-GUESS configuration
# -----------------------------------------------------------------------------

lpjg_time_step_sec=86400
+ lpjg_time_step_sec=86400
lpjg_numproc=
+ lpjg_numproc=

has_config lpjg       && lpjg_on=1
+ has_config lpjg
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' lpjg == any ']'
+ [[ lpjg == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ lpjg ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ lpjg ]]
+ for __m in '$config'
+ [[ lim3 =~ lpjg ]]
+ for __m in '$config'
+ [[ rnfmapper =~ lpjg ]]
+ for __m in '$config'
+ [[ xios:detached =~ lpjg ]]
+ for __m in '$config'
+ [[ oasis =~ lpjg ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ lpjg ]]
+ ((  __f  ))
+ return 1
has_config lpjg:fdbck && lpjg_fdbck=1
+ has_config lpjg:fdbck
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' lpjg:fdbck == any ']'
+ [[ lpjg:fdbck == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ lpjg ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ lpjg ]]
+ for __m in '$config'
+ [[ lim3 =~ lpjg ]]
+ for __m in '$config'
+ [[ rnfmapper =~ lpjg ]]
+ for __m in '$config'
+ [[ xios:detached =~ lpjg ]]
+ for __m in '$config'
+ [[ oasis =~ lpjg ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ lpjg ]]
+ ((  __f  ))
+ return 1
has_config tm5:co2    && lpjg_fdbck_tm5=1 || lpjg_fdbck_tm5=0
+ has_config tm5:co2
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' tm5:co2 == any ']'
+ [[ tm5:co2 == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ tm5 ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ tm5 ]]
+ for __m in '$config'
+ [[ lim3 =~ tm5 ]]
+ for __m in '$config'
+ [[ rnfmapper =~ tm5 ]]
+ for __m in '$config'
+ [[ xios:detached =~ tm5 ]]
+ for __m in '$config'
+ [[ oasis =~ tm5 ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ tm5 ]]
+ ((  __f  ))
+ return 1
+ lpjg_fdbck_tm5=0

lpjg_fixNdepafter=-1
+ lpjg_fixNdepafter=-1
lpjg_fixLUafter=-1
+ lpjg_fixLUafter=-1
[ -z "${lpjg_fixNdepafter}" ] && lpjg_fixNdepafter=-1
+ '[' -z -1 ']'
[ -z "${lpjg_fixLUafter}" ] && lpjg_fixLUafter=-1
+ '[' -z -1 ']'
export lpjg_fixNdepafter lpjg_fixLUafter
+ export lpjg_fixNdepafter lpjg_fixLUafter

info '!!!! CMIP FIX YEAR SETTINGS:'
+ info '!!!! CMIP FIX YEAR SETTINGS:'
+ echo '*II* !!!! CMIP FIX YEAR SETTINGS:'
info "ifs_cmip_fixyear:  $ifs_cmip_fixyear"
+ info 'ifs_cmip_fixyear:  0'
+ echo '*II* ifs_cmip_fixyear:  0'
info "lpjg_fixNDepAfter: $lpjg_fixNdepafter"
+ info 'lpjg_fixNDepAfter: -1'
+ echo '*II* lpjg_fixNDepAfter: -1'
info "lpjg_fixLUAfter:   $lpjg_fixLUafter"
+ info 'lpjg_fixLUAfter:   -1'
+ echo '*II* lpjg_fixLUAfter:   -1'
info '!!!!'  
+ info '!!!!'
+ echo '*II* !!!!'

lpjg_res=T${ifs_res_hor}
+ lpjg_res=T255
lpjg_exe_file=${ecearth_src_dir}/lpjg/build/guess_${lpjg_res}
+ lpjg_exe_file=/gpfs/scratch/bsc32/bsc32627/a3tb/auto-ecearth3/sources/sources/lpjg/build/guess_T255

# -----------------------------------------------------------------------------
# *** AMIP-reader configuration
# -----------------------------------------------------------------------------

amip_exe_file=${ecearth_src_dir}/amip-forcing/bin/amip-forcing.exe
+ amip_exe_file=/gpfs/scratch/bsc32/bsc32627/a3tb/auto-ecearth3/sources/sources/amip-forcing/bin/amip-forcing.exe
amip_numproc=1
+ amip_numproc=1

# -----------------------------------------------------------------------------
# *** TM5 configuration
# -----------------------------------------------------------------------------

if $(has_config tm5)
then
    # With TM5, NPRTRV is set to 1 in the namelist. To avoid some out-of-bound
    # arrays in IFS, we must limit the number of cores for IFS
    if (( ifs_numproc > (ifs_res_hor+1) ))
    then
        error "too much cores requested for IFS, max is $((ifs_res_hor+1))"
    fi

    # TM5 settings
    has_config tm5:co2 && tmversion="co2" || tmversion="cb05"
    has_config tm5:co2 && export tm5_co2=1 || export tm5_co2=0
    export tm5_exch_nlevs=34
    tm5_time_step_sec=3600
    export tm5_numproc_x=1
    export tm5_numproc_y=45
    tm5_numproc=$(( tm5_numproc_x * tm5_numproc_y ))
    export tm5_emiss_fixyear=0

    # limited number of levels for feedback (aerosols, currently set to lmax_conv in TM5)
    case ${tm5_exch_nlevs} in
        34) export tm5_exch_nlevs_cutoff=23 ;;
        10) export tm5_exch_nlevs_cutoff=10 ;;
         4) export tm5_exch_nlevs_cutoff=4  ;;
         *) error "not supported number of levels for TM5"
    esac

    # executable
    tm5_exe_file=${ecearth_src_dir}/tm5mp/build-${tmversion}-ml${tm5_exch_nlevs}/appl-tm5-${tmversion}.x

    # path to initial conditions, modify as needed
    tm5_restart_file_path=${ini_data_dir}/tm5/restart/${tmversion}-ml${tm5_exch_nlevs}

    # fields sent back to IFS
    has_config tm5:o3fb   && tm5_to_ifs=O3 || tm5_to_ifs=
    has_config tm5:ch4fb  && tm5_to_ifs=${tm5_to_ifs},CH4
    has_config tm5:aerfb && tm5_to_ifs=${tm5_to_ifs},"\
N2,SU2,BC2,OC2,N3,SU3,BC3,OC3,SS3,DU3,\
N4,SU4,BC4,OC4,SS4,DU4,N5,BC5,OC5,N6,DU6,N7,DU7,\
NO3,MSA,\
AOD_01,AOD_02,AOD_03,AOD_04,AOD_05,AOD_06,AOD_07,AOD_08,AOD_09,AOD_10,AOD_11,AOD_12,AOD_13,AOD_14,\
SSA_01,SSA_02,SSA_03,SSA_04,SSA_05,SSA_06,SSA_07,SSA_08,SSA_09,SSA_10,SSA_11,SSA_12,SSA_13,SSA_14,\
ASF_01,ASF_02,ASF_03,ASF_04,ASF_05,ASF_06,ASF_07,ASF_08,ASF_09,ASF_10,ASF_11,ASF_12,ASF_13,ASF_14"
    has_config tm5:co2fb && tm5_to_ifs=${tm5_to_ifs},CO2

    export tm5_to_ifs=$(echo ${tm5_to_ifs} | sed "s/^,//")

    # coupled to LPJ-Guess and/or PISCES?
    has_config tm5:co2 lpjg   && export cpl_tm_guess=T  || export cpl_tm_guess=F
    has_config tm5:co2 pisces && export cpl_tm_pisces=T || export cpl_tm_pisces=F
fi
has_config tm5
++ has_config tm5
++ ((  1  ))
++ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
++ local __c
++ local __m
++ '[' tm5 == any ']'
++ [[ tm5 == \a\l\l ]]
++ local __f
++ for __c in '"$@"'
++ __f=0
++ for __m in '$config'
++ [[ ifs =~ tm5 ]]
++ for __m in '$config'
++ [[ nemo:start_from_restart:elpin =~ tm5 ]]
++ for __m in '$config'
++ [[ lim3 =~ tm5 ]]
++ for __m in '$config'
++ [[ rnfmapper =~ tm5 ]]
++ for __m in '$config'
++ [[ xios:detached =~ tm5 ]]
++ for __m in '$config'
++ [[ oasis =~ tm5 ]]
++ for __m in '$config'
++ [[ save_ic:end_leg =~ tm5 ]]
++ ((  __f  ))
++ return 1

# -----------------------------------------------------------------------------
# *** OASIS configuration
# -----------------------------------------------------------------------------

# Restart files for the coupling fields (note 8 character limit in OASIS)
#   rstas.nc : atmosphere single-category fields
#   rstam.nc : atmosphere multi-category fields
#   rstos.nc : ocean single-category fields
#   rstom.nc : ocean multi-category fields
oas_rst_ifs_nemo="rstas.nc rstos.nc"
+ oas_rst_ifs_nemo='rstas.nc rstos.nc'

oas_rst_ifs_lpjg="vegin.nc lpjgv.nc"
+ oas_rst_ifs_lpjg='vegin.nc lpjgv.nc'

# Met fields from IFS to TM (always required)
oas_rst_ifs_tm5="r_hum.nc r_g2d.nc r_udr.nc r_div.nc r_vor.nc \
                 r_ddr.nc r_tmp.nc r_dmf.nc r_s2d.nc r_umf.nc"
+ oas_rst_ifs_tm5='r_hum.nc r_g2d.nc r_udr.nc r_div.nc r_vor.nc                  r_ddr.nc r_tmp.nc r_dmf.nc r_s2d.nc r_umf.nc'

has_config tm5:chem && \
    oas_rst_ifs_tm5=${oas_rst_ifs_tm5}' r_cc_.nc r_clw.nc r_cco.nc r_ciw.nc r_ccu.nc'
+ has_config tm5:chem
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' tm5:chem == any ']'
+ [[ tm5:chem == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ tm5 ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ tm5 ]]
+ for __m in '$config'
+ [[ lim3 =~ tm5 ]]
+ for __m in '$config'
+ [[ rnfmapper =~ tm5 ]]
+ for __m in '$config'
+ [[ xios:detached =~ tm5 ]]
+ for __m in '$config'
+ [[ oasis =~ tm5 ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ tm5 ]]
+ ((  __f  ))
+ return 1

has_config tm5:o3fb || has_config tm5:ch4fb && oas_rst_ifs_tm5=$oas_rst_ifs_tm5' o3ch4.nc'
+ has_config tm5:o3fb
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' tm5:o3fb == any ']'
+ [[ tm5:o3fb == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ tm5 ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ tm5 ]]
+ for __m in '$config'
+ [[ lim3 =~ tm5 ]]
+ for __m in '$config'
+ [[ rnfmapper =~ tm5 ]]
+ for __m in '$config'
+ [[ xios:detached =~ tm5 ]]
+ for __m in '$config'
+ [[ oasis =~ tm5 ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ tm5 ]]
+ ((  __f  ))
+ return 1
+ has_config tm5:ch4fb
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' tm5:ch4fb == any ']'
+ [[ tm5:ch4fb == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ tm5 ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ tm5 ]]
+ for __m in '$config'
+ [[ lim3 =~ tm5 ]]
+ for __m in '$config'
+ [[ rnfmapper =~ tm5 ]]
+ for __m in '$config'
+ [[ xios:detached =~ tm5 ]]
+ for __m in '$config'
+ [[ oasis =~ tm5 ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ tm5 ]]
+ ((  __f  ))
+ return 1
has_config tm5:aerfb && oas_rst_ifs_tm5=$oas_rst_ifs_tm5' C???????'
+ has_config tm5:aerfb
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' tm5:aerfb == any ']'
+ [[ tm5:aerfb == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ tm5 ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ tm5 ]]
+ for __m in '$config'
+ [[ lim3 =~ tm5 ]]
+ for __m in '$config'
+ [[ rnfmapper =~ tm5 ]]
+ for __m in '$config'
+ [[ xios:detached =~ tm5 ]]
+ for __m in '$config'
+ [[ oasis =~ tm5 ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ tm5 ]]
+ ((  __f  ))
+ return 1

# C-cycle configuration
has_config tm5:co2 lpjg   && oas_rst_ifs_tm5=$oas_rst_ifs_tm5' l_co2.nc rlpjg.nc'
+ has_config tm5:co2 lpjg
+ ((  2  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' tm5:co2 == any ']'
+ [[ tm5:co2 == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ tm5 ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ tm5 ]]
+ for __m in '$config'
+ [[ lim3 =~ tm5 ]]
+ for __m in '$config'
+ [[ rnfmapper =~ tm5 ]]
+ for __m in '$config'
+ [[ xios:detached =~ tm5 ]]
+ for __m in '$config'
+ [[ oasis =~ tm5 ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ tm5 ]]
+ ((  __f  ))
+ return 1
has_config tm5:co2 pisces && oas_rst_ifs_tm5=$oas_rst_ifs_tm5' o_co2.nc pisce.nc'
+ has_config tm5:co2 pisces
+ ((  2  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' tm5:co2 == any ']'
+ [[ tm5:co2 == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ tm5 ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ tm5 ]]
+ for __m in '$config'
+ [[ lim3 =~ tm5 ]]
+ for __m in '$config'
+ [[ rnfmapper =~ tm5 ]]
+ for __m in '$config'
+ [[ xios:detached =~ tm5 ]]
+ for __m in '$config'
+ [[ oasis =~ tm5 ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ tm5 ]]
+ ((  __f  ))
+ return 1
has_config tm5:co2fb      && oas_rst_ifs_tm5=$oas_rst_ifs_tm5' co2mx.nc'
+ has_config tm5:co2fb
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' tm5:co2fb == any ']'
+ [[ tm5:co2fb == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ tm5 ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ tm5 ]]
+ for __m in '$config'
+ [[ lim3 =~ tm5 ]]
+ for __m in '$config'
+ [[ rnfmapper =~ tm5 ]]
+ for __m in '$config'
+ [[ xios:detached =~ tm5 ]]
+ for __m in '$config'
+ [[ oasis =~ tm5 ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ tm5 ]]
+ ((  __f  ))
+ return 1

# final list of files depends on the activated components - this is used in save_ic as well
#oas_rst_files="${oas_rst_ifs_nemo} ${oas_rst_ifs_tm5} vegin.nc lpjgv.nc"
oas_rst_files=""
+ oas_rst_files=
has_config ifs nemo && oas_rst_files+=" ${oas_rst_ifs_nemo}"
+ has_config ifs nemo
+ ((  2  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' ifs == any ']'
+ [[ ifs == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ ifs ]]
+ [[ ifs =~ ifs ]]
+ __f=1
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ ifs ]]
+ for __m in '$config'
+ [[ lim3 =~ ifs ]]
+ for __m in '$config'
+ [[ rnfmapper =~ ifs ]]
+ for __m in '$config'
+ [[ xios:detached =~ ifs ]]
+ for __m in '$config'
+ [[ oasis =~ ifs ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ ifs ]]
+ ((  __f  ))
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ nemo ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ nemo ]]
+ [[ nemo:start_from_restart:elpin =~ nemo ]]
+ __f=1
+ for __m in '$config'
+ [[ lim3 =~ nemo ]]
+ for __m in '$config'
+ [[ rnfmapper =~ nemo ]]
+ for __m in '$config'
+ [[ xios:detached =~ nemo ]]
+ for __m in '$config'
+ [[ oasis =~ nemo ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ nemo ]]
+ ((  __f  ))
+ return 0
+ oas_rst_files+=' rstas.nc rstos.nc'
has_config ifs lpjg && oas_rst_files+=" ${oas_rst_ifs_lpjg}"
+ has_config ifs lpjg
+ ((  2  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' ifs == any ']'
+ [[ ifs == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ ifs ]]
+ [[ ifs =~ ifs ]]
+ __f=1
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ ifs ]]
+ for __m in '$config'
+ [[ lim3 =~ ifs ]]
+ for __m in '$config'
+ [[ rnfmapper =~ ifs ]]
+ for __m in '$config'
+ [[ xios:detached =~ ifs ]]
+ for __m in '$config'
+ [[ oasis =~ ifs ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ ifs ]]
+ ((  __f  ))
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ lpjg ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ lpjg ]]
+ for __m in '$config'
+ [[ lim3 =~ lpjg ]]
+ for __m in '$config'
+ [[ rnfmapper =~ lpjg ]]
+ for __m in '$config'
+ [[ xios:detached =~ lpjg ]]
+ for __m in '$config'
+ [[ oasis =~ lpjg ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ lpjg ]]
+ ((  __f  ))
+ return 1
has_config ifs tm5 && oas_rst_files+=" ${oas_rst_ifs_tm5}"
+ has_config ifs tm5
+ ((  2  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' ifs == any ']'
+ [[ ifs == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ ifs ]]
+ [[ ifs =~ ifs ]]
+ __f=1
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ ifs ]]
+ for __m in '$config'
+ [[ lim3 =~ ifs ]]
+ for __m in '$config'
+ [[ rnfmapper =~ ifs ]]
+ for __m in '$config'
+ [[ xios:detached =~ ifs ]]
+ for __m in '$config'
+ [[ oasis =~ ifs ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ ifs ]]
+ ((  __f  ))
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ tm5 ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ tm5 ]]
+ for __m in '$config'
+ [[ lim3 =~ tm5 ]]
+ for __m in '$config'
+ [[ rnfmapper =~ tm5 ]]
+ for __m in '$config'
+ [[ xios:detached =~ tm5 ]]
+ for __m in '$config'
+ [[ oasis =~ tm5 ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ tm5 ]]
+ ((  __f  ))
+ return 1

# Decide whether the OASIS weight files for interpolation should be linked from
# the setup directory (true) or not (false). In the latter case, the weights
# are re-computed at the start of the run.
oas_link_weights=true
+ oas_link_weights=true

# Flux correction for runoff (not calving) sent from Oasis to ocean.
# 1.07945 is computed to compensate for a P-E=-0.016 mm/day (valid for std res)
case "${ifs_grid}" in
    T159L* ) has_config nemo && oas_mb_fluxcorr=1.08652 ;;
    * ) has_config nemo && oas_mb_fluxcorr=1.07945 ;;
esac
+ case "${ifs_grid}" in
+ has_config nemo
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' nemo == any ']'
+ [[ nemo == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ nemo ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ nemo ]]
+ [[ nemo:start_from_restart:elpin =~ nemo ]]
+ __f=1
+ for __m in '$config'
+ [[ lim3 =~ nemo ]]
+ for __m in '$config'
+ [[ rnfmapper =~ nemo ]]
+ for __m in '$config'
+ [[ xios:detached =~ nemo ]]
+ for __m in '$config'
+ [[ oasis =~ nemo ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ nemo ]]
+ ((  __f  ))
+ return 0
+ oas_mb_fluxcorr=1.07945

# -----------------------------------------------------------------------------
# *** XIOS configuration
# -----------------------------------------------------------------------------

xio_exe_file=${ecearth_src_dir}/xios-2.5/bin/xios_server.exe
+ xio_exe_file=/gpfs/scratch/bsc32/bsc32627/a3tb/auto-ecearth3/sources/sources/xios-2.5/bin/xios_server.exe

xio_numproc=47
+ xio_numproc=47

# -----------------------------------------------------------------------------
# *** Extra initial conditions saved during the run
# -----------------------------------------------------------------------------
if has_config save_ic
then
    source ./libsave_ic.sh
    declare -a save_ic_date save_ic_date1 save_ic_sec save_ic_day save_ic_ppt_file save_ic_nemo_ts
    oas_rst_files="${oas_rst_ifs_nemo} ${oas_rst_ifs_tm5} vegin.nc lpjgv.nc"
fi
+ has_config save_ic
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' save_ic == any ']'
+ [[ save_ic == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ save_ic ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ save_ic ]]
+ for __m in '$config'
+ [[ lim3 =~ save_ic ]]
+ for __m in '$config'
+ [[ rnfmapper =~ save_ic ]]
+ for __m in '$config'
+ [[ xios:detached =~ save_ic ]]
+ for __m in '$config'
+ [[ oasis =~ save_ic ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ save_ic ]]
+ [[ save_ic:end_leg =~ save_ic ]]
+ __f=1
+ ((  __f  ))
+ return 0
+ source ./libsave_ic.sh
#================================================================================
# save_ic functions - these are used to save EC-Earth states during a run,
# which can be used as Initial Conditions (ICs) for other runs.
#
# save_ic_get_config : define the config to use based on the save_ic config parameter
#
# save_ic has two options available:
#   save_ic:end_leg (save ICs at the end of every leg)
#   save_ic:end_run (save ICs at the end of the run)
#
# By default, without an option, ICs are saved at given offsets from the leg
# start, possibly if conditions are met. These offset(s) and condition(s) must
# be coded in the "else" clause below. See commented example.
#
function save_ic_get_config ()
{
    do_save_ic=false
    save_ic_custom=false
    if $(has_config save_ic)
    then

        tmp_leg_length_sec=${leg_length_sec}
        has_config nemo && ! has_config ifs && tmp_leg_length_sec=$(( tmp_leg_length_sec + $(leap_days "${leg_start_date}" "${leg_end_date}")*24*3600 ))

        if $(has_config save_ic:end_leg)
        then
            # save ICs at the end of every leg
            do_save_ic=true
            save_ic_date_offset=( "+ ${tmp_leg_length_sec} sec" )
        elif $(has_config save_ic:end_run)
        then
            # save ICs at the end of the run
            leg_end_date_yyyymmdd=$(date -u -d "${leg_end_date}" +%Y%m%d)
            run_end_date_yyyymmdd=$(date -u -d "${run_end_date}" +%Y%m%d)
            tmp_leg_end_date_yyyymmdd=${leg_end_date_yyyymmdd}
            has_config nemo && ! has_config ifs && tmp_leg_end_date_yyyymmdd=$(date -u --date="${leg_end_date_yyyymmdd} + $(leap_days "${leg_start_date_yyyymmdd}" "${run_end_date_yyyymmdd}") day" +%Y%m%d )

            if [ ${tmp_leg_end_date_yyyymmdd} -eq ${run_end_date_yyyymmdd} ]
            then
                do_save_ic=true && save_ic_date_offset=( "+ ${tmp_leg_length_sec} sec" )
            fi
        else
            echo "WARNING: save_ic requires to define condition and offset here or in the runscript"
            save_ic_custom=true
            do_save_ic=false
            save_ic_date_offset=( )
            # edit here or in the runscript the conditional statement to decide which legs to save ICs and the offset from leg start
            # e.g. to save at the end of every 10 legs:
            #EXAMPLE if (( leg_number%10 == 0 ))
            #EXAMPLE then
            #EXAMPLE     do_save_ic=true
            #EXAMPLE     save_ic_date_offset=( "+ ${leg_length_sec} sec" )
            #EXAMPLE fi
        fi
    fi
}


# save_ic_define_vars : define variables used in the save_ic functions
#
# The following variables must be defined in the runscript before calling this function
#
# 1) do_save_ic : set to true/false to enable save_ic feature
#
# 2) save_ic_date_offset : array of dates (offset from start of chunk, at 00:00h) to save ICs.
#    e.g. save_ic_date_offset=( "+1 month" "+2 month" )
#    The following restrictions apply, it is up to the user to ensure compliance
#     - first timestep of run is not supported
#     - if requesting first timestep of leg start, NEMO restarts from previous legs will
#       be used if present
#     - only one IC per month is supported (to simplify the IFS filter)
#     - maximum 9 ICs per leg (because of current restrictions in NEMO namelist)
#     - if requesting model-level output for IFS (e.g. for PRIMAVERA),
#       there will be no model-level variables in output on the timestep which
#       ICs are requested, unless requesting the last timestep of the leg
#
function save_ic_define_vars ()
{
    if true
    then
        # save original value of ifs_lastout
        has_config ifs && [[ -z "${save_ic_ifs_lastout_orig:-}" ]] && save_ic_ifs_lastout_orig=${ifs_lastout}
        save_ic_ifs_lastout=false

        # sanity checks
        [ ${#save_ic_date_offset[@]} -gt 9 ] && error "Cannot create more than 9 ICs in a given leg"

        # fill vars
        for (( i=0; i< ${#save_ic_date_offset[@]}; i++ ))
        do
            # sanity check for custom offsets
            # check leg_start_date + current save_ic_date_offset <  leg_end_date
            end_current_save_ic_date=$(date -u -d "${leg_start_date} ${curren_save_ic_offset}" +%Y%m%d)
            if has_config nemo && ! has_config ifs ; then
               # nemo works with no leap calendar in case to be needed this day must be removed.
               end_current_save_ic_date=$(date -u --date="${end_current_save_ic_date} - $(leap_days "${run_start_date}" "${curren_save_ic_offset}") day" +%Y%m%d )
            fi
            tmp_leg_end_date=$(date -u -d "${leg_end_date}" +%Y%m%d)
            [[  ${tmp_leg_end_date}  -lt ${end_current_save_ic_date} ]] && error "error requested IC date: ${end_current_save_ic_date} is out of leg boundaries: ${tmp_leg_end_date}"
            #save_ic_date1[$i]=${save_ic_date[$i]}
            save_ic_date1[$i]=$(date -u -d "${leg_start_date} ${save_ic_date_offset[$i]}" +%Y%m%d)
            save_ic_date[$i]=$(date -uR -d "${save_ic_date1[$i]}")
            save_ic_sec[$i]=$(( $(date -d "${save_ic_date[$i]}" +%s) - $(date -d "${run_start_date}" +%s) ))
            # Correct for leap days because NEMO standalone uses no-leap calendar
            has_config nemo && ! has_config ifs && save_ic_sec[$i]=$(( save_ic_sec[$i] - $(leap_days "${run_start_date}" "${save_ic_date[$i]}")*24*3600 ))
            save_ic_day[$i]=$(( ${save_ic_sec[$i]} / 86400 ))
            has_config ifs && save_ic_ppt_files[$i]="ppt"$(printf %06d ${save_ic_day[$i]})"0000"
            has_config nemo && save_ic_nemo_ts[$i]=$(( save_ic_sec[$i] / nem_time_step_sec ))
            # sanity checks
            if [ ${save_ic_sec[$i]} -eq 0 ]
            then
                info "Initial conditions cannot be created for first timestep of run"
                has_config ifs && save_ic_ppt_files[$i]=""
                has_config nemo && save_ic_nemo_ts[$i]=-1
            elif [ ${save_ic_sec[$i]} -eq $leg_start_sec ]
            then
                if has_config nemo
                then
                    info "Cannot create NEMO ICs for first timestep of leg at save_ic_date1[$i], will use restarts from previous leg"
                    save_ic_nemo_ts[$i]=-1
                fi
            fi
            # if saving ICs at the end of the leg, set ifs_lastout=true
            if [ ${save_ic_sec[$i]} -eq $leg_end_sec ]
            then
                if $(has_config ifs) && ! ${save_ic_ifs_lastout_orig}
                then
                    ifs_lastout=true
                    save_ic_ifs_lastout=true
                fi
            fi
        done
    fi
}

# prepare output configuration (NEMO namelist, IFS ppt file)
function save_ic_prepare_output()
{
    # For NEMO this is handled in the namelist variables ln_rst_list and nn_stocklist in namelist.nemo.ref.sh
    if has_config nemo
    then
        nemo_stocklist=""
        #for (( i=0; i< ${#save_ic_date[@]}; i++ )) ; do nemo_stocklist+=$(( save_ic_sec[$i] / nem_time_step_sec )), ; done
        for (( i=0; i< ${#save_ic_date[@]}; i++ ))
        do
            [ ${save_ic_nemo_ts[$i]} -ne -1 ] && nemo_stocklist+=${save_ic_nemo_ts[$i]},
        done
        nemo_stocklist+=$(( leg_end_sec / nem_time_step_sec )),
        for (( i=${#save_ic_date[@]}; i<9 ; i++ )) ; do nemo_stocklist+=0, ; done
        nemo_stocklist+=$'\n'
    fi

    # For IFS we need to create links to special ppt file
    if has_config ifs
    then
        # create the special ppt file
        save_ic_ppt_filename=pptdddddd0000_save_ic
        save_ic_ppt_ifile=pptdddddd0000 # change this for e.g. primavera
        # these are the codes which must be added to the special ppt file used for generating
        # required IFS model-level output at each requested timestep
        save_ic_MFP2DF_param_ids="129,152"
        save_ic_MFPPHY_param_ids="139,170,183,236,39,40,41,42,141,198,235,31,238,32,33,34,35,36,37,38,148,172,173,174,15,17,16,18,66,67,74,43,160,161,162,163,234,28,27,30,29" #all variables needed
        #save_ic_MFPPHY_param_ids="198,36,37,38,148,172,173,174,15,17,16,18,66,67,74,43,160,161,162,163,234,28,27,30,29" #all variables (old)
        #save_ic_MFPPHY_param_ids="198,36,37,38,148,172,173,174,15,17,16,18,74,43,160,161,162,163,234" #trunk
        #save_ic_MFPPHY_param_ids="173,174,15,17,16,18,66,67,74,43,160,161,162,163,234,28,27,30,29" #primavera
        save_ic_filter_param_ids=""
        # get requested output codes as a single line without space and without the last comma
        NFPPHY=`grep NFPPHY postins/$save_ic_ppt_ifile | sed -e 's/^\s*NFPPHY\s*=\s*//' -e 's/[,]$//'`
        MFPPHY=$(sed -e '1h;2,$H;$!d;g' -re 's/.*MFPPHY *=([0-9,\. \n]+).*/\1/' -e 's/[ \n]//g' -e 's/,$//' <postins/$save_ic_ppt_ifile)
        [[ $MFPPHY == "" ]] && error "IFS ppt file postins/$save_ic_ppt_ifile does not contain any entries in MFPPHY."

        NFP2DF=`grep NFP2DF postins/$save_ic_ppt_ifile | sed -e 's/^\s*NFP2DF\s*=\s*//' -e 's/[,]$//'`
        MFP2DF=$(sed -e '1h;2,$H;$!d;g' -re 's/.*MFP2DF *=([0-9,\. \n]+).*/\1/' -e 's/[ \n]//g' -e 's/,$//' <postins/$save_ic_ppt_ifile)
        [[ $MFP2DF == "" ]] && NFP2DF=0 && MFP2DF=","

        IFS=","
        old_MFPPHY_params=( ${MFPPHY} )
        old_MFP2DF_params=( ${MFP2DF} )
        new_MFPPHY_params=( ${save_ic_MFPPHY_param_ids} )
        new_MFP2DF_params=( ${save_ic_MFP2DF_param_ids} )
        unset IFS
        # update requested output codes by adding necessary variables
        for new_param_id in ${new_MFPPHY_params[@]}
        do
            new_param_id=`printf "%0*d" 3 $new_param_id`
            found=false
            for old_param_id in ${old_MFPPHY_params[@]}
            do
                old_param_id=`printf "%03g" $old_param_id`
                [ ${old_param_id} -eq ${new_param_id} ] && found=true && break
            done
            if ! $found
            then
                NFPPHY=$(( $NFPPHY + 1 ))
                MFPPHY="${MFPPHY},$new_param_id"
                save_ic_filter_param_ids="${save_ic_filter_param_ids}$new_param_id,"
            fi
        done
        for new_param_id in ${new_MFP2DF_params[@]}
        do
            new_param_id=`printf "%0*d" 3 $new_param_id`
            found=false
            for old_param_id in ${old_MFP2DF_params[@]}
            do
                old_param_id=`printf "%03g" $old_param_id`
                [ ${old_param_id} -eq ${new_param_id} ] && found=true && break
            done
            if ! $found
            then
                NFP2DF=$(( $NFP2DF + 1 ))
                MFP2DF="${MFP2DF},$new_param_id"
            fi
        done

        # create special ppt file with modified codes from original ppt file
        sed -r -e '1h;2,$H;$!d;g' \
            -e 's|(NFPPHY\s*=)[0-9 ]+|\1'${NFPPHY}'|' \
            -e 's|(MFPPHY\s*=)[0-9, \n]+\n|\1'${MFPPHY}',\n|' \
            -e 's|(NFP2DF\s*=)\s*[0-9]+|\1'${NFP2DF}'|' \
            -e 's|(MFP2DF\s*=)[0-9, \n]+\n|\1'${MFP2DF}',\n|' \
            -e 's|NFP3DFS\s*=\s*[0-9]+,||' \
            -e 's|MFP3DFS\s*=[0-9, \n]+\n|\n|' \
            -e 's|NRFP3S\s*=\s*\-?[0-9]+,||' \
            -e 's/&NAMFPC/\&NAMFPC\n    NFP3DFS = 9,\n    MFP3DFS = 133,75,76,246,247,248,138,155,130,\n    NRFP3S  = -99\n    NFP2DF = 2,\n    MFP2DF = 152,129,/' \
            <postins/$save_ic_ppt_ifile >postins/${save_ic_ppt_filename}

        # cleanup existing links from previous legs (required when save_ic_ifs_lastout=true)
        ls -ltr postins
        find -L postins/ -xtype l -samefile postins/${save_ic_ppt_filename} -exec rm {} \;
        # create links to ppt file for all requested timesteps
        for (( i=0; i< ${#save_ic_date[@]}; i++ ))
        do
            if [ ! -f postins/$save_ic_ppt_filename ]
            then
                error "IFS IC file $save_ic_ppt_filename does not exist."
            elif [ "${save_ic_ppt_files[$i]}" != "" ]  ; then
                ln -sf $save_ic_ppt_filename postins/${save_ic_ppt_files[$i]}
            fi
        done
        /bin/ls -1 postins/* > dirlist
        ls -ltr postins
    fi
}

# post-process save_ic results
function save_ic_postproc()
{
    # cleanup existing links for subsequent legs (see issue #345)
    if has_config ifs
    then
        ls -ltr postins
        find -L  postins/ -xtype l -samefile postins/${save_ic_ppt_filename} -exec rm {} \;
        /bin/ls -1 postins/* > dirlist
        ls -ltr postins
    fi
    for (( i=0; i< ${#save_ic_date[@]}; i++ ))
    do
        outdir="save_ic/"${save_ic_date1[$i]}

        # save IFS ICs
        if has_config ifs
        then
            [ "${save_ic_ppt_files[$i]}" != "" ] && save_ic_ifs_out2init $i
        fi
        # copy oasis restarts
        if has_config oasis
        then
            # copy oasis restarts if this is the last or first timestep of the leg
            # oasis files depend on the components and configuration
            # so they go in their own sub-directory, easier to separate from ifs and nemo files
            if [ ${save_ic_sec[$i]} -eq $leg_end_sec ]
            then
                mkdir -p ${outdir}/oasis
                for f in ${oas_rst_files} ; do test -f ${f} && cp ${f} ${outdir}/oasis ; done
            elif [ ${save_ic_sec[$i]} -eq $leg_start_sec ]
            then
                mkdir -p ${outdir}/oasis
                oas_rst_dir="restart/oasis/$(printf %03d $((leg_number)))"
                for f in ${oas_rst_files} ; do test -f ${oas_rst_dir}/${f} && cp ${oas_rst_dir}/${f} ${outdir}/oasis ; done
            fi
        fi
        # copy NEMO restarts
        if has_config nemo
        then
            nemo_ts=$(( save_ic_sec[$i] / nem_time_step_sec ))
            ns=$(printf %08d $(( nemo_ts - nem_restart_offset )))
	    # copy each restart type seperately, depending on config
	    extensions="oce"
	    has_config lim3 && extensions+=" ice"
	    has_config pisces && extensions+=" trc"
	    for ext in $extensions
	    do
		if ls ${exp_name}_${ns}_restart_${ext}_????.nc > /dev/null 2>&1
		then
                    mkdir -p ${outdir}/nemo
                    if [[ ${save_ic_sec[$i]} -eq $leg_start_sec ]] || [[ ${save_ic_sec[$i]} -eq $leg_end_sec ]]
                    then
			cp -f ${exp_name}_${ns}_restart_${ext}_????.nc ${outdir}/nemo
                    else
			mv -f ${exp_name}_${ns}_restart_${ext}_????.nc ${outdir}/nemo
                    fi
		else
                    error "Cannot find NEMO ($ext) initial conditions for date ${save_ic_date1[$i]}"
		fi
	    done
        fi
        # copy TM5 restarts
        leg_end_date_yyyymmdd=$(date -u -d "${leg_end_date}" +%Y%m%d)
        if has_config tm5 && [[ $leg_end_date_yyyymmdd == ${save_ic_date1[$i]} ]]
        then
	    found=false
	    for f in TM5_restart_${leg_end_date_yyyymmdd}_0000_glb300x200.nc save_${leg_end_date_yyyymmdd}00_glb300x200.hdf
	    do
		if [[ -f "$f" ]] ; then
		    mkdir -p ${outdir}/tm5
		    cp -f $f ${outdir}/tm5
		    found=true
		fi
	    done
            ! $found && error "Cannot find TM5 restarts for date ${save_ic_date1[$i]}" || true
	fi
    done
}

# Function to create IC files for IFS from special output based on a script from K. Wyser
function save_ic_ifs_out2init()
{
    icdate=${save_ic_date1[$1]}
    #srcdir=$2
    filter_output=true
    grib_filter=${GRIB_BIN_PATH}${GRIB_BIN_PATH:+/}grib_filter

    # temporary directory
    tmpdir=tmp_save_ic
    [ -d $tmpdir ] && rm -rf ${tmpdir}
    mkdir -p $tmpdir

    # first IFS timestep of a month is saved in last output file
    if [[ "$(date -d "${icdate}" +%d)" == "01" ]] ; then
        yyyymm=$(date -d "${icdate} - 6 hours" +%Y%m)
    else
        yyyymm=$(date -d "${icdate}" +%Y%m)
    fi

    # find the ICMSH/GG files containing the last timestep, which are in $srcdir/output/ifs/???
    #local ifile_sh=`find $srcdir -name ICMSH${exp_name}+$yyyymm -print -quit`
    #local ifile_gg=`find $srcdir -name ICMGG${exp_name}+$yyyymm -print -quit`
    ifile_sh=ICMSH${exp_name}+$yyyymm
    ifile_gg=ICMGG${exp_name}+$yyyymm
    if [[ ! -f "$ifile_sh" ]] ; then error "cannot find ICMSH file $ifile_sh to create initial conditions at $icdate!" ; return 0 ; fi
    if [[ ! -f "$ifile_gg" ]] ; then error "cannot find ICMGG file $ifile_gg to create initial conditions at $icdate!" ; return 0 ; fi

    if $filter_output
    then
        # define grib filters to separate data for IC from normal output
        # filter_time_init is to define which timestep contains the ICs, currently only one per month is supported
        filter_time_init="dataDate == ${icdate} && dataTime == 0"
        # filter_time_out is to filter any timesteps which might contain ICs
        filter_time_out="dataTime == 0 && ( 0 "
        for (( j=0; j< ${#save_ic_date[@]}; j++ )) ; do filter_time_out+=" || ( dataDate == ${save_ic_date1[$j]} )"; done
        filter_time_out+=" )"
        $save_ic_ifs_lastout && filter_last="( dataTime == 0 ) && ( dataDate == ${save_ic_date1[$((j-1))]} )" || filter_last="0"
        # build expression to filter the paramIDs which we have to filter out, set them in ini_filter_param_ids in the runscript
        #save_ic_filter_param_ids="198,36,37,38,148,172,173,174,15,17,16,18,74,43,160,161,162,163,234"
        filter_params="1"
        IFS=","
        for p in $save_ic_filter_param_ids ; do filter_params+=" && paramId != $p" ; done
        unset IFS

        # SH file
        ofile_sh_init=${tmpdir}/sh_init.grb
        ofile_sh_out=${tmpdir}/sh_out.grb
        ofile_sh_last=${tmpdir}/sh_last.grb
        filter_sh=${tmpdir}/filter_sh
        write_sh_init="if ( $filter_time_init ) { write \"${ofile_sh_init}\"; }"
        write_sh_out="if ( $filter_last ) { write \"${ofile_sh_last}\"; } else { write \"${ofile_sh_out}\"; }"

        cat > $filter_sh << EOT
if ( ! ( $filter_time_out ) ) { write "${ofile_sh_out}"; }
else {
if ( typeOfLevel is "hybrid" ) { ${write_sh_init}; }
else { ${write_sh_out}; }
}
EOT

        #GG file
        ofile_gg_init=${tmpdir}/gg_init.grb
        ofile_gg_out=${tmpdir}/gg_out.grb
        ofile_gg_last=${tmpdir}/gg_last.grb
        filter_gg=${tmpdir}/filter_gg
        write_gg_init="if ( $filter_time_init ) { write \"${ofile_gg_init}\"; }"
        write_gg_out="if ( $filter_last ) { write \"${ofile_gg_last}\"; } else { write \"${ofile_gg_out}\"; }"

        cat > $filter_gg << EOT
if ( ! ( $filter_time_out ) ) { write "${ofile_gg_out}"; }
else {
if ( levelType is "ml" ) { ${write_gg_init}; }
else {
  if ( levelType is "pl" ) { ${write_gg_out}; }
  else {
    if ( $filter_params ) {
      ${write_gg_init};
      ${write_gg_out};
    }
    else {
      ${write_gg_init};
    }
  }
}
}
EOT

        #run grib_filter on SH and GG files
        $grib_filter $filter_sh $ifile_sh
        $grib_filter $filter_gg $ifile_gg
        if [[ ! -f "$ofile_sh_init" ]] ; then error "ICMSH file $ifile_sh does not contain data to create initial conditions at $icdate" ; return 0 ; fi
        if [[ ! -f "$ofile_gg_init" ]] ; then error "ICMGG file $ifile_gg does not contain data to create initial conditions at $icdate" ; return 0 ; fi

        #rename files
        ifile_sh=${tmpdir}/$(basename $ifile_sh)
        ifile_gg=${tmpdir}/$(basename $ifile_gg)
        mv ${ofile_sh_init} ${ifile_sh}
        mv ${ofile_gg_init} ${ifile_gg}
        #srcdir=${tmpdir}

    fi # $filter_output

    if [[ ! -f "$ifile_sh" ]] ; then error "cannot find ICMSH file $ifile_sh to create initial conditions at $icdate!" ; return 0 ; fi
    if [[ ! -f "$ifile_gg" ]] ; then error "cannot find ICMGG file $ifile_gg to create initial conditions at $icdate!" ; return 0 ; fi

    tgtdir=save_ic/$icdate/ifs
    ofile_sh=${tgtdir}/ICMSH${exp_name}INIT
    ofile_gg_init=${tgtdir}/ICMGG${exp_name}INIT
    ofile_gg_iniua=${tgtdir}/ICMGG${exp_name}INIUA

    # make sure the output folder is created and delete any existing files
    #[ -d $tgtdir ] && rm -rf ${tgtdir}/* || mkdir -p $tgtdir
    mkdir -p $tgtdir
    rm -f $ofile_sh $ofile_gg_init $ofile_gg_iniua

    cat > ${tmpdir}/gf1 << EOT
if ( typeOfLevel is "hybrid" ) { write "${tmpdir}/shinit.[shortName].[level]"; }
EOT
    $grib_filter ${tmpdir}/gf1 $ifile_sh

    cp -f ${tmpdir}/shinit.lnsp.1 $ofile_sh
    for lev in {1..91}
    do
        for var in vo d t
        do
            cat ${tmpdir}/shinit.$var.$lev >> $ofile_sh
        done
    done
    cat ${tmpdir}/shinit.z.1 >> $ofile_sh

    cat > ${tmpdir}/gf2 << EOT
write "${tmpdir}/gginit.[shortName]";
EOT
    $grib_filter ${tmpdir}/gf2 $ifile_gg

    for var in stl1 stl2 stl3 stl4 swvl1 swvl2 swvl3 swvl4 sd src skt ci tsn asn \
        rsn sst istl1 istl2 istl3 istl4 chnk lsm sr al aluvp alnip aluvd alnid \
        lai_lv lai_hv sdfor slt sdor isor anor slor lsrh cvh cvl tvh tvl
    do
        cat ${tmpdir}/gginit.$var >> $ofile_gg_init
    done

    cat > ${tmpdir}/gf3 << EOT
write "${tmpdir}/gginiua.[shortName].[level]";
EOT
    $grib_filter ${tmpdir}/gf3 $ifile_gg

    #   ${tmpdir}/gginiua.o3 \ check this!
    for lev in {1..91}
    do
        for var in q
        do
            cat ${tmpdir}/gginiua.$var.$lev >> $ofile_gg_iniua
        done
    done

    for lev in {1..91}
    do
        for var in crwc cswc clwc ciwc cc
        do
            cat ${tmpdir}/gginiua.$var.$lev >> $ofile_gg_iniua
        done
    done

    # copy filtered files to the appropriate location
    if $filter_output
    then
        # move filter files to $tgtdir, in case something went wrong
        mv -f $filter_sh $filter_gg $tgtdir
        ifile_sh=$(basename $ifile_sh)
        ifile_gg=$(basename $ifile_gg)
        # keep the full output files as a backup
        mv -f $ifile_sh $tgtdir/${ifile_sh}-out+init
        mv -f $ifile_gg $tgtdir/${ifile_gg}-out+init
        # move the filtered output file to the  runtime folder
        [ -f $ofile_sh_out ] && mv -f $ofile_sh_out $ifile_sh || error "save_ic_ifs_out2init - warning! SH file $ofile_sh_out missing!"
        [ -f $ofile_gg_out ] && mv -f $ofile_gg_out $ifile_gg || error "save_ic_ifs_out2init - warning! GG file $ofile_gg_out missing!"
        # keep a copy of the last timestep
        if $save_ic_ifs_lastout
        then
            [ -f $ofile_sh_last ] && mv -f $ofile_sh_last $tgtdir/${ifile_sh}-ifs_lastout
            [ -f $ofile_gg_last ] && mv -f $ofile_gg_last $tgtdir/${ifile_gg}-ifs_lastout
        fi
    fi #$filter_output

    # delete tmp folder
    rm -rf $tmpdir

    echo "save_ic_ifs_out2init ended successfully, results are in $tgtdir"
}
+ declare -a save_ic_date save_ic_date1 save_ic_sec save_ic_day save_ic_ppt_file save_ic_nemo_ts
+ oas_rst_files='rstas.nc rstos.nc r_hum.nc r_g2d.nc r_udr.nc r_div.nc r_vor.nc                  r_ddr.nc r_tmp.nc r_dmf.nc r_s2d.nc r_umf.nc vegin.nc lpjgv.nc'

# -----------------------------------------------------------------------------
# *** Carbon cycle configuration
# -----------------------------------------------------------------------------
# set to true to write co2 fluxes sent to TM5
ccycle_debug_fluxes=true
+ ccycle_debug_fluxes=true

# =============================================================================
# *** END of User configuration
# =============================================================================

# =============================================================================
# *** This is where the code begins ...
# =============================================================================

# -----------------------------------------------------------------------------
# *** Create the run dir if necessary and go there
#     Everything is done from here.
# -----------------------------------------------------------------------------
if [ ! -d ${run_dir} ]
then
    mkdir -p ${run_dir}
fi
+ '[' '!' -d /gpfs/scratch/bsc32/bsc32627/a3tb/19930101/fc01/runtime ']'
+ mkdir -p /gpfs/scratch/bsc32/bsc32627/a3tb/19930101/fc01/runtime
cd ${run_dir}
+ cd /gpfs/scratch/bsc32/bsc32627/a3tb/19930101/fc01/runtime

# -----------------------------------------------------------------------------
# Autosubmit sanity check
# -----------------------------------------------------------------------------
CHUNK=1
+ CHUNK=1
if [ -f ece.info ]; then
  ece_info_leg_number=$(grep leg_number ece.info | tail -n 1 | awk -F"=" '{print $2}')
  current_leg_number=$((ece_info_leg_number + 1))
  if [[ "${CHUNK}" != "${current_leg_number}" ]]; then
    echo "Runscript leg_number" $current_leg_number
    echo "Don't match with Autosubmit CHUNK" ${CHUNK}
    exit 1
  fi
fi
+ '[' -f ece.info ']'

# -----------------------------------------------------------------------------
# *** Determine the time span of this run and whether it's a restart leg
# -----------------------------------------------------------------------------

# Regularise the format of the start and end date of the simulation
run_start_date=$(date -uR -d "${run_start_date}")
date -uR -d "${run_start_date}"
++ date -uR -d 19930101
+ run_start_date='Fri, 01 Jan 1993 00:00:00 +0000'
run_end_date=$(date -uR -d "${run_end_date}")
date -uR -d "${run_end_date}"
++ date -uR -d '19930101 + 1 month'
+ run_end_date='Mon, 01 Feb 1993 00:00:00 +0000'


# -----------------------------------------------------------------------------
# *** Set path to grib_set
# -----------------------------------------------------------------------------

grib_set=${GRIB_BIN_PATH}${GRIB_BIN_PATH:+/}grib_set
+ grib_set=/apps/GRIB/1.14.0/INTEL//bin/grib_set

# Loop over the number of legs
for (( ; run_num_legs>0 ; run_num_legs-- ))
do

    # Check for restart information file and set the current leg start date
    #   Ignore restart information file if force_run_from_scratch is true
    if ${force_run_from_scratch} || ! [ -r ${ece_info_file} ]
    then
        leg_is_restart=false
        leg_start_date=${run_start_date}
        leg_number=1
    else
        leg_is_restart=true
        . ./${ece_info_file}
        leg_start_date=${leg_end_date}
        leg_number=$((leg_number+1))
    fi

    # Compute the end date of the current leg
    if [ -n "${rst_freq}" ]
    then
        leg_end_date=$(date -uR -d "${leg_start_date} + ${rst_freq}")
    else
        leg_end_date=${run_end_date}
    fi

    # Check if legs are integer multiples of full years if LPJG is used
    if has_config lpjg
    then
        
        if [[ $(date +%m%d%T -u -d "${leg_start_date}") != "010100:00:00" || \
            $(date +%m%d%T -u -d "${leg_start_date} + ${rst_freq}") != "010100:00:00" ]]
        then
            error "LPJ-GUESS runs must start on Jan 1 and end on Dec 31. Multi-year legs are allowed."
        fi
    fi              

    if [[ "FALSE" == "TRUE" ]]
    then
        leg_end_date=${run_end_date}
        ifs_lastout=true
    fi

    # Some time variables needed later
    leg_length_sec=$(( $(date -u -d "${leg_end_date}" +%s) - $(date -u -d "${leg_start_date}" +%s) ))
    leg_start_sec=$(( $(date -u -d "${leg_start_date}" +%s) - $(date -u -d "${run_start_date}" +%s) ))
    leg_end_sec=$(( $(date -u -d "${leg_end_date}" +%s) - $(date -u -d "${run_start_date}" +%s) ))
    leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d)
    leg_start_date_yyyy=$(date -u -d "${leg_start_date}" +%Y)
    leg_end_date_yyyy=$(date -u -d "${leg_end_date}" +%Y)

    # Check whether there's actually time left to simulate - exit otherwise
    if [ ${leg_length_sec} -le 0 ]
    then
        info "Leg start date equal to or after end of simulation."
        info "Nothing left to do. Exiting."
        exit 0
    fi

    # Initial conditions saved during the run
    do_save_ic=false
    save_ic_custom=false
    has_config save_ic && save_ic_get_config
    # if you do not use an option with save_ic, you must define 'do_save_ic' and
    # 'save_ic_date_offset' here or in ../libsave_ic.sh/save_ic_get_config()
    # with AS runtime, no need to edit the script, set SAVE_IC_OFFSET (and optionally SAVE_IC_COND)
    if $save_ic_custom
    then
        [[ "true" = "" ]] && save_ic_cond=true || save_ic_cond='true'
        if eval $save_ic_cond ; then do_save_ic=true ; else do_save_ic=false ; fi
        save_ic_date_offset=( "+1 month" "+4 month" "+ 7 month" "+ 10 month" "+ 12 month" )
    fi
    ${do_save_ic} && save_ic_define_vars

    # -------------------------------------------------------------------------
    # *** Prepare the run directory for a run from scratch
    # -------------------------------------------------------------------------
    if ! $leg_is_restart
    then
        # ---------------------------------------------------------------------
        # *** Check if run dir is empty. If not, and if we are allowed to do so
        #     by ${force_run_from_scratch}, remove everything
        # ---------------------------------------------------------------------
        if $(ls * >& /dev/null)
        then
            if ${force_run_from_scratch}
            then
                rm -fr ${run_dir}/*
            else
                error "Run directory not empty and \$force_run_from_scratch not set."
            fi
        fi

        # ---------------------------------------------------------------------
        # *** Copy executables of model components
        # *** Additionally, create symlinks to the original place for reference
        # ---------------------------------------------------------------------
        cp    ${ifs_exe_file} .
        ln -s ${ifs_exe_file} $(basename ${ifs_exe_file}).lnk

        if $(has_config amip)
        then
            cp    ${amip_exe_file} .
            ln -s ${amip_exe_file} $(basename ${amip_exe_file}).lnk
        fi

        if $(has_config nemo)
        then
            cp    ${nem_exe_file} .
            ln -s ${nem_exe_file} $(basename ${nem_exe_file}).lnk

            cp    ${rnf_exe_file} .
            ln -s ${rnf_exe_file} $(basename ${rnf_exe_file}).lnk

            cp    ${xio_exe_file} .
            ln -s ${xio_exe_file} $(basename ${xio_exe_file}).lnk
        fi

        if $(has_config lpjg)
        then
            cp    ${lpjg_exe_file} .
            ln -s ${lpjg_exe_file} $(basename ${lpjg_exe_file}).lnk
        fi

        if $(has_config tm5)
        then
            cp    ${tm5_exe_file} .
            ln -s ${tm5_exe_file} $(basename ${tm5_exe_file}).lnk
        fi

        # ---------------------------------------------------------------------
        # *** Files needed for IFS (linked)
        # ---------------------------------------------------------------------

        # Initial data
        ln -s \
        ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMGGECE3INIUA \
                                                            ICMGG${exp_name}INIUA
        ln -s \
        ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMSHECE3INIT \
                                                            ICMSH${exp_name}INIT
        rm -f ICMGG${exp_name}INIT
        cp ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMGGECE3INIT \
                                                            ICMGG${exp_name}INIT

        # add bare_soil_albedo to ICMGG*INIT
        tempfile=tmp.$$
        ${grib_set} -s dataDate=$(date -u -d "$run_start_date" +%Y%m%d) \
            ${ini_data_dir}/ifs/${ifs_grid}/climate/bare_soil_albedos.grb \
            ${tempfile}

        cat ${tempfile} >> ICMGG${exp_name}INIT
        rm -f ${tempfile}

        # add land ice mask if needed
        if ${ifs_landice}
        then
            tempfile=tmp.$$
            cdo divc,10 -setcode,82 -selcode,141 ICMGG${exp_name}INIT ${tempfile}
            ${grib_set} -s gridType=reduced_gg ${tempfile} ${tempfile}
            cat ${tempfile} >> ICMGG${exp_name}INIT
            rm -f ${tempfile}
        fi

        # Other stuff
        ln -s ${ini_data_dir}/ifs/rtables/* .
      
        if $(has_config atmnudg) ; then
            ln -s ${ini_data_dir}/rlxml* .
        fi
        
        # Output control (ppt files)
        if [ ! -f ${output_control_files_dir}/pptdddddd0600 ] &&  [ ! -f ${output_control_files_dir}/pptdddddd0300 ];then
           echo "Error from ece-esm.sh: Neither the file pptdddddd0600 or pptdddddd0300 exists in the directory:"
           echo " " ${output_control_files_dir}
           exit -1
        fi
        mkdir postins
        cp ${output_control_files_dir}/ppt* postins/
        if [ -f postins/pptdddddd0600 ];then
           ln -s pptdddddd0600 postins/pptdddddd0000
           ln -s pptdddddd0600 postins/pptdddddd1200
           ln -s pptdddddd0600 postins/pptdddddd1800
        fi
        if [ -f postins/pptdddddd0300 ];then
           ln -s pptdddddd0300 postins/pptdddddd0900
           ln -s pptdddddd0300 postins/pptdddddd1500
           ln -s pptdddddd0300 postins/pptdddddd2100
           if [ ! -f postins/pptdddddd0600 ];then
               ln -s pptdddddd0300 postins/pptdddddd0000
               ln -s pptdddddd0300 postins/pptdddddd0600
               ln -s pptdddddd0300 postins/pptdddddd1200
               ln -s pptdddddd0300 postins/pptdddddd1800
           fi
        fi
        /bin/ls -1 postins/* > dirlist

        # ---------------------------------------------------------------------
        # *** Files needed for LPJ-GUESS
        # ---------------------------------------------------------------------
        if $(has_config lpjg)
        then
            # Check for valid grid
            if [ $lpjg_res != "T255" -a $lpjg_res != "T159" ]
            then
                error "LPJG-gridlist doesn't exist for ifs-grid: ${ifs_grid}" 
            fi
            # Initial data - saved state for LPJ-GUESS (.bin format)
            lpjgstartdir=$(printf "lpjg_state_%04d" $leg_start_date_yyyy)
            ln -sf ${ini_data_dir}/lpjg/ini_state/${lpjg_res}/${lpjgstartdir} ${run_dir}/${lpjgstartdir}

            # Control files (i.e. .ins, landuse, N deposition, soil type files etc.)
            cp  -f ${ecearth_src_dir}/lpjg/data/ins/*.ins .
            # activate the new litterfall scheme for C4MIP - for the coupled model this is done when both pisces and lpjg are activated
            has_config pisces lpjg && echo -e "!override for EC-Earth-CC in runscript\nifpftlitterfall 1\ncalc_phen_after_restart 0" >> global.ins
            mkdir -p ${run_dir}/landuse

        fi

        # ---------------------------------------------------------------------
        # *** Files needed for NEMO (linked)
        # ---------------------------------------------------------------------
        if $(has_config nemo)
        then
            # Link initialisation files for matching ORCA grid
            for f in \
                bathy_meter.nc coordinates.nc \
                ahmcoef.nc \
                K1rowdrg.nc M2rowdrg.nc mask_itf.nc \
                decay_scale_bot.nc decay_scale_cri.nc \
                mixing_power_bot.nc mixing_power_cri.nc mixing_power_pyc.nc \
                runoff_depth.nc subbasins.nc
            do
                [ -f ${ini_data_dir}/nemo/initial/${nem_grid}/$f ] && ln -s ${ini_data_dir}/nemo/initial/${nem_grid}/$f
            done

            # Copying the time independent NEMO files for the matching ORCA grid in order to facilitate cmorisation:
            for f in \
                bathy_meter.nc subbasins.nc
            do
                mkdir -p output/nemo/ofx-data
                [ -f ${ini_data_dir}/nemo/initial/${nem_grid}/$f ] && cp -f ${ini_data_dir}/nemo/initial/${nem_grid}/$f output/nemo/ofx-data/
            done

            # Link geothermal heating file (independent of grid) and matching weight file
            ln -s ${ini_data_dir}/nemo/initial/Goutorbe_ghflux.nc
            ln -s ${ini_data_dir}/nemo/initial/weights_Goutorbe1_2_orca${nem_res_hor}_bilinear.nc

            # Link the salinity climatology file (needed for diagnostics)
            ln -s ${ini_data_dir}/nemo/climatology/${nem_grid}/sali_ref_clim_monthly.nc

            # Link either restart files or climatology files for the initial state
            if $(has_config nemo:start_from_restart)
            then
                # When linking restart files, we accept three options:
                # (1) Merged files for ocean and ice, i.e.
                #     restart_oce.nc and restart_ice.nc
                # (2) One-file-per-MPI-rank, i.e.
                #     restart_oce_????.nc and restart_ice_????.nc
                #     No check is done whether the number of restart files agrees
                #     with the number of MPI ranks for NEMO!
                # (3) One-file-per-MPI-rank with a prefix, i.e.
                #     <exp_name>_<time_step>_restart_oce_????.nc (similar for the ice)
                #     The prefix is ignored.
                # The code assumes that one of the options can be applied! If more
                # options are applicable, the first is chosen. If none of the
                # options apply, NEMO will crash with missing restart file.
                if   ls -U ${nem_restart_file_path}/restart_[oi]ce.nc > /dev/null 2>&1
                then
                    ln -s ${nem_restart_file_path}/restart_[oi]ce.nc ./

                elif ls -U ${nem_restart_file_path}/restart_[oi]ce_????.nc > /dev/null 2>&1
                then
                    ln -s ${nem_restart_file_path}/restart_[oi]ce_????.nc ./

                else
                    for f in ${nem_restart_file_path}/????_????????_restart_[oi]ce_????.nc
                    do
                        ln -s $f $(echo $f | sed 's/.*_\(restart_[oi]ce_....\.nc\)/\1/')
                    done
                fi
            else

                # Temperature and salinity files for initialisation
                ln -s ${ini_data_dir}/nemo/climatology/absolute_salinity_WOA13_decav_Reg1L75_clim.nc
                ln -s ${ini_data_dir}/nemo/climatology/conservative_temperature_WOA13_decav_Reg1L75_clim.nc
                ln -s ${ini_data_dir}/nemo/climatology/weights_WOA13d1_2_orca${nem_res_hor}_bilinear.nc

                # Grid dependent runoff files
                case ${nem_grid} in
                    ORCA1*)   ln -s ${ini_data_dir}/nemo/climatology/runoff-icb_DaiTrenberth_Depoorter_ORCA1_JD.nc ;;
                    ORCA025*) ln -s ${ini_data_dir}/nemo/climatology/ORCA_R025_runoff_v1.1.nc ;;
                esac
            fi

            # for ocean_nudging
            if $(has_config nemo:ocenudg) ; then
                ln -s ${ini_data_dir}/nemo/oce_nudg/resto.nc
            fi

            # XIOS files
            . ${ctrl_file_dir}/iodef.xml.sh > iodef.xml
            ln -s ${ctrl_file_dir}/context_nemo.xml
            ln -s ${ctrl_file_dir}/domain_def_nemo.xml
            ln -s ${ctrl_file_dir}/axis_def_nemo.xml
            ln -s ${ctrl_file_dir}/grids_def_nemo.xml
            ln -s ${ctrl_file_dir}/field_def_nemo-lim.xml
            ln -s ${ctrl_file_dir}/field_def_nemo-opa.xml
            ln -s ${ctrl_file_dir}/field_def_nemo-pisces.xml
            ln -s ${ctrl_file_dir}/field_def_nemo-inerttrc.xml
            ln -s ${output_control_files_dir}/file_def_nemo-lim3.xml file_def_nemo-lim.xml
            ln -s ${output_control_files_dir}/file_def_nemo-opa.xml
            ln -s ${output_control_files_dir}/file_def_nemo-pisces.xml

            if [ -f ${ini_data_dir}/xios/ORCA${nem_res_hor}/coordinates_xios.nc ]
            then
                cp ${ini_data_dir}/xios/ORCA${nem_res_hor}/coordinates_xios.nc ./
            else
                info "File 'coordinates_xios.nc' not found. NEMO can not be run with land domain removal!"
            fi

            # Files needed for TOP/PISCES
            if $(has_config pisces)
            then
                ln -fs ${ini_data_dir}/nemo/pisces/dust_INCA_ORCA_R1.nc
                ln -fs ${ini_data_dir}/nemo/pisces/ndeposition_Duce_ORCA_R1.nc
                ln -fs ${ini_data_dir}/nemo/pisces/pmarge_etopo_ORCA_R1.nc
                ln -fs ${ini_data_dir}/nemo/pisces/river_global_news_ORCA_R1.nc
                ln -fs ${ini_data_dir}/nemo/pisces/Solubility_T62_Mahowald_ORCA_R1.nc

                ln -fs ${ini_data_dir}/nemo/pisces/par_fraction_gewex_clim90s00s_ORCA_R1.nc
                ln -fs ${ini_data_dir}/nemo/pisces/DIC_GLODAP_annual_ORCA_R1.nc
                ln -fs ${ini_data_dir}/nemo/pisces/Alkalini_GLODAP_annual_ORCA_R1.nc
                ln -fs ${ini_data_dir}/nemo/pisces/O2_WOA2009_monthly_ORCA_R1.nc
                ln -fs ${ini_data_dir}/nemo/pisces/PO4_WOA2009_monthly_ORCA_R1.nc
                ln -fs ${ini_data_dir}/nemo/pisces/Si_WOA2009_monthly_ORCA_R1.nc
                ln -fs ${ini_data_dir}/nemo/pisces/DOC_PISCES_monthly_ORCA_R1.nc
                ln -fs ${ini_data_dir}/nemo/pisces/Fer_PISCES_monthly_ORCA_R1.nc
                ln -fs ${ini_data_dir}/nemo/pisces/NO3_WOA2009_monthly_ORCA_R1.nc

                # create co2 concentration file atcco2.txt if required
                if { [ $ifs_cmip_fixyear -gt 0 ] || [[ "${ifs_A4xCO2}" = "TRUE" ]]; } && [[ "${bgc_1PCTCO2}" = "FALSE" ]]
                then
                    rm -f atcco2.txt
                elif [[ "${bgc_1PCTCO2}" = "TRUE" ]]
                then
                    cp -f ${ini_data_dir}/nemo/pisces/mole-fraction-of-carbon-dioxide-in-air_1pctCO2_1849-2016.txt atcco2.txt
                else
                    # determine scenario-name and co2-file middle-fix
                    case $(echo ${ifs_cmip6_scenario} | tr '[:upper:]' '[:lower:]') in
                        hist*)     pis_scen="ssp585"; pis_sco2_mfix="REMIND-MAGPIE-ssp585-1-2-1";;
                        ssp2-4.5*) pis_scen="ssp245"; pis_sco2_mfix="MESSAGE-GLOBIOM-ssp245-1-2-1";;
                        ssp5-3.4*) pis_scen="ssp534os"; pis_sco2_mfix="REMIND-MAGPIE-ssp534-over-1-2-1";;
                        ssp5-8.5*) pis_scen="ssp585"; pis_sco2_mfix="REMIND-MAGPIE-ssp585-1-2-1";;
                        *)  error "Scenario ${ifs_cmip6_scenario} not defined for PISCES" ;;
                    esac

                    # concatenate historic and scenario (2015+) co2 concentration file
                    pis_sco2_pfix="${ini_data_dir}/nemo/pisces/mole-fraction-of-carbon-dioxide-in-air_input4MIPs_GHGConcentrations"
                    cat ${pis_sco2_pfix}_CMIP_UoM-CMIP-1-2-0_gr1-GMNHSH_1849-2014.txt ${pis_sco2_pfix}_ScenarioMIP_UoM-${pis_sco2_mfix}_gr1-GMNHSH_2015-2500.txt > atcco2.txt
                fi
            fi

            #linking surface boundary conditions for CFCs (even if CFCs are not included)
            ln -fs ${ini_data_dir}/nemo/cfc/CFCs_CDIAC_extension_1637_2019.dat CFCs_CDIAC.dat

            if $(has_config pisces:start_from_restart)
            then
            # Same three options as for nemo:start_from_restart
                if   ls -U ${nem_restart_file_path}/restart_trc.nc > /dev/null 2>&1
                then
                    ln -s ${nem_restart_file_path}/restart_trc.nc ./

                elif ls -U ${nem_restart_file_path}/restart_trc_????.nc > /dev/null 2>&1
                then
                    ln -s ${nem_restart_file_path}/restart_trc_????.nc ./

                else
                    for f in ${nem_restart_file_path}/????_????????_restart_trc_????.nc
                    do
                        ln -s $f $(echo $f | sed 's/.*_\(restart_trc_....\.nc\)/\1/')
                    done
                fi
            fi

        fi

        # ---------------------------------------------------------------------
        # *** Files needed for the Runoff mapper (linked)
        # ---------------------------------------------------------------------

        has_config rnfmapper && \
            ln -s ${ini_data_dir}/runoff-mapper/runoff_maps.nc

        # -------------------------------------------------------------------------
        # *** File and dir needed for TM5
        # -------------------------------------------------------------------------
        if $(has_config tm5)
        then
            tm5_istart=33

            case ${tm5_istart} in
                33|32) ln -s \
                    ${tm5_restart_file_path}/TM5_restart_${leg_start_date_yyyymmdd}_0000_glb300x200.nc
                    ;;
                31) ln -s ${tm5_restart_file_path}/tm5_save.hdf
                    ;;
                5)  ln -s ${tm5_restart_file_path}/tm5_mmix.hdf
                    ;;
                2|9) 
                    ;;
                *)  error "Unsupported initial fields option (TM5): ${tm5_istart}"
                    ;;
            esac

            # spectral info
            ln -s ${ini_data_dir}/tm5/TM5_INPUT/T${ifs_res_hor}_info.txt

            # Profiling dir for TM5
            mkdir -p ${run_dir}/tm5_profile
        fi

        # ---------------------------------------------------------------------
        # *** Files needed for OASIS (linked)
        # ---------------------------------------------------------------------

        # Name table file
        ln -s ${ini_data_dir}/oasis/cf_name_table.txt

        # -- Get grid definition and weight files for IFS/NEMO or IFS/AMIP coupling
        has_config nemo && \
            oas_grid_dir=${ini_data_dir}/oasis/T${ifs_res_hor}-ORCA${nem_res_hor} && \
            mycp='cp'

        has_config amip && \
            oas_grid_dir=${ini_data_dir}/oasis/AMIP && \
            mycp='cdo -f nc copy' # to enforce nc format, needed for 'cdo merge' to work (have nc4c with Primavera files)

        # Grid definition files
        if $(has_config tm5)
        then
            ${mycp} ${oas_grid_dir}/areas.nc gcm_areas.nc
            ${mycp} ${oas_grid_dir}/grids.nc gcm_grids.nc
            ${mycp} ${oas_grid_dir}/masks.nc gcm_masks.nc
        else
            ln -s ${oas_grid_dir}/areas.nc
            ln -s ${oas_grid_dir}/grids.nc
            ln -s ${oas_grid_dir}/masks.nc
        fi

        # Weight files
        case ${ifs_res_hor} in
            159)  oas_agrd=080
                  ;;
            255)  oas_agrd=128
                  ;;
            511)  oas_agrd=256
                  ;;
            799)  oas_agrd=400
                  ;;
            *)    error "Unsupported horizontal resolution (IFS): ${ifs_res_hor}"
                  ;;
        esac

        if $(has_config nemo)
        then
            case ${nem_res_hor} in
                1)  oas_ogrd=O1t0
                    ;;
                025)  oas_ogrd=Ot25
                    ;;
                *)  error "Unsupported horizontal resolution (NEMO): ${nem_res_hor}"
                    ;;
            esac
        fi

        if ${oas_link_weights}
        then
            for f in ${oas_grid_dir}/rmp_????_to_????_GAUSWGT.nc
            do
                ln -s $f
            done
        fi

        if $(has_config ifs nemo)
        then
            for f in ${oas_rst_ifs_nemo}
            do
                cp ${oas_grid_dir}/rst/$f .
            done
        fi

        # -- Get grid definition, weight and restart files for TM5 coupling
        if $(has_config tm5)
        then
            oas_grid_dir=${ini_data_dir}/oasis/T${ifs_res_hor}-TM5-LPJG

            cp ${oas_grid_dir}/tm5_areas.nc .
            cp ${oas_grid_dir}/tm5_grids.nc .
            cp ${oas_grid_dir}/tm5_masks.nc .

            if ${oas_link_weights}
            then
                for f in ${oas_grid_dir}/rmp_????_to_????_*.nc
                do
                    ln -s $f
                done
            fi

            # -- Get restart files for TM5-IFS/LPJG/PISCES 
            for f in ${oas_rst_ifs_tm5}
            do
                cp ${oas_grid_dir}/rst/${tm5_exch_nlevs}-levels/$f .
            done

            # -- Merge grid definition files
            cdo merge gcm_areas.nc tm5_areas.nc areas.nc
            cdo merge gcm_grids.nc tm5_grids.nc grids.nc
            cdo merge gcm_masks.nc tm5_masks.nc masks.nc
        fi

    else # i.e. $leg_is_restart == true

        # ---------------------------------------------------------------------
        # *** Remove all leftover output files from previous legs
        # ---------------------------------------------------------------------

        # IFS files
        rm -f ICM{SH,GG}${exp_name}+??????

        # NEMO files
        rm -f ${exp_name}_??_????????_????????_{grid_U,grid_V,grid_W,grid_T,icemod,SBC,scalar,SBC_scalar,diad_T}.nc

        # TM5 restart file type
        tm5_istart=33

        if [ $tm5_istart -eq 31 ] && $(has_config tm5)
        then
            ln -sf save_${leg_start_date_yyyymmdd}00_glb300x200.hdf tm5_save.hdf
        fi

    fi # ! $leg_is_restart

    #--------------------------------------------------------------------------
    # *** Surface restoring and ocean nudging options
    #--------------------------------------------------------------------------
    if $(has_config nemo:ocenudg) ; then
        ln -fs ${ini_data_dir}/nemo/oce_nudg/temp_sal*.nc ./
    fi

    if $(has_config nemo:surfresto) ; then
        ln -fs ${ini_data_dir}/nemo/surface_restoring/sss_restore_data*.nc  ./
        ln -fs ${ini_data_dir}/nemo/surface_restoring/sst_restore_data*.nc  ./
        ln -fs ${ini_data_dir}/nemo/surface_restoring/mask_restore*.nc ./
    fi

    # -------------------------------------------------------------------------
    # *** Remove land grid-points
    # -------------------------------------------------------------------------
    if $(has_config nemo:elpin)
    then
        if [ ! -f coordinates_xios.nc ]
        then
            error "ELpIN requested, but file 'coordinates_xios.nc' was not found"
        fi
        jpns=($(${ecearth_src_dir}/util/ELPiN/ELPiNv2.cmd ${nem_numproc}))
        info "nemo domain decompostion from ELpIN: ${jpns[@]}"
        nem_numproc=${jpns[0]}
        nem_jpni=${jpns[1]}
        nem_jpnj=${jpns[2]}
    elif has_config nemo
    then
        info "nemo original domain decomposition (not using ELPiN)"
    fi

    # -------------------------------------------------------------------------
    # *** Initial conditions saved during the run
    # -------------------------------------------------------------------------
    ${do_save_ic} && save_ic_prepare_output

    # -------------------------------------------------------------------------
    # *** Create some control files
    # -------------------------------------------------------------------------

    # Create TM5 runtime rcfile
    tm5_start_date=$(date -u -d "${leg_start_date}" +%F\ %T)
    tm5_end_date=$(date -u -d "${leg_end_date}" +%F\ %T)

    if $(has_config tm5)
    then
        cp -f ${ctrl_file_dir}/tm5-config-run.rc ${run_dir}
        ${ecearth_src_dir}/tm5mp/setup_tm5 --no-compile \
            --time-start="${tm5_start_date}" --time-final="${tm5_end_date}" \
            --istart=${tm5_istart} ${run_dir}/tm5-config-run.rc
    fi

    # IFS frequency output for namelist
    if [ -f postins/pptdddddd0300 ]
    then
        ifs_output_freq=$(( 3 * 3600 / ifs_time_step_sec ))
    elif [ -f postins/pptdddddd0600 ]
    then
        ifs_output_freq=$(( 6 * 3600 / ifs_time_step_sec ))
    else
        error "IFS output frequency undefined."
    fi

    # IFS, NEMO, LIM, AMIP namelist and OASIS namcouple files
    has_config ifs       && . ${ctrl_file_dir}/namelist.ifs.sh                          > fort.4
    has_config nemo      && . ${ctrl_file_dir}/namelist.nemo.ref.sh                     > namelist_ref
    has_config ifs nemo  && . ${ctrl_file_dir}/namelist.nemo-${nem_grid}-coupled.cfg.sh > namelist_cfg
    has_config lim3      && . ${ctrl_file_dir}/namelist.lim3.ref.sh                     > namelist_ice_ref
    has_config lim3      && . ${ctrl_file_dir}/namelist.lim3-${nem_grid}.cfg.sh         > namelist_ice_cfg
    has_config rnfmapper && . ${ctrl_file_dir}/namelist.runoffmapper.sh                 > namelist.runoffmapper
    has_config amip      && . ${ctrl_file_dir}/namelist.amip.sh                         > namelist.amip
    has_config pisces    && . ${ctrl_file_dir}/namelist.nemo.top.ref.sh                 > namelist_top_ref
    has_config pisces    && . ${ctrl_file_dir}/namelist.nemo.top.cfg.sh                 > namelist_top_cfg
    has_config pisces    && . ${ctrl_file_dir}/namelist.nemo.pisces.ref.sh              > namelist_pisces_ref
    has_config pisces    && . ${ctrl_file_dir}/namelist.nemo.pisces.cfg.sh              > namelist_pisces_cfg
    has_config nemo      && . ${ctrl_file_dir}/namelist.nemo.age.ref.sh                 > namelist_age_ref
    has_config nemo      && . ${ctrl_file_dir}/namelist.nemo.age.cfg.sh                 > namelist_age_cfg
    #include CFCs namelist even if CFCs are not included
    has_config nemo      && . ${ctrl_file_dir}/namelist.nemo.cfc.ref.sh                 > namelist_cfc_ref
    has_config nemo      && . ${ctrl_file_dir}/namelist.nemo.cfc.cfg.sh                 > namelist_cfc_cfg
    # C-cycle - overwrite coupled nemo namelist
    has_config pisces tm5:co2  && \
        . ${ctrl_file_dir}/namelist.nemo-${nem_grid}-carboncycle.cfg.sh > namelist_cfg

    lucia=
    . ${ctrl_file_dir}/namcouple.sh > namcouple

    # -------------------------------------------------------------------------
    # *** LPJ-GUESS initial data
    # -------------------------------------------------------------------------
    if $(has_config lpjg)
    then
        # LPJG runtime rcfile - update with leg dates
        . ${ctrl_file_dir}/namelist.lpjg.sh > lpjg_steps.rc

        # determine lpjg scenario-name and co2-file middle-fix
        case $(echo ${ifs_cmip6_scenario} | tr '[:upper:]' '[:lower:]') in
            hist*)     lpjg_scen="ssp370"; lu_src="AIM"    ; lpjg_sco2_mfix="AIM-ssp370-1-2-1"            ; lu_file_posfix="2018_10_08.txt";;
            ssp1-1.9*) lpjg_scen="ssp119"; lu_src="IMAGE"  ; lpjg_sco2_mfix="IMAGE-ssp119-1-2-1"          ; lu_file_posfix="2019_03_13.txt";;
            ssp1-2.6*) lpjg_scen="ssp126"; lu_src="IMAGE"  ; lpjg_sco2_mfix="IMAGE-ssp126-1-2-1"          ; lu_file_posfix="2018_10_08.txt";;
            ssp2-4.5*) lpjg_scen="ssp245"; lu_src="MESSAGE"; lpjg_sco2_mfix="MESSAGE-GLOBIOM-ssp245-1-2-1"; lu_file_posfix="2018_10_08.txt";;
            ssp3-7.0*) lpjg_scen="ssp370"; lu_src="AIM"    ; lpjg_sco2_mfix="AIM-ssp370-1-2-1"            ; lu_file_posfix="2018_10_08.txt";;
            ssp4-3.4*) lpjg_scen="ssp434"; lu_src="GCAM"   ; lpjg_sco2_mfix="GCAM4-ssp434-1-2-1"            ; lu_file_posfix="2018_10_08.txt";;
            ssp5-3.4*) lpjg_scen="ssp534os"; lu_src="MAGPIE"; lpjg_sco2_mfix="REMIND-MAGPIE-ssp534-over-1-2-1"            ; lu_file_posfix="2019_05_10.txt";;
            ssp5-8.5*) lpjg_scen="ssp585"; lu_src="MAGPIE" ; lpjg_sco2_mfix="REMIND-MAGPIE-ssp585-1-2-1"  ; lu_file_posfix="2018_10_08.txt";;
            *)  error "Scenario ${ifs_cmip6_scenario} not defined for LPJ-GUESS" ;;
        esac

        lpjg_scenario_new="historical + $lpjg_sco2_mfix"
        lpjg_scenario_info=${run_dir}/lpjg_scenario.info

        if [ -f $lpjg_scenario_info ]
        then
            source $lpjg_scenario_info
            if [ "$lpjg_scenario_new" = "$lpjg_scenario" ]
            then
                lpjg_copy_rte=false
            else
                lpjg_copy_rte=true
            fi
        else
            lpjg_copy_rte=true
        fi

        # copy RTE only if necessary (at beginning of a run or when scenario changes)
        if $lpjg_copy_rte
        then
            # write info about installed scenarios to file
            echo "lpjg_scenario=\"historical + $lpjg_sco2_mfix\"" > $lpjg_scenario_info
            # set file prefixes depending on scenario
            lu_file_prefix="1850_2100_luh2_Hist_ScenarioMIP_UofMD"
            lu_file_midfix="2_1_f"

            lu_path="${ini_data_dir}/lpjg/landuse/${lpjg_res}/${lpjg_scen}"

            # copy and reference-link landuse,  gross transitions, crops, n-fertilisation
            for inp in lu gross crop nfert
            do
                if [ $inp == "crop" -o $inp == "nfert" ]
                then
                    lu_src_file="${lu_path}/${inp}_rfirr_${lu_file_prefix}_${lu_src}_${lpjg_scen}_${lu_file_midfix}_${lpjg_res}_${lu_file_posfix}"
                else
                    lu_src_file="${lu_path}/${inp}_${lu_file_prefix}_${lu_src}_${lpjg_scen}_${lu_file_midfix}_${lpjg_res}_${lu_file_posfix}"
                fi
                cp -f $lu_src_file ${run_dir}/landuse/${inp}_luh2.txt
                ln -fs $lu_src_file ${run_dir}/landuse/${inp}_luh2.txt.lnk
            done
            
            # nitrogen deposition files
            mkdir -p ${run_dir}/ndep
            for inp in drynhx2 drynoy2 wetnhx2 wetnoy2
            do
                ndep_src_file="${ini_data_dir}/lpjg/ndep/${lpjg_res}/${lpjg_scen}/${lpjg_scen}_${lpjg_res}_${inp}.nc"
                cp -f  $ndep_src_file ${run_dir}/ndep/${inp}.nc
                ln -fs  $ndep_src_file ${run_dir}/ndep/${inp}.nc.lnk
            done

            # concatenate historic and scenario (2015+) co2 concentration file
            # hist co2 file 
            lpjg_hco2_file="${ini_data_dir}/ifs/cmip6-data/mole-fraction-of-carbon-dioxide-in-air_input4MIPs_GHGConcentrations_CMIP_UoM-CMIP-1-2-0_gr1-GMNHSH_0000-2014.nc"
            # scenario co2 file 
            lpjg_sco2_file="${ini_data_dir}/ifs/cmip6-data/mole-fraction-of-carbon-dioxide-in-air_input4MIPs_GHGConcentrations_ScenarioMIP_UoM-${lpjg_sco2_mfix}_gr1-GMNHSH_2015-2500.nc"
            # combined file
            lpjg_co2_file="${run_dir}/mole_fraction_of_carbon_dioxide_in_air_input4MIPs_lpjg.nc"
            rm -f $lpjg_co2_file
            cdo mergetime $lpjg_hco2_file $lpjg_sco2_file $lpjg_co2_file
        fi

        # Populate or update LPJG run directories
        for (( n=1; n<=${lpjg_numproc}; n++ ))
        do
            # if run from scratch or number of procs has been extended
            if ! $leg_is_restart || [ ! -d ${run_dir}/run${n} ]
            then
                # Make output directories
                mkdir -p ${run_dir}/run${n}/output

                # Copy *.ins, lpjg_steps.rc and OASIS-MCT restart files
                cp ${run_dir}/*.ins ${run_dir}/run${n}

                # Copy output control files
                cp ${output_control_files_dir}/lpjg_cmip6_output.ins ${run_dir}/run${n}
                ln -s ${output_control_files_dir}/lpjg_cmip6_output.ins ${run_dir}/run${n}/lpjg_cmip6_output.ins.lnk

                cp ${ini_data_dir}/lpjg/oasismct/ghg*.txt ${run_dir}/run${n}
                cp ${ini_data_dir}/lpjg/oasismct/${lpjg_res}/ece_gridlist_${lpjg_res}.txt ${run_dir}/run${n}/ece_gridlist.txt
                ln -s ${ini_data_dir}/lpjg/oasismct/${lpjg_res}/ece_gridlist_${lpjg_res}.txt ${run_dir}/run${n}/ece_gridlist.txt.lnk

                # Data only needed by master
                if [ $n == 1 ]
                then
                    cp ${ini_data_dir}/lpjg/oasismct/${lpjg_res}/*.nc ${run_dir}
                    cp ${ini_data_dir}/lpjg/oasismct/lpjgv.txt ${run_dir}/run${n}
                fi
            fi
            # Refresh output-dirs after they hav been removed at end of the last leg
            mkdir -p ${run_dir}/run${n}/output/CMIP6
            mkdir -p ${run_dir}/run${n}/output/CRESCENDO
        done

        if $leg_is_restart
        then
            lpjg_restart_dir="restart/lpjg/$(printf %03d $((leg_number-1)))"
            lpjg_rst_state="${lpjg_restart_dir}/lpjg_state_${leg_start_date_yyyy}"
            if [ -d "$lpjg_rst_state" ]
            then
                ln -sf $lpjg_rst_state
            else
                echo "lpjg restart dir $lpjg_rst_state not available"
                exit -1
            fi
        fi
    fi


    # -------------------------------------------------------------------------
    # *** Create ICMCL file with vegetation fields
    #     not needed if LPJG is used with feedback
    # -------------------------------------------------------------------------
    tempfile=tmp.$$

    case ${ifs_veg_source} in
    "ifs" )
        # Vegetation from IFS (climatology)

        icmclfile=${ini_data_dir}/ifs/${ifs_grid}/climate/ICMCL_ONLY_VEG_PD

        # Create data for december, the year before the leg starts
        ${grib_set} \
            -s dataDate=$(printf "%04d" $((leg_start_date_yyyy-1)))1215 \
            ${icmclfile}-12 ICMCL${exp_name}INIT

        # Create data for all month in the years of the leg
        for (( y=${leg_start_date_yyyy} ; y<=${leg_end_date_yyyy} ; y++ ))
        do
            yy=$(printf "%04d" $y)
            for m in {1..12}
            do
                mm=$(printf "%02d" $m)
                ${grib_set} -s dataDate=${yy}${mm}15 ${icmclfile}-${mm} ${tempfile}
                cat ${tempfile} >> ICMCL${exp_name}INIT
            done
        done

        # Create data for january, the year after the leg ends
        ${grib_set} \
            -s dataDate=$(printf "%04d" $((leg_end_date_yyyy+1)))0115 \
            ${icmclfile}-01 ${tempfile}
        cat ${tempfile} >> ICMCL${exp_name}INIT
        ;;
    "era20c"|"cmip6"|"custom_"* )
        # Vegetation from a LPJG run (off-line or EC-Earth3-Veg)

        rm -f ICMCL${exp_name}INIT

        # Create data for all years of the leg, including one year
        # before and one year after
        for (( yr=leg_start_date_yyyy-1 ; yr<=leg_end_date_yyyy+1 ; yr+=1 ))
        do

            case ${ifs_veg_source} in
            'era20c' )
                # no scenario needed with era20c
                icmcl_scenario="" ;;
            'custom_'* )
                # no scenario implemented yet with custom_dir
                icmcl_scenario="" ;;
            'cmip6' )
                # select scenario, use SSP3-7.0 as default
                # if not otherwise specified
                icmcl_scenario="historical"
                if ( [ $ifs_cmip_fixyear -le 0 ] && [ $yr -ge 2015 ] ) || \
                     [ $ifs_cmip_fixyear -ge 2015 ]
                then
                    [[ ${ifs_cmip6_scenario} =~ ^SSP ]] \
                        && icmcl_scenario=${ifs_cmip6_scenario} \
                        || if [ ${ifs_covid19}  == TRUE ]
                           then
                               icmcl_scenario='SSP2-4.5'
                           else
                               icmcl_scenario='SSP3-7.0'
                           fi
                fi ;;
            esac
            
            if [ $ifs_cmip_fixyear -le 0 ] || [[ ${ifs_veg_source} == custom_* ]]
            then
                cat ${ini_data_dir}/ifs/${ifs_grid}/icmcl_${veg_version}/${icmcl_scenario}/icmcl_$yr.grb >> ICMCL${exp_name}INIT
            else
                # Fixed year forcing, requires cdo! (only when not using ifs_veg_source=custom_exp*)
                # If cdo is not available at runtime you need to fix proper
                # icmcl files beforehand and use them here
                cdo setyear,$yr ${ini_data_dir}/ifs/${ifs_grid}/icmcl_${veg_version}/${icmcl_scenario}/icmcl_${ifs_cmip_fixyear}.grb ${tempfile}
                cat ${tempfile} >> ICMCL${exp_name}INIT
            fi
        done
        ;;
    "none" )
        info "no ICMCL file is created"
        ;;
    * )
        error "Vegetation from ${ifs_veg_source} not implemented"
        ;;
    esac

    # Clean up
    rm -f ${tempfile}

    # -------------------------------------------------------------------------
    # *** Link the appropriate NEMO restart files of the previous leg
    # -------------------------------------------------------------------------
    if $leg_is_restart && $(has_config nemo)
    then
        ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec - nem_restart_offset )))
        for (( n=0 ; n<nem_numproc ; n++ ))
        do
            np=$(printf %04d ${n})
            ln -fs ${exp_name}_${ns}_restart_oce_${np}.nc restart_oce_${np}.nc
            ln -fs ${exp_name}_${ns}_restart_ice_${np}.nc restart_ice_${np}.nc
            has_config pisces && \
                ln -fs ${exp_name}_${ns}_restart_trc_${np}.nc restart_trc_${np}.nc
        done

        # Make sure there are no global restart files
        # If links are found, they will be removed. We are cautious and do
        # _not_ remove real files! However, if real global restart files are
        # present, NEMO/LIM will stop because time stamps will not match.
        [ -h restart_oce.nc ] && rm restart_oce.nc
        [ -h restart_ice.nc ] && rm restart_ice.nc
        [ -h restart_trc.nc ] && rm restart_trc.nc
    fi

    # -------------------------------------------------------------------------
    # *** Remove some OASIS files of the previous leg
    # -------------------------------------------------------------------------
    if $leg_is_restart
    then
        rm -f anaisout_*
    fi

    # -------------------------------------------------------------------------
    # *** Remove any ccycle debug output files
    # -------------------------------------------------------------------------
    if ${ccycle_debug_fluxes} && $leg_is_restart && $(has_config tm5:co2)
    then
      if $(has_config lpjg)
      then
          rm -f GUE_{CNAT,CANT,CNPP}_*.nc run1/GUE_{CNAT,CANT,CNPP}_*.nc
          rm -f TM5_Land{CNAT,CANT,CNPP}_*.nc
      fi
      if $(has_config pisces)
      then
          rm -f O_CO2FLX_*.nc
          rm -f TM5_OceCFLX_*.nc
      fi
    fi

    # -------------------------------------------------------------------------
    # *** Restart sanity check before launch
    # -------------------------------------------------------------------------
    # This code must run before launch command don't put anything in between

    echo "Restart files sanity check before launch:"
    if $leg_is_restart  ; then
      # oasis
      for oasis_restart_file in $(ls restart/oasis/$(printf %03d $((leg_number)))/*); do
        [ -n "$( diff $(basename ${oasis_restart_file}) ${oasis_restart_file} -q )" ] && \
        cp -f ${oasis_restart_file} . || true
      done
      # ifs rcf
      rcf_restart_file=restart/ifs/$(printf %03d $((leg_number)))/rcf
      [ -n "$(diff $(basename ${rcf_restart_file}) ${rcf_restart_file} -q )" ] && \
      cp -f ${rcf_restart_file} . || true
    fi
    # -------------------------------------------------------------------------
    # *** End restart sanity check before launch
    # -------------------------------------------------------------------------
    # -------------------------------------------------------------------------
    # *** Start the run
    # -------------------------------------------------------------------------
    export DR_HOOK_IGNORE_SIGNALS='-1'
    export CPLNG='active'

    # Use the launch function from the platform configuration file
    has_config nemo && \
        cmd="${xio_numproc} ${xio_exe_file} -- \
             ${nem_numproc} ${nem_exe_file} -- \
             ${ifs_numproc} ${ifs_exe_file} -v ecmwf -e ${exp_name}" || \
        cmd="${ifs_numproc} ${ifs_exe_file} -v ecmwf -e ${exp_name}"

    has_config lpjg && cmd=${cmd}" -- ${lpjg_numproc} ${lpjg_exe_file} guess.ins -parallel"
    has_config tm5  && cmd=${cmd}" -- ${tm5_numproc}  ${tm5_exe_file} tm5-run.rc"
    has_config amip && cmd=${cmd}" -- ${amip_numproc} ${amip_exe_file}"
    has_config nemo && cmd=${cmd}" -- ${rnf_numproc} ${rnf_exe_file}" 

    t1=$(date +%s)
    launch $cmd
    t2=$(date +%s)

    tr=$(date -d "0 -$t1 sec + $t2 sec" +%T)

    # -------------------------------------------------------------------------
    # *** Check for signs of success
    #     Note the tests provide no guarantee that things went fine! They are
    #     just based on the IFS, NEMO and TM5 log files. More tests (e.g. checking
    #     restart files) could be implemented.
    # -------------------------------------------------------------------------

    # Checking for IFS success
    if [ -f ifs.stat ]
    then
        if [ "$(awk 'END{print $3}' ifs.stat)" == "CNT0" ]
        then
            info "Leg successfully completed according to IFS log file 'ifs.stat'."
        else
            error "Leg not completed according to IFS log file 'ifs.stat'."
        fi
    else
        error "IFS log file 'ifs.stat' not found after run."
    fi

    # Check for NEMO success
    if $(has_config nemo)
    then
        if [ -f ocean.output ]
        then
            if [ "$(sed -n '/New day/h; ${g;s:.*\([0-9/]\{10\}\).*:\1:;p;}' ocean.output)" == "$(date -u -d "${leg_end_date} - 1 day" +%Y/%m/%d)" ]
            then
                info "Leg successfully completed according to NEMO log file 'ocean.output'."
            else
                error "Leg not completed according to NEMO log file 'ocean.output'."
            fi
        else
            error "NEMO log file 'ocean.output' not found after run."
        fi
    fi

    # Check for TM5 success
    if $(has_config tm5)
    then
        if [ -f tm5.ok ]
        then
            info "Leg successfully completed according to existing TM5 file 'tm5.ok'."
        else
            error "Leg not completed according to non-existing TM5 file 'tm5.ok'."
        fi
    fi

    # -------------------------------------------------------------------------
    # *** Post-process initial conditions saved during the run if requested
    # -------------------------------------------------------------------------
    ${do_save_ic} && save_ic_postproc

    # -------------------------------------------------------------------------
    # *** Move IFS output files to archive directory
    # -------------------------------------------------------------------------
    outdir="output/ifs/$(printf %03d $((leg_number)))"
    mkdir -p ${outdir}

    prv_leg=$(printf %03d $((leg_number-1)))

    # This takes care of a special IFS feature: The output for the last time
    # step of each leg is written at the first time step of the new leg. The
    # following code makes sure that the output is appended to the appropriate
    # file. Since GRIB files are just streams, its done with a simple cat
    # command.
    for f in ICMSH${exp_name}+?????? ICMGG${exp_name}+??????
    do
        if [ -f output/ifs/${prv_leg}/${f} ]
        then
            cat ${f} >> output/ifs/${prv_leg}/${f}
            rm -f ${f}
        else
            mv ${f} ${outdir}
        fi
    done

    # -------------------------------------------------------------------------
    # *** Move NEMO output files to archive directory
    # -------------------------------------------------------------------------
    if $(has_config nemo)
    then
        outdir="output/nemo/$(printf %03d $((leg_number)))"
        mkdir -p ${outdir}

        for v in grid_U grid_V grid_W grid_T icemod SBC scalar SBC_scalar diad_T ptrc_T bioscalar \
                 grid_T_2D grid_U_2D grid_V_2D grid_W_2D grid_T_3D grid_U_3D grid_V_3D grid_W_3D \
                 grid_T_SFC grid_1point grid_T_3D_ncatice vert_sum \
                 grid_ptr_W_3basin_3D grid_ptr_T_3basin_2D grid_ptr_T_2D \
                 zoom_700_sum zoom_300_sum zoom_2000_sum
        do
            for f in ${exp_name}_*_????????_????????_*${v}.nc
            do
                test -f $f && mv $f $outdir/
            done
        done
    fi

    # -------------------------------------------------------------------------
    # *** Move LPJ-GUESS output files to archive directory
    # -------------------------------------------------------------------------
    if $(has_config lpjg)
    then
        outdir="output/lpjg/$(printf %03d $((leg_number)))"
        if [ -d ${outdir} ]
        then
            rm -rf  ${outdir}
        fi
        mkdir -p ${outdir}

        # LPJG run directories
        # concatenate *.out (or compressed *.out.gz) files from each run* into output dir
        flist=$(cd ${run_dir}/run1/output && find *.out.gz -type f 2>/dev/null || true)
        if [ "$flist" = "" ]
        then
            lpjg_compress_output=false
            flist=$(cd ${run_dir}/run1/output && find *.out -type f 2>/dev/null)
        else
            lpjg_compress_output=true
        fi

        mkdir ${outdir}/CMIP6

        for (( n=1; n<=${lpjg_numproc}; n++ ))
        do
            for ofile in $flist
            do
                if $lpjg_compress_output
                then
                    [ $n == 1 ] && gzip -c ${run_dir}/run${n}/output/`basename ${ofile} .gz`.hdr > ${outdir}/$ofile
                    cat ${run_dir}/run${n}/output/${ofile} >> ${outdir}/$ofile
                else
                    if (( n == 1 ))
                    then
                        cat ${run_dir}/run${n}/output/${ofile} > ${outdir}/$ofile
                    else
                        awk '(FNR!=1){print $0}' ${run_dir}/run${n}/output/${ofile} >> ${outdir}/$ofile
                    fi
                fi
            done
            rm -rf ${run_dir}/run${n}/output
        done
        
        # move monthly file if available
        if [ -f ${run_dir}/LPJ-GUESS_monthlyoutput.txt ]
        then
            mv ${run_dir}/LPJ-GUESS_monthlyoutput.txt ${outdir}
        fi
    fi

    # -------------------------------------------------------------------------
    # *** Move TM5 output files to archive directory
    # -------------------------------------------------------------------------
    if $(has_config tm5)
    then
        outdir="output/tm5/$(printf %03d $((leg_number)))"
        mkdir -p ${outdir}

        set +e
        mv budget_??????????_??????????_global.hdf      ${outdir}
        mv j_statistics_??????????_??????????.hdf       ${outdir}
        mv mmix_??????????_??????????_glb???x???.hdf    ${outdir}
        mv aerocom?_TM5_*_????????_daily.nc             ${outdir}
        mv aerocom?_TM5_*_??????_monthly.nc             ${outdir}
        mv AOD_????_??_??.nc                            ${outdir}
        mv -f TM5MP_${exp_name}_griddef.nc              ${outdir}
        mv TM5MP_${exp_name}_TP_????_??_??.nc           ${outdir}
        mv TM5MP_${exp_name}_vmr3_????_??_??.nc         ${outdir}
        mv general_TM5_${exp_name}_??????????_hourly.nc ${outdir}
        mv general_TM5_${exp_name}_??????_monthly.nc    ${outdir}
        mv *EC-Earth3-*_${exp_name}_*.nc                ${outdir}
        set -e

        # move profiling files if any
        if [ "$(ls -A ${run_dir}/tm5_profile)" ]
        then
            outdir="output/tm5/profile_$(printf %03d $((leg_number)))"
            mkdir -p ${outdir}

            for f in ${run_dir}/tm5_profile/*
            do
                test -f ${f} && mv $f ${outdir}
            done
        fi
    fi

    # -------------------------------------------------------------------------
    # *** Move IFS restart files to archive directory
    # -------------------------------------------------------------------------
    if $leg_is_restart
    then
        outdir="restart/ifs/$(printf %03d $((leg_number)))"
        mkdir -p ${outdir}

        # Figure out the time part of the restart files (cf. CTIME on rcf files)
        # NOTE: Assuming that restarts are at full days (time=0000) only!
        nd="$(printf %06d $((leg_start_sec/(24*3600))))0000"

        mv srf${nd}.???? ${outdir}

    fi

    # -------------------------------------------------------------------------
    # *** Move ccycle debug output files to archive directory
    # -------------------------------------------------------------------------
    if ${ccycle_debug_fluxes} && $(has_config tm5:co2)
    then
      outdir="output/tm5/$(printf %03d $((leg_number)))"
      mkdir -p ${outdir}
      if $(has_config lpjg)
      then
          for f in CNAT CANT CNPP ; do
              mv TM5_Land${f}_*.nc ${outdir}
              rm -f GUE_${f}_*.nc run1/GUE_${f}_*.nc
              #gf=`ls -1 GUE_${f}_*.nc | head -n 1`
              #cdo mergetime ${gf} run1/${gf} ${outdir}/${gf}
          done
      fi
      if $(has_config pisces)
      then
          mv TM5_OceCFLX_*.nc ${outdir}
          rm -f O_CO2FLX_*.nc
          #mv O_CO2FLX_*.nc ${outdir}
      fi
    fi

    # -------------------------------------------------------------------------
    # *** Move LPJ-GUESS restart files to archive directory
    # -------------------------------------------------------------------------
    if $(has_config lpjg)
    then
        outdir="restart/lpjg/$(printf %03d $((leg_number)))"
        if [ -d ${outdir} ]
        then
            rm -rf  ${outdir}
        fi
        mkdir -p ${outdir}

        state_dir="./lpjg_state_$(printf %04d $((leg_end_date_yyyy)))"
        mv ${state_dir} ${outdir}
        # LPJG writes into run1 dir, so mv to main rundir
        mv -f run1/lpjgv.nc .
        has_config tm5:co2 lpjg && mv -f run1/rlpjg.nc .

        # remove restart link
        if $leg_is_restart
        then
            old_state_dir="./lpjg_state_$(printf %04d $((leg_start_date_yyyy)))"
            if [ -L $old_state_dir ]
            then
                rm -f "$old_state_dir"
            fi
        fi
    fi

    # -------------------------------------------------------------------------
    # *** Move NEMO restart files to archive directory
    # -------------------------------------------------------------------------
    if $leg_is_restart && $(has_config nemo)
    then
        outdir="restart/nemo/$(printf %03d $((leg_number)))"
        mkdir -p ${outdir}

        ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec - nem_restart_offset )))
        for f in oce ice
        do
            mv ${exp_name}_${ns}_restart_${f}_????.nc ${outdir}
        done

        if has_config pisces
        then
            mv ${exp_name}_${ns}_restart_trc_????.nc ${outdir}
        fi
    fi

    # -------------------------------------------------------------------------
    # *** Move TM5 restart file to archive directory
    # -------------------------------------------------------------------------
    if $leg_is_restart && $(has_config tm5)
    then
        outdir="restart/tm5/$(printf %03d $((leg_number)))"
        mkdir -p ${outdir}

        case ${tm5_istart} in
            33|32) f=TM5_restart_${leg_start_date_yyyymmdd}_0000_glb300x200.nc
                ;;
            31) f=save_${leg_start_date_yyyymmdd}00_glb300x200.hdf
                ;;
        esac

        mv $f ${outdir}
    fi

    # -------------------------------------------------------------------------
    # *** Copy OASIS restart files to archive directory
    #     NOTE: These files are copied and not moved as they are used in the
    #           next leg!
    #           Note also that the OASIS restart files present at the end of
    #           the leg correspond to the start of the next leg!
    # -------------------------------------------------------------------------
    outdir="restart/oasis/$(printf %03d $((leg_number+1)))"
    mkdir -p ${outdir}

    for f in ${oas_rst_files}
    do
        test -f ${f} && cp ${f} ${outdir}
    done

    # -------------------------------------------------------------------------
    # *** Copy rcf files to the archive directory (of the next leg!)
    # -------------------------------------------------------------------------
    outdir="restart/ifs/$(printf %03d $((leg_number+1)))"
    mkdir -p ${outdir}

    for f in rcf
    do
        test -f ${f} && cp ${f} ${outdir}
    done

    # -------------------------------------------------------------------------
    # *** Move log files to archive directory
    # -------------------------------------------------------------------------
    outdir="log/$(printf %03d $((leg_number)))"
    mkdir -p ${outdir}

    for f in \
        ifs.log ifs.stat fort.4 ocean.output \
        time.step solver.stat guess.log run1/guess0.log \
        amip.log namelist.amip \
        nout.000000 debug.root.?? debug.??.?????? lucia.??.?????? \
        ctm.tm5.log.0
    do
        test -f ${f} && mv ${f} ${outdir}
    done
    has_config pisces && cp ocean.carbon ${outdir}

    for f in ctm.tm5.log.*
    do
        if [[ -f ${f} ]]
        then
            [[ -s ${f} ]] && mv ${f} ${outdir} || \rm -f ${f}
        fi
    done

    # -------------------------------------------------------------------------
    # *** Write the restart control file
    # -------------------------------------------------------------------------

    # Compute CPMIP performance
    sypd="$(cpmip_sypd $leg_length_sec $(($t2 - $t1)))"
    ncores=0
    has_config nemo      && (( ncores+=${nem_numproc}  )) || :
    has_config ifs       && (( ncores+=${ifs_numproc}  )) || :
    has_config xios      && (( ncores+=${xio_numproc}  )) || :
    has_config rnfmapper && (( ncores+=${rnf_numproc}  )) || :
    has_config lpjg      && (( ncores+=${lpjg_numproc} )) || :
    has_config tm5       && (( ncores+=${tm5_numproc}  )) || :
    has_config amip      && (( ncores+=${amip_numproc} )) || :
    chpsy="$(cpmip_chpsy  $leg_length_sec $(($t2 - $t1)) $ncores)"

    echo "#"                                             | tee -a ${ece_info_file}
    echo "# Finished leg at `date '+%F %T'` after ${tr} (hh:mm:ss)" \
                                                         | tee -a ${ece_info_file}
    echo "# CPMIP performance: $sypd SYPD   $chpsy CHPSY"| tee -a ${ece_info_file}
    echo "leg_number=${leg_number}"                      | tee -a ${ece_info_file}
    echo "leg_start_date=\"${leg_start_date}\""          | tee -a ${ece_info_file}
    echo "leg_end_date=\"${leg_end_date}\""              | tee -a ${ece_info_file}

    # Need to reset force_run_from_scratch in order to avoid destroying the next leg
    force_run_from_scratch=false

done # loop over legs
+ (( 1 ))
+ (( run_num_legs>0  ))
+ true
+ leg_is_restart=false
+ leg_start_date='Fri, 01 Jan 1993 00:00:00 +0000'
+ leg_number=1
+ '[' -n '1 month' ']'
date -uR -d "${leg_start_date} + ${rst_freq}"
++ date -uR -d 'Fri, 01 Jan 1993 00:00:00 +0000 + 1 month'
+ leg_end_date='Mon, 01 Feb 1993 00:00:00 +0000'
+ has_config lpjg
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' lpjg == any ']'
+ [[ lpjg == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ lpjg ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ lpjg ]]
+ for __m in '$config'
+ [[ lim3 =~ lpjg ]]
+ for __m in '$config'
+ [[ rnfmapper =~ lpjg ]]
+ for __m in '$config'
+ [[ xios:detached =~ lpjg ]]
+ for __m in '$config'
+ [[ oasis =~ lpjg ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ lpjg ]]
+ ((  __f  ))
+ return 1
+ [[ FALSE == \T\R\U\E ]]
date -u -d "${leg_end_date}" +%s
++ date -u -d 'Mon, 01 Feb 1993 00:00:00 +0000' +%s
date -u -d "${leg_start_date}" +%s
++ date -u -d 'Fri, 01 Jan 1993 00:00:00 +0000' +%s
+ leg_length_sec=2678400
date -u -d "${leg_start_date}" +%s
++ date -u -d 'Fri, 01 Jan 1993 00:00:00 +0000' +%s
date -u -d "${run_start_date}" +%s
++ date -u -d 'Fri, 01 Jan 1993 00:00:00 +0000' +%s
+ leg_start_sec=0
date -u -d "${leg_end_date}" +%s
++ date -u -d 'Mon, 01 Feb 1993 00:00:00 +0000' +%s
date -u -d "${run_start_date}" +%s
++ date -u -d 'Fri, 01 Jan 1993 00:00:00 +0000' +%s
+ leg_end_sec=2678400
date -u -d "${leg_start_date}" +%Y%m%d
++ date -u -d 'Fri, 01 Jan 1993 00:00:00 +0000' +%Y%m%d
+ leg_start_date_yyyymmdd=19930101
date -u -d "${leg_start_date}" +%Y
++ date -u -d 'Fri, 01 Jan 1993 00:00:00 +0000' +%Y
+ leg_start_date_yyyy=1993
date -u -d "${leg_end_date}" +%Y
++ date -u -d 'Mon, 01 Feb 1993 00:00:00 +0000' +%Y
+ leg_end_date_yyyy=1993
+ '[' 2678400 -le 0 ']'
+ do_save_ic=false
+ save_ic_custom=false
+ has_config save_ic
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' save_ic == any ']'
+ [[ save_ic == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ save_ic ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ save_ic ]]
+ for __m in '$config'
+ [[ lim3 =~ save_ic ]]
+ for __m in '$config'
+ [[ rnfmapper =~ save_ic ]]
+ for __m in '$config'
+ [[ xios:detached =~ save_ic ]]
+ for __m in '$config'
+ [[ oasis =~ save_ic ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ save_ic ]]
+ [[ save_ic:end_leg =~ save_ic ]]
+ __f=1
+ ((  __f  ))
+ return 0
+ save_ic_get_config
+ do_save_ic=false
+ save_ic_custom=false
has_config save_ic
++ has_config save_ic
++ ((  1  ))
++ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
++ local __c
++ local __m
++ '[' save_ic == any ']'
++ [[ save_ic == \a\l\l ]]
++ local __f
++ for __c in '"$@"'
++ __f=0
++ for __m in '$config'
++ [[ ifs =~ save_ic ]]
++ for __m in '$config'
++ [[ nemo:start_from_restart:elpin =~ save_ic ]]
++ for __m in '$config'
++ [[ lim3 =~ save_ic ]]
++ for __m in '$config'
++ [[ rnfmapper =~ save_ic ]]
++ for __m in '$config'
++ [[ xios:detached =~ save_ic ]]
++ for __m in '$config'
++ [[ oasis =~ save_ic ]]
++ for __m in '$config'
++ [[ save_ic:end_leg =~ save_ic ]]
++ [[ save_ic:end_leg =~ save_ic ]]
++ __f=1
++ ((  __f  ))
++ return 0
+ tmp_leg_length_sec=2678400
+ has_config nemo
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' nemo == any ']'
+ [[ nemo == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ nemo ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ nemo ]]
+ [[ nemo:start_from_restart:elpin =~ nemo ]]
+ __f=1
+ for __m in '$config'
+ [[ lim3 =~ nemo ]]
+ for __m in '$config'
+ [[ rnfmapper =~ nemo ]]
+ for __m in '$config'
+ [[ xios:detached =~ nemo ]]
+ for __m in '$config'
+ [[ oasis =~ nemo ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ nemo ]]
+ ((  __f  ))
+ return 0
+ has_config ifs
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' ifs == any ']'
+ [[ ifs == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ ifs ]]
+ [[ ifs =~ ifs ]]
+ __f=1
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ ifs ]]
+ for __m in '$config'
+ [[ lim3 =~ ifs ]]
+ for __m in '$config'
+ [[ rnfmapper =~ ifs ]]
+ for __m in '$config'
+ [[ xios:detached =~ ifs ]]
+ for __m in '$config'
+ [[ oasis =~ ifs ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ ifs ]]
+ ((  __f  ))
+ return 0
has_config save_ic:end_leg
++ has_config save_ic:end_leg
++ ((  1  ))
++ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
++ local __c
++ local __m
++ '[' save_ic:end_leg == any ']'
++ [[ save_ic:end_leg == \a\l\l ]]
++ local __f
++ for __c in '"$@"'
++ __f=0
++ for __m in '$config'
++ [[ ifs =~ save_ic ]]
++ for __m in '$config'
++ [[ nemo:start_from_restart:elpin =~ save_ic ]]
++ for __m in '$config'
++ [[ lim3 =~ save_ic ]]
++ for __m in '$config'
++ [[ rnfmapper =~ save_ic ]]
++ for __m in '$config'
++ [[ xios:detached =~ save_ic ]]
++ for __m in '$config'
++ [[ oasis =~ save_ic ]]
++ for __m in '$config'
++ [[ save_ic:end_leg =~ save_ic ]]
++ [[ save_ic:end_leg =~ end_leg ]]
++ __f=1
++ ((  __f  ))
++ return 0
+ do_save_ic=true
+ save_ic_date_offset=("+ ${tmp_leg_length_sec} sec")
+ false
+ true
+ save_ic_define_vars
+ true
+ has_config ifs
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' ifs == any ']'
+ [[ ifs == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ ifs ]]
+ [[ ifs =~ ifs ]]
+ __f=1
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ ifs ]]
+ for __m in '$config'
+ [[ lim3 =~ ifs ]]
+ for __m in '$config'
+ [[ rnfmapper =~ ifs ]]
+ for __m in '$config'
+ [[ xios:detached =~ ifs ]]
+ for __m in '$config'
+ [[ oasis =~ ifs ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ ifs ]]
+ ((  __f  ))
+ return 0
+ [[ -z '' ]]
+ save_ic_ifs_lastout_orig=false
+ save_ic_ifs_lastout=false
+ '[' 1 -gt 9 ']'
+ (( i=0 ))
+ (( i< 1 ))
date -u -d "${leg_start_date} ${curren_save_ic_offset}" +%Y%m%d
./libsave_ic.sh: line 93: curren_save_ic_offset: unbound variable
+ end_current_save_ic_date=
