[INFO] JOBID=19268645

job_name_ptrn='/gpfs/scratch/bsc32/bsc32627/t0ni/LOG_t0ni/t0ni_19931101_fc0_INI'
+ job_name_ptrn=/gpfs/scratch/bsc32/bsc32627/t0ni/LOG_t0ni/t0ni_19931101_fc0_INI
echo $(date +%s) > ${job_name_ptrn}_STAT
date +%s
++ date +%s
+ echo 1639809342

###################
# Autosubmit job
###################

set -xuve
+ set -xuve

#
# Var instantiation
#

STAMP=$(date +%Y_%m_%d_%H_%M)
date +%Y_%m_%d_%H_%M
++ date +%Y_%m_%d_%H_%M
+ STAMP=2021_12_18_07_35
HPCARCH=marenostrum4
+ HPCARCH=marenostrum4
SCRATCH_DIR=/gpfs/scratch
+ SCRATCH_DIR=/gpfs/scratch
HPCPROJ=bsc32
+ HPCPROJ=bsc32
HPCUSER=bsc32627
+ HPCUSER=bsc32627
EXPID=t0ni
+ EXPID=t0ni
exp_name=t0ni
+ exp_name=t0ni
JOBNAME=t0ni_19931101_fc0_INI
+ JOBNAME=t0ni_19931101_fc0_INI
PROJDEST=auto-ecearth3
+ PROJDEST=auto-ecearth3
ROOTDIR=/gpfs/scratch/bsc32/bsc32627/t0ni
+ ROOTDIR=/gpfs/scratch/bsc32/bsc32627/t0ni
TEMPLATE_NAME=ecearth3
+ TEMPLATE_NAME=ecearth3
CURRENT_ARCH=marenostrum4
+ CURRENT_ARCH=marenostrum4

OASIS_ini=a2s5
+ OASIS_ini=a2s5
OASIS_ini_date=22990101
+ OASIS_ini_date=22990101
OASIS_ini_member="fc0"
+ OASIS_ini_member=fc0
oas_numproc=
+ oas_numproc=

OCEAN_ini=a2s5
+ OCEAN_ini=a2s5
OCEAN_ini_date=22990101
+ OCEAN_ini_date=22990101
OCEAN_ini_member="fc0"
+ OCEAN_ini_member=fc0

ICE=LIM3
+ ICE=LIM3
ICE_ini=a2s5
+ ICE_ini=a2s5
ICE_ini_date=22990101
+ ICE_ini_date=22990101
ICE_ini_member="fc0"
+ ICE_ini_member=fc0

NEMO_resolution=ORCA025L75
+ NEMO_resolution=ORCA025L75
nem_grid_wol=$(echo ${NEMO_resolution} | cut -d 'L' -f 1) # without level (wol)
echo ${NEMO_resolution} | cut -d 'L' -f 1
++ cut -d L -f 1
++ echo ORCA025L75
+ nem_grid_wol=ORCA025
nem_res_hor=$(echo ${NEMO_resolution} | sed 's:ORCA\([0-9]\+\)L[0-9]\+:\1:')
echo ${NEMO_resolution} | sed 's:ORCA\([0-9]\+\)L[0-9]\+:\1:'
++ sed 's:ORCA\([0-9]\+\)L[0-9]\+:\1:'
++ echo ORCA025L75
+ nem_res_hor=025

#PISCES vars
PISCES=FALSE
+ PISCES=FALSE
PISCES_ini=
+ PISCES_ini=
PISCES_ini_date=
+ PISCES_ini_date=
PISCES_ini_member=""
+ PISCES_ini_member=
PISCES_OFF=FALSE
+ PISCES_OFF=FALSE
PISCES_OFF_DYN=
+ PISCES_OFF_DYN=

ATM_ini=a2s5
+ ATM_ini=a2s5
ATM_ini_date=22990101
+ ATM_ini_date=22990101
ifs_grid=T511L91
+ ifs_grid=T511L91
ifs_grid_wol=$(echo ${ifs_grid} | cut -d 'L' -f 1) # without level (wol)
echo ${ifs_grid} | cut -d 'L' -f 1
++ cut -d L -f 1
++ echo T511L91
+ ifs_grid_wol=T511
ifs_res_hor=$(echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\1:')
echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\1:'
++ sed 's:T\([0-9]\+\)L\([0-9]\+\):\1:'
++ echo T511L91
+ ifs_res_hor=511
ATM_ini_member="fc0"
+ ATM_ini_member=fc0
ATM_refnud=
+ ATM_refnud=
ATM_ini_member_perturb=FALSE
+ ATM_ini_member_perturb=FALSE
if [[ -z "$ATM_ini_member " ]]; then ATM_ini_member_perturb=FALSE; fi
+ [[ -z fc0  ]]
ATM_NUDGING=FALSE
+ ATM_NUDGING=FALSE

[[ "FALSE" == TRUE ]] && OSM=TRUE || OSM=FALSE
+ [[ FALSE == TRUE ]]
+ OSM=FALSE
[[ "FALSE" == TRUE ]] && LPJG=TRUE || LPJG=FALSE
+ [[ FALSE == TRUE ]]
+ LPJG=FALSE
LPJG_ini=
+ LPJG_ini=
[ -z $LPJG_ini ] && LPJG_ini=""
+ '[' -z ']'
+ LPJG_ini=
LPJG_ini_date=
+ LPJG_ini_date=
LPJG_ini_member=""
+ LPJG_ini_member=
LPJG_STATE_DIR=
+ LPJG_STATE_DIR=
[ -z $LPJG_STATE_DIR ] && LPJG_STATE_DIR=""
+ '[' -z ']'
+ LPJG_STATE_DIR=
LPJG_STATE_DIR_INI=TRUE
+ LPJG_STATE_DIR_INI=TRUE
[ -z $LPJG_STATE_DIR_INI ] && LPJG_STATE_DIR_INI="TRUE"
+ '[' -z TRUE ']'

[[ "FALSE" == TRUE ]] && TM5=TRUE || TM5=FALSE
+ [[ FALSE == TRUE ]]
+ TM5=FALSE
TM5_ini=a2c0
+ TM5_ini=a2c0
TM5_ini_date=
+ TM5_ini_date=
TM5_ini_member="fc0"
+ TM5_ini_member=fc0
TM5_NLEVS=34
+ TM5_NLEVS=34
TM5_CONFIG="tm5:chem,o3fb,ch4fb,aerfb"
+ TM5_CONFIG=tm5:chem,o3fb,ch4fb,aerfb

#need vars for MEMBER generation/nudging
START_date=19931101
+ START_date=19931101
START_date_1=19931031
+ START_date_1=19931031
MEMBER=fc0
+ MEMBER=fc0
MODEL=ecearth
+ MODEL=ecearth
VERSION=trunk
+ VERSION=trunk
PROJDIR=$ROOTDIR/$PROJDEST
+ PROJDIR=/gpfs/scratch/bsc32/bsc32627/t0ni/auto-ecearth3

#NEM_FORCING_SET=CoreII
nem_forcing_set=CoreII
+ nem_forcing_set=CoreII
nem_fixed_forcing_year=-1
+ nem_fixed_forcing_year=-1

#CHUNK number
CHUNK=1
+ CHUNK=1
#size of each CHUNK
CHUNKSIZE=1
+ CHUNKSIZE=1
#number of chunks of the experiment
num_chunks=2
+ num_chunks=2
#unit of the chuk size. Can be hour, day, month or year.
chunk_size_unit=month
+ chunk_size_unit=month
Chunk_start_date=19931101
+ Chunk_start_date=19931101
Chunk_end_date=19931130
+ Chunk_end_date=19931130

#Ocean vars (surface regeneration)
SURF_RESTO=FALSE
+ SURF_RESTO=FALSE
SURF_RESTO_DATA=s4_surfresto
+ SURF_RESTO_DATA=s4_surfresto
SURF_RESTO_DATA_member=all_members
+ SURF_RESTO_DATA_member=all_members
SURF_RESTO_MASK=DEFAULT
+ SURF_RESTO_MASK=DEFAULT

OCE_NUDG=FALSE
+ OCE_NUDG=FALSE
OCE_NUDG_DATA=
+ OCE_NUDG_DATA=
OCE_NUDG_DATA_member=
+ OCE_NUDG_DATA_member=
OCE_NUDG_COEFF=
+ OCE_NUDG_COEFF=

#Ini generation flags
DELETE_INI_DIR_ON_INI=TRUE
+ DELETE_INI_DIR_ON_INI=TRUE
FORCE_RUN_FROM_SCRATCH=TRUE
+ FORCE_RUN_FROM_SCRATCH=TRUE
USE_INTERMEDIATE_STORAGE=FALSE
+ USE_INTERMEDIATE_STORAGE=FALSE
DT_HOST=
+ DT_HOST=
DT_USER=
+ DT_USER=

[ "ifs" == "" ] && ifs_veg_source="era20c" || ifs_veg_source="ifs"
+ '[' ifs == '' ']'
+ ifs_veg_source=ifs
[[ "FALSE" == TRUE ]] && DEBUG_MODE=TRUE || DEBUG_MODE=FALSE
+ [[ FALSE == TRUE ]]
+ DEBUG_MODE=FALSE

export HPCPROJ
+ export HPCPROJ

# Setup basic configuration for the choosen platform
. ${PROJDIR}/platforms/${CURRENT_ARCH}/configure.sh
+ . /gpfs/scratch/bsc32/bsc32627/t0ni/auto-ecearth3/platforms/marenostrum4/configure.sh
#####################################################################################################################
# Globals:
#   PROJDIR
# Arguments:
#   None
# Returns:
#   None
# Purpose: main function to prepare execution environment for the underlying platform, that basically consist on
#   setting up the modules to be used, common environment variables and the common paths
#####################################################################################################################
function load_platform_environment() {
  # load utils/filesystem libraries for common env functions
  . ${PROJDIR}/platforms/marenostrum4/filesystem.sh
  . ${PROJDIR}/platforms/marenostrum4/utils.sh
  #load global environment variables
  setup_modules
  #configure all modules needed for run the job
  setup_environment_vars
  # define all paths needed
  setup_paths
}

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: Prepare the modules needed to execute ece2cmor component for Marenostrum4 platform
#####################################################################################################################
function load_modules_ece2cmor() {
  set +xuve
  module purge
  module load impi/2017.4 intel/2017.4 mkl/2017.4 python/2.7.16 CMOR/3.5.0 CDO/1.8.2 gsl/2.4 nco/4.6.7 git eccodes/2.8.0
  set -xuve
}

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: Prepare the modules needed to execute the OASIS restart generation script for Marenostrum4 platform
#####################################################################################################################
function load_modules_oas_rsts_gen() {
  set +xuve
  module load hdf5/1.8.19_serial
  module load netcdf/4.4.1.1_serial
  module load udunits/2.2.25
  module load gsl/2.4
  module load nco/4.6.7
  module load CDO
  set -xuve
}

#####################################################################################################################
# Globals:
#  PATH, PROJDIR
# Arguments:
#   None
# Returns:
#   None
# Purpose: define the global vars with the path to the grib modules needed to run the ece2cmor module
#####################################################################################################################
function define_paths_ece2cmor() {
  grib_api_path=/apps/ECCODES/2.8.0/INTEL/
  PATH=${grib_api_path}/bin:${PATH}
  export GRIB_BIN_PATH=${grib_api_path}/bin
  export GRIB_DEFINITION_PATH=${PROJDIR}/sources/sources/util/grib_table_126:${grib_api_path}/share/eccodes/definitions
  export GRIB_SAMPLES_PATH=${grib_api_path}/share/eccodes/ifs_samples/grib1
}
# Dealing with initial conditions (plugin)
. ${PROJDIR}/plugins/utils.sh
+ . /gpfs/scratch/bsc32/bsc32627/t0ni/auto-ecearth3/plugins/utils.sh


#####################################################################################################################
# blocking_rsync using dt_commands
# Globals:
#   None
# Arguments:
#   options, source, destination, target_group USE_DT_COMMANDS
# Returns:
#   None
#####################################################################################################################
function blocking_rsync() {
  options=$1
  source=$2
  destination=$3
  target_group=$4
  if [[ -z ${USE_DT_COMMANDS-} ]]; then USE_DT_COMMANDS=TRUE; fi

  if [[ ${USE_DT_COMMANDS} == "FALSE" ]]; then
    sg ${target_group} "rsync ${options} ${source} ${destination}"
  else
    t1=$(date +%s)
    jid=$(dtsgrsync ${target_group} $options $source $destination | awk '{print $4}')
    if [[ -z ${jid} ]]; then
      USE_DT_COMMANDS=FALSE
      blocking_rsync "${options}" ${source} ${destination} ${target_group}
    else
      jsc=$(dtq -j $jid | wc -l)
      jst=$(dtq -j $jid | awk '{print $4}' | tail -1)
      echo "The job $jid is $jst"

      while [[ $jsc -gt 1 && $jst != "STATE" ]]; do
        sleep ${sleep_time:-60}
        jsc=$(dtq -j $jid | wc -l)
        jst=$(dtq -j $jid | awk '{print $4}' | tail -1)
        echo "The job $jid is $jst"
      done

      t2=$(date +%s)
      tr=$(date -d "0 -$t1 sec + $t2 sec" +%T)

      # exit ok
      # .err empty and .err file exists
      if [[ -f dtsgrsync_${jid}.err ]] && [[ ! -s dtsgrsync_${jid}.err ]]; then
        rm -f dtsgrsync_${jid}*
        return
      else
        echo "the "dtsgrsync_${jid}.err " is not empty"
        cat dtsgrsync_${jid}.err
        exit 1
      fi
    fi
  fi

}

# Functions to get the grid size to be used for CDO and S2DV
#
# Written by Pablo Echevarria
#
# Barcelona Supercomputing Center - Earth Science

#####################################################################################################################
# Get grid dims
# Globals:  NEMO_GRID
# Arguments:
#   TEMPLATE NEMGRID IFSGRID
# Returns:
#   GRID_S2DV GRID_CDO
#####################################################################################################################
function get_grid_dims() {

  local MODEL=${TEMPLATE/3/}

  NEMO_GRID=$(echo $NEMGRID | awk -F\L '{print $1}')

  if [[ $MODEL == 'ecearth' ]] || [[ $MODEL == 'ifs' ]] || [[ $MODEL == 'lsm' ]]; then
    GRID_POST=$IFSGRID
    case ${IFSGRID} in
    'T159L62')
      GRID_CDO='t106grid'
      GRID_PLOT='320x160'
      ;; # For cdo, the T159 is the t106
      # Correspondance betweeen reduced and regular gaussian grid
    'T255L91' | 'T255L62') GRID_CDO='t170grid' GRID_PLOT='512x256' ;;
    'T511L91') GRID_CDO='t340grid' GRID_PLOT='1024x512' ;;
    'T799L62' | 'T799L91')
      echo "Is the mask ready for this configuration?"
      stop
      ;;
    *)
      echo "IFS grid is not defined"
      exit 1
      ;;
    esac

  elif [[ $MODEL == 'nemo' ]]; then
    case $NEMO_GRID in
    'ORCA2')
      GRID_POST='T159L62'
      GRID_CDO='t106grid'
      GRID_PLOT='320x160'
      ;;
    'ORCA1')
      GRID_POST='T255L91'
      GRID_CDO='t170grid'
      GRID_PLOT='512x256'
      ;;
    'ORCA025')
      GRID_POST='T511L91'
      GRID_CDO='t340grid'
      GRID_PLOT='1024x512'
      ;;
    *)
      echo "Nemo grid is not defined"
      exit 1
      ;;
    esac
  else
    echo "Model is not defined"
    exit 1
  fi
  export GRID_POST
  export GRID_CDO
  export GRID_PLOT

}

#####################################################################################################################
# generate a checksum and store it in a file for the inidata folder
# Globals:
#   None
# Arguments:
#   Checksum_filename
# Returns:
#   None
#####################################################################################################################
function inidata_checksum_generate() {

  filename=$1

  if [[ "$filename" == "" ]]; then
    echo "Filename argument is required to store the checksum results, exiting now"
    exit 1
  fi

  # this generates the md5sum for every file present in the experiment inidata folder and a global md5sum of the complete list
  echo "Generating list of files (and md5sum of each) in the directory, time taken:"
  files_time="$(time (find inidata/ -type f -exec md5sum {} \; | sort -k 2 &> ${filename}_tmpl_files.txt ) 2>&1 1>/dev/null )"
  echo "${files_time}"

  # this creates a list of the paths of all the links and files and generates the md5sum for that list
  echo "Generating list of links in the directory, time taken:"
  links_time="$(time (find -L inidata/ -type f -exec readlink -f {} \; | sort &> ${filename}_tmpl_links.txt ) 2>&1 1>/dev/null )"
  echo "${links_time}"
}
. ${PROJDIR}/platforms/${HPCARCH}/filesystem.sh
+ . /gpfs/scratch/bsc32/bsc32627/t0ni/auto-ecearth3/platforms/marenostrum4/filesystem.sh
#####################################################################################################################
# Functions for handling filesystem operations for marenostrum4 platform
# Author: J.R.Berlin
#####################################################################################################################

. ${PROJDIR}/platforms/common/common.filesystem.sh
++ . /gpfs/scratch/bsc32/bsc32627/t0ni/auto-ecearth3/platforms/common/common.filesystem.sh
#####################################################################################################################
# Common Functions for handling file system operations on the different host types
# of the supported machines
# Author/Maintainer: J.R.Berlin
#####################################################################################################################

#####################################################################################################################
# Executes mkdir + chmod to handle proper rights in the target folder in esarchive partition
# Globals:
#   None
# Arguments:
#   target_dir
# Returns:
#   None
#####################################################################################################################
function mkdir_esarchive() {
  target_dir=$1
  mkdir -p ${target_dir}
  chmod_esarchive ${target_dir}
}

#####################################################################################################################
# Executes chmod with g+s so users of a given group doesnt have access problems to certain outputs and files
# Globals:
#   None
# Arguments:
#   target_dir
# Returns:
#   None
#####################################################################################################################
function chmod_esarchive() {
  target_dir=$1
  chmod =0770,g+s ${target_dir}
}


#####################################################################################################################
# Globals:
# RUN_dir,  SCRATCH_TMP_DIR, MEMBER, out_member
# Arguments:
#   None
# Returns:
#   None
# Purpose: creates the folder passed as parameter and assign proper group rights
#
#####################################################################################################################
function mkdir_intermediate_storage() {
  #default group
  target_group=bsc32
  target_dir=$1
  if [[ "$#" -eq 2 ]]; then
    target_group=$2
  fi

  if [[ ! -d "${target_dir}" ]]; then
    sg ${target_group} "mkdir -p ${target_dir}"
    chmod g+w ${target_dir}
  fi
}

#####################################################################################################################
# Globals:
#   None,
# Arguments:
#   $1: the folder to be removed
# Returns:
#   None
# Purpose: removes the intermediate storage folder passed as parameter
#
#####################################################################################################################
function delete_intermediate_storage() {
  rm -rf $1 2>/dev/null
}

#####################################################################################################################
# Globals:
#   None,
# Arguments:
#   $1: the source folder/file to be copied
#   $2: the target directory
# Returns:
#   None
# Purpose: copies the intermediate storage folder passed as parameter
#
#####################################################################################################################
function copy_intermediate_storage() {
  cp -f $1 $2
}

#####################################################################################################################
# Globals:
#   None,
# Arguments:
#   $1: the ecm folder whose content we want to enumerate
# Returns:
#   None
# Purpose: enumerate the content of the intermediate storage folder passed as parameter
#
#####################################################################################################################
function list_intermediate_storage() {
  ls $1
}

#####################################################################################################################
# Globals:
#   None,
# Arguments:
#   $1: the folder whose permissions we want to change
# Returns:
#   None
# Purpose: change the permissions of the intermediate storage folder passed as parameter
#
#####################################################################################################################
function chmod_intermediate_storage() {
  chmod 744 $1
}

#####################################################################################################################
# Globals:
#   None,
# Arguments:
#   $1: the folder we want to test
# Returns:
#   None
# Purpose: test the intermediate storage folder passed as parameter
#
#####################################################################################################################
function test_intermediate_storage() {
  test $1
}

# check if directory is empty and doesnt contain any file
#####################################################################################################################
# Globals:
# RUN_dir,  SCRATCH_TMP_DIR, MEMBER, out_member
# Arguments:
#   None
# Returns:
#   None
# Purpose: Main function for initialization - ifs3
#
#####################################################################################################################
function is_empty_directory() {
  local target_path=$1
  local result=false
  if [[ -z "$(ls -A ${target_path})" ]]; then
    result=true
  fi
  return result
}

#####################################################################################################################
# Globals:
# RUN_dir,
# Arguments:
#   source= source directory where are the files to be synchronized
#   destination=path to the directory that will be synchronized with the source dir
#   flags=options for the rsync
# Returns:
#   None
# Purpose: synchronizes two directories from different file systems
#          depending of the context, dt commands will be used by using blocking_rsync,
#          and rsync otherwise, <how_many_retry_copy_file> attempts will allowed to try to transfer the files in case
#          of failure
#####################################################################################################################
function rsync_with_retry() {

  source=$2
  destination=$3
  flags=$1
  group=$4
  retry=0
  how_many_retry_copy_file=5
  use_dt_rsync=FALSE

  exist_dtrsync=$(echo $(command -v dtrsync))
  if [[ ! -z "${exist_dtrsync-}" ]]; then
    use_dt_rsync=TRUE
  fi

  while [[ $retry -le $how_many_retry_copy_file ]]; do

    t1=$(date +%s)
    if [[ "${use_dt_rsync-}" == "FALSE" ]]; then
      USE_DT_COMMANDS=FALSE
      echo "Calling blocking_rsync with DT_COMMANDS=FALSE since they are not available on this machine"
    fi
    blocking_rsync "${flags}" ${source} ${destination} ${group}

    if [[ $? ]]; then
      t2=$(date +%s)
      tr=$(date -d "0 -$t1 sec + $t2 sec" +%T)
      echo "done."
      break
    else
      ((retry++))
    fi
  done
  if [[ $retry -eq how_many_retry_copy_file ]]; then
    echo "Transfer error in rsync"
    exit 1
  fi
}

#####################################################################################################################
# Globals:
# PROJDIR, target_destination, REMOTE_PATH
# Arguments:
#   None
# Returns:
#   None
# Purpose: synchronizes the auto-ecearth source folder with changes done in autosubmit auto-ecearth folder of the
# experiment
#
#####################################################################################################################
function transfer_model() {
  local folders_that_will_not_be_transferred=""
  . ${PROJDIR}/plugins/utils.sh
  source=${PROJDIR}
  destination=${target_destination}
  exclusions="--exclude={'templates','utils','test_cases','*.o','*.mod','*.exe','/BLD','ifs-36r4/bin','ifs-36r4/lib/*.a','COMPILE/arch*','COMPILE/cpp*','lpjg/build'}"
  options="${exclusions} --no-group --delete --force --recursive --links --perms --times --specials --verbose"
  if [[ "$#" -eq 1 ]]; then
    folders_that_will_not_be_transferred="$1"
    for folder in ${folders_that_will_not_be_transferred-}; do
      options+=" --exclude='${source}/${folder}' "
    done
  fi
  group=$(id -gn)
  rsync_with_retry "${options}" ${source} "${REMOTE_PATH}${destination}" ${group}
}

#####################################################################################################################
# Globals:
# RUN_dir,
# Arguments:
#   filenames: the file or list of files to be moved to the target location,
#   target_dir: the target directory where the files will be moved
#   is_transfer: defines if the target filesystem is the intermediate storage or esarchive ( permanent storage)
# Returns:
#   None
# Purpose: move files across different file systems, used mainly in CLEAN and TRANSFER jobs, this is the base function,
# each platform can have its own custom implementation if needed ( for instance ecmwf-sc40 has its own version of this
# function, the moved files are removed from the source location
#
#####################################################################################################################
function move_files() {
  local filenames=$1
  local target_dir=$2
  local is_transfer=$3

  if [[ "${is_transfer-}" == "TRUE" ]]; then
    # we are transferring from Scratch to Esarchive directly
    blocking_rsync '--no-perms --links --times --recursive --remove-source-files --no-group  --omit-dir-times --chmod=g+rwxs,o-rwx' "${filenames}" ${target_dir} Earth
  else
    files_to_send=$(echo "${filenames}")
    rsync --perms --links --times --recursive --remove-source-files --no-group  --omit-dir-times --chmod=g+rwxs,o-rwx ${files_to_send} ${target_dir}
  fi
  # check if there were errors
  if [[ $? != 0 ]]; then
    echo "Error occurred on move_files....exiting"
    exit 1
  fi
}

#####################################################################################################################
# Globals:
# RUN_dir, ini_data_dir
# Arguments:
#   None
# Returns:
#   None
# Purpose: copy the initialization files to inidata directory in the scratch directory of the experiment, the command
#   creates links to the files instead of copying these physically for optimizing storage
#
#####################################################################################################################
function copy_inidata() {
  cp --symbolic-link --recursive --force --update ${ini_data_dir} ${RUN_dir}
}

#####################################################################################################################
# Globals:
#
# Arguments:
#   src: source folder for the needed restarts
#   dst: target location where these will be copied
# Returns:
#   None
# Purpose: prepare restarts to be linked in to the inidata folder
#####################################################################################################################
function copy_default_restarts() {
  local src=$1
  local dst=$2
  cp --symbolic-link --recursive --force --update ${src} ${dst}
}

#####################################################################################################################
# Globals:
# RUN_dir, SCRATCH_TMP_DIR, MEMBER, ROOTDIR
# Arguments:
#   None
# Returns:
#   None
# Purpose: clean up the temporary directories used and set run dir, used in almost all platforms
#
#####################################################################################################################
function setup_paths_transfer() {
  rm -rf ${SCRATCH_TMP_DIR}
  mkdir -p ${SCRATCH_TMP_DIR}
  RUN_dir=${ROOTDIR}/${START_date}/${MEMBER}/runtime
}

#####################################################################################################################
# Globals:
# SCRATCH_DIR, HPCPROJ, HPCUSER, EXPID,PROJDEST
# Arguments:
#   None
# Returns:
#   None
# Purpose:  check the existence of the auto-ecearth folder in the experiment folder in the cluster
#
#####################################################################################################################
function check_model_existence() {
  SCRATCH_MODEL_PATH=${SCRATCH_DIR}/${HPCPROJ}/${HPCUSER}/${EXPID}/${PROJDEST}
  MODEL_EXISTS="FALSE"
  if [[ -d ${SCRATCH_MODEL_PATH} ]]; then
    MODEL_EXISTS="TRUE"
    export MODEL_EXISTS
  fi
}

#####################################################################################################################
# Globals:
# PROJDIR, SCRATCH_DIR, HPCPROJ, HPCUSER, EXPID, HPCARCH
# Arguments:
#   None
# Returns:
#   None
# Purpose: prepare/load needed libraries in order to execute the synchronize process
#
#####################################################################################################################
function setup_synchronize() {
  export target_destination=$SCRATCH_DIR/$HPCPROJ/$HPCUSER/$EXPID/
  export REMOTE_PATH=' '
  # import custom architecture
  . ${PROJDIR}/platforms/${HPCARCH}/filesystem.sh
}

#####################################################################################################################
# Globals:
# RUN_dir, USE_REMOTE_HOST, HPCUSER,  HPCHOST
# Arguments:
#    source_cmor_dir= source location for the cmorized files , usually the folder that corresponds to the chunk folder
#    target_cmor_dir= target destination where the files will be moved
#    is_transfer= indicates if the temporary storage will be used or not
# Returns:
#   None
# Purpose: upper level function to transfer the cmorized outputs either in CLEAN ( when USE_INTERMEDIATE_STORAGE=FALSE )
#    or the standard TRANSFER, this is the base function and other platforms may have different implementations such as
#    ecmwf-xc40
#####################################################################################################################
function move_cmor_files() {
  local source_cmor_dir=$1
  local target_cmor_dir=$2
  local is_transfer=$3
  local remote_host=' '

  # used in bscearth000 machine, maybe wise to create a function in platforms for this particular case
  if [[ "${USE_REMOTE_HOST-}" == "TRUE" ]]; then
    cmor_folders=$(ssh ${HPCUSER}@${HPCHOST} "ls -d ${source_cmor_dir}/cmor_*_${CHUNK}/" 2>/dev/null) && returncode=$? || returncode=$?
    if [[ ! -z "${cmor_folders-}" && ${returncode} -eq 0 ]]; then
      for cmor_remote_dir in ${cmor_folders}; do
        list_of_cmor_files=$(ssh ${HPCUSER}@${HPCHOST} "ls ${cmor_remote_dir}" 2>/dev/null) && returncode=$? || returncode=$?
        if [[ ! -z "${list_of_cmor_files-}" && ${returncode} -eq 0 ]]; then
          echo "moving cmor files ....."
          move_files ${HPCUSER}@${HPCHOST}:${cmor_remote_dir} ${target_cmor_dir} ${is_transfer}
          # remove cmor folder after moving the files
          # if there where some files left behind, this method will throw an exception making the job to crash
          ssh ${HPCUSER}@${HPCHOST} "find ${cmor_remote_dir} -type d -delete"
        fi
      done
    fi
  else
    if [[ -d ${source_cmor_dir} ]]; then
      echo "moving cmor files ....."
      cmor_folders=$(ls -d ${source_cmor_dir}/cmor_*_${CHUNK}/ 2>/dev/null) && returncode=$? || returncode=$?
      if [[ ! -z "${cmor_folders-}" && ${returncode} -eq 0 ]]; then
        base_storage_cmor_target_dir=${target_cmor_dir}
        for cmor_remote_dir in ${cmor_folders}; do
          #defaults when we use intermediate storage structure must be the same in the intermediate storage
          if [[ "${is_transfer-}" != "TRUE" ]]; then
            target_cmor_dir=${base_storage_cmor_target_dir}/$(basename ${cmor_remote_dir})
          fi
          move_files ${cmor_remote_dir} ${target_cmor_dir} ${is_transfer}
          # delete the folder, if there where some files left behind, this method will throw an exception making the job to crash
          find ${cmor_remote_dir} -type d -delete
        done
      fi
    fi
  fi
}

#####################################################################################################################
# Globals:
# RUN_dir, ROOTDIR, EXPID
# Arguments:
#   source_pattern: is the destination path for the output
#   destination_path: is the name of the final folder of the destination path /
#   list_of_files_to_copy: list of files to copy
# Returns:
#   None
# Purpose: Local transfer of model output, the function first retrieve the list of files to be transferred and then
#     if such file is not empty proceed to transfer them
#####################################################################################################################
function do_transfer() {

  #  to leave the dt command output in log folder
  cd ${ROOTDIR}/LOG_${EXPID}

  source_pattern=$1
  destination_path=$2
  source_path=$(dirname ${source_pattern[0]})
  local remote_host=' '

  get_list_of_files_to_copy && returncode=$? || returncode=$?
  #apply mkdir and let the rsync apply the chmod
  if [[ ! -z "${list_of_files_to_copy-}" ]]; then
    mkdir -p ${destination_path}
    for source_file_name in $list_of_files_to_copy; do
      echo "transferring file ... "${source_file_name} ${destination_path}
      blocking_rsync '--no-perms --links --times --recursive --remove-source-files --no-group  --omit-dir-times --chmod=g+rwxs,o-rwx' ${remote_host}${source_file_name} ${destination_path} Earth
    done
  fi
}

#####################################################################################################################
# Globals:
# USE_REMOTE_HOST, HPCUSER, HPCHOST, source_pattern
# Arguments:
#   None
# Returns:
#   None
# Purpose: get the list of files to be copied to the cluster, used by transfer/clean templates
#
#####################################################################################################################
function get_list_of_files_to_copy() {
  if [[ "${USE_REMOTE_HOST-}" == "TRUE" ]]; then
    list_of_files_to_copy=$(ssh ${HPCUSER}@${HPCHOST} "ls ${source_pattern}" 2>/dev/null)
    remote_host=${HPCUSER}@${HPCHOST}:
  else
    list_of_files_to_copy=$(ls ${source_pattern} 2>/dev/null)
  fi
}

#####################################################################################################################
# Globals:
# RUN_dir,
# Arguments:
#   None
# Returns:
#   None
# Purpose: performs the synchronization by transferring the model files to the cluster for preparation before running
# the experiment, is the common interface for the synchronization template, implementation may be different depending
# on the underlying architecture where the sync needs to be executed
#
#####################################################################################################################
function do_synchronize() {
  folders_that_will_not_be_transferred=""
  if [[ "$#" -eq 1 ]] ; then
    folders_that_will_not_be_transferred="$1"
  fi
  echo "Calling function transfer_model"
  transfer_model ${folders_that_will_not_be_transferred}
  echo "common.synchronize Done"
}

#####################################################################################################################
# Globals:
# RUN_dir, INTERMEDIATE_EXP_DIR, MODEL, EXPID
# Arguments:
#   None
# Returns:
#   None
# Purpose: Prepare source and target paths for CLEAN/TRANSFER
#####################################################################################################################
function setup_paths_localtrans() {
  src=${INTERMEDIATE_EXP_DIR}
  local_INTERMEDIATE_EXP_DIR=/gpfs/archive/bsc32/esarchive/exp/${MODEL}
  destination_folder=${local_INTERMEDIATE_EXP_DIR}/${EXPID}/original_files/
}

# copy to scratch the platforms files
#####################################################################################################################
# Globals:
# ROOTDIR, PROJDEST, PROJDIR
# Arguments:
#   None
# Returns:
#   None
# Purpose: send the initial set of libraries used by the synchronize job in order to be able to run
#
#####################################################################################################################
function copy_platform_environment_to_host() {
  mkdir -p ${ROOTDIR}/${PROJDEST}/platforms
  rsync --recursive --links --perms --times --owner --devices --specials --verbose --omit-dir-times ${PROJDIR}/platforms/ ${ROOTDIR}/${PROJDEST}/platforms
  rsync --recursive --links --perms --times --owner --devices --specials --verbose --omit-dir-times ${PROJDIR}/plugins/ ${ROOTDIR}/${PROJDEST}/plugins
}

#####################################################################################################################
# Globals:
# RUN_dir,  LOGDIR, CHUNK, START_date, MEMBER, STAMP, PATHOUT
# Arguments:
#   None
# Returns:
#   None
# Purpose: copy logs to the proper storage location before wiping out the experiment folder ( used by CLEAN_EXPERIMENT )
#####################################################################################################################
function save_logs() {
  echo "We are now storing log files"
  cd ${LOGDIR}
  logs_tarfile="logfiles_${EXPID}_${STAMP}.tar"
  IS_TRANSFER=TRUE
  prepare_logs && returncode=$? || returncode=$?
  if [[ ${returncode} -eq 0 ]] || [[ ${returncode} -eq 123 ]]; then
    [[ ! -f ../${logs_tarfile}.gz ]] && gzip -9 ../${logs_tarfile}
    local pathout_logfiles="${PATHOUT}/logfiles/"
    move_files ../${logs_tarfile}.gz ${pathout_logfiles} ${IS_TRANSFER}
    rm -f ../${logs_tarfile}.gz
  fi
}

#####################################################################################################################
# Globals:
# INTERMEDIATE_EXP_DIR, EXPID, ROOTDIR, RM ( alias for deleting files )
# Arguments:
#   None
# Returns:
#   None
# Purpose: wipe out the experiment folders (used intermediate storage, scratch)
#####################################################################################################################
function remove_experiment_folders() {
  #remove the folder in intermediate storage (if it was used)
  delete_intermediate_storage ${INTERMEDIATE_EXP_DIR}/${EXPID}
  #delete scratch folder
  rm -rf ${ROOTDIR}/*
  # needed for indicating that the job finished successfully by autosubmit
  mkdir -p ${ROOTDIR}/LOG_${EXPID}/
}

#####################################################################################################################
# any override from the common libraries defined in filesystem.sh or new function goes here
#####################################################################################################################

#####################################################################################################################
# Globals:
#   EXPID, MODEL, INTERMEDIATE_EXP_DIR, EXPID, USE_INTERMEDIATE_STORAGE
# Arguments:
#   None
# Returns:
#   None
# Purpose: template function for Marenostrum4 for setting up common paths ( used as setup_paths_transfer_${HPCARCH} )
#
#####################################################################################################################
function setup_paths_transfer_marenostrum4() {
  #base function
  setup_paths_transfer
  #custom settings
  if [[ "${USE_INTERMEDIATE_STORAGE-}" == "TRUE" ]]; then
    PATHOUT="${INTERMEDIATE_EXP_DIR}/${EXPID}/"
  else
    local_INTERMEDIATE_EXP_DIR=/gpfs/archive/bsc32/esarchive/exp/${MODEL}
    PATHOUT="${local_INTERMEDIATE_EXP_DIR}/${EXPID}/original_files/"
  fi
}

#prepare running environment
load_platform_environment
+ load_platform_environment
+ . /gpfs/scratch/bsc32/bsc32627/t0ni/auto-ecearth3/platforms/marenostrum4/filesystem.sh
#####################################################################################################################
# Functions for handling filesystem operations for marenostrum4 platform
# Author: J.R.Berlin
#####################################################################################################################

. ${PROJDIR}/platforms/common/common.filesystem.sh
++ . /gpfs/scratch/bsc32/bsc32627/t0ni/auto-ecearth3/platforms/common/common.filesystem.sh
#####################################################################################################################
# Common Functions for handling file system operations on the different host types
# of the supported machines
# Author/Maintainer: J.R.Berlin
#####################################################################################################################

#####################################################################################################################
# Executes mkdir + chmod to handle proper rights in the target folder in esarchive partition
# Globals:
#   None
# Arguments:
#   target_dir
# Returns:
#   None
#####################################################################################################################
function mkdir_esarchive() {
  target_dir=$1
  mkdir -p ${target_dir}
  chmod_esarchive ${target_dir}
}

#####################################################################################################################
# Executes chmod with g+s so users of a given group doesnt have access problems to certain outputs and files
# Globals:
#   None
# Arguments:
#   target_dir
# Returns:
#   None
#####################################################################################################################
function chmod_esarchive() {
  target_dir=$1
  chmod =0770,g+s ${target_dir}
}


#####################################################################################################################
# Globals:
# RUN_dir,  SCRATCH_TMP_DIR, MEMBER, out_member
# Arguments:
#   None
# Returns:
#   None
# Purpose: creates the folder passed as parameter and assign proper group rights
#
#####################################################################################################################
function mkdir_intermediate_storage() {
  #default group
  target_group=bsc32
  target_dir=$1
  if [[ "$#" -eq 2 ]]; then
    target_group=$2
  fi

  if [[ ! -d "${target_dir}" ]]; then
    sg ${target_group} "mkdir -p ${target_dir}"
    chmod g+w ${target_dir}
  fi
}

#####################################################################################################################
# Globals:
#   None,
# Arguments:
#   $1: the folder to be removed
# Returns:
#   None
# Purpose: removes the intermediate storage folder passed as parameter
#
#####################################################################################################################
function delete_intermediate_storage() {
  rm -rf $1 2>/dev/null
}

#####################################################################################################################
# Globals:
#   None,
# Arguments:
#   $1: the source folder/file to be copied
#   $2: the target directory
# Returns:
#   None
# Purpose: copies the intermediate storage folder passed as parameter
#
#####################################################################################################################
function copy_intermediate_storage() {
  cp -f $1 $2
}

#####################################################################################################################
# Globals:
#   None,
# Arguments:
#   $1: the ecm folder whose content we want to enumerate
# Returns:
#   None
# Purpose: enumerate the content of the intermediate storage folder passed as parameter
#
#####################################################################################################################
function list_intermediate_storage() {
  ls $1
}

#####################################################################################################################
# Globals:
#   None,
# Arguments:
#   $1: the folder whose permissions we want to change
# Returns:
#   None
# Purpose: change the permissions of the intermediate storage folder passed as parameter
#
#####################################################################################################################
function chmod_intermediate_storage() {
  chmod 744 $1
}

#####################################################################################################################
# Globals:
#   None,
# Arguments:
#   $1: the folder we want to test
# Returns:
#   None
# Purpose: test the intermediate storage folder passed as parameter
#
#####################################################################################################################
function test_intermediate_storage() {
  test $1
}

# check if directory is empty and doesnt contain any file
#####################################################################################################################
# Globals:
# RUN_dir,  SCRATCH_TMP_DIR, MEMBER, out_member
# Arguments:
#   None
# Returns:
#   None
# Purpose: Main function for initialization - ifs3
#
#####################################################################################################################
function is_empty_directory() {
  local target_path=$1
  local result=false
  if [[ -z "$(ls -A ${target_path})" ]]; then
    result=true
  fi
  return result
}

#####################################################################################################################
# Globals:
# RUN_dir,
# Arguments:
#   source= source directory where are the files to be synchronized
#   destination=path to the directory that will be synchronized with the source dir
#   flags=options for the rsync
# Returns:
#   None
# Purpose: synchronizes two directories from different file systems
#          depending of the context, dt commands will be used by using blocking_rsync,
#          and rsync otherwise, <how_many_retry_copy_file> attempts will allowed to try to transfer the files in case
#          of failure
#####################################################################################################################
function rsync_with_retry() {

  source=$2
  destination=$3
  flags=$1
  group=$4
  retry=0
  how_many_retry_copy_file=5
  use_dt_rsync=FALSE

  exist_dtrsync=$(echo $(command -v dtrsync))
  if [[ ! -z "${exist_dtrsync-}" ]]; then
    use_dt_rsync=TRUE
  fi

  while [[ $retry -le $how_many_retry_copy_file ]]; do

    t1=$(date +%s)
    if [[ "${use_dt_rsync-}" == "FALSE" ]]; then
      USE_DT_COMMANDS=FALSE
      echo "Calling blocking_rsync with DT_COMMANDS=FALSE since they are not available on this machine"
    fi
    blocking_rsync "${flags}" ${source} ${destination} ${group}

    if [[ $? ]]; then
      t2=$(date +%s)
      tr=$(date -d "0 -$t1 sec + $t2 sec" +%T)
      echo "done."
      break
    else
      ((retry++))
    fi
  done
  if [[ $retry -eq how_many_retry_copy_file ]]; then
    echo "Transfer error in rsync"
    exit 1
  fi
}

#####################################################################################################################
# Globals:
# PROJDIR, target_destination, REMOTE_PATH
# Arguments:
#   None
# Returns:
#   None
# Purpose: synchronizes the auto-ecearth source folder with changes done in autosubmit auto-ecearth folder of the
# experiment
#
#####################################################################################################################
function transfer_model() {
  local folders_that_will_not_be_transferred=""
  . ${PROJDIR}/plugins/utils.sh
  source=${PROJDIR}
  destination=${target_destination}
  exclusions="--exclude={'templates','utils','test_cases','*.o','*.mod','*.exe','/BLD','ifs-36r4/bin','ifs-36r4/lib/*.a','COMPILE/arch*','COMPILE/cpp*','lpjg/build'}"
  options="${exclusions} --no-group --delete --force --recursive --links --perms --times --specials --verbose"
  if [[ "$#" -eq 1 ]]; then
    folders_that_will_not_be_transferred="$1"
    for folder in ${folders_that_will_not_be_transferred-}; do
      options+=" --exclude='${source}/${folder}' "
    done
  fi
  group=$(id -gn)
  rsync_with_retry "${options}" ${source} "${REMOTE_PATH}${destination}" ${group}
}

#####################################################################################################################
# Globals:
# RUN_dir,
# Arguments:
#   filenames: the file or list of files to be moved to the target location,
#   target_dir: the target directory where the files will be moved
#   is_transfer: defines if the target filesystem is the intermediate storage or esarchive ( permanent storage)
# Returns:
#   None
# Purpose: move files across different file systems, used mainly in CLEAN and TRANSFER jobs, this is the base function,
# each platform can have its own custom implementation if needed ( for instance ecmwf-sc40 has its own version of this
# function, the moved files are removed from the source location
#
#####################################################################################################################
function move_files() {
  local filenames=$1
  local target_dir=$2
  local is_transfer=$3

  if [[ "${is_transfer-}" == "TRUE" ]]; then
    # we are transferring from Scratch to Esarchive directly
    blocking_rsync '--no-perms --links --times --recursive --remove-source-files --no-group  --omit-dir-times --chmod=g+rwxs,o-rwx' "${filenames}" ${target_dir} Earth
  else
    files_to_send=$(echo "${filenames}")
    rsync --perms --links --times --recursive --remove-source-files --no-group  --omit-dir-times --chmod=g+rwxs,o-rwx ${files_to_send} ${target_dir}
  fi
  # check if there were errors
  if [[ $? != 0 ]]; then
    echo "Error occurred on move_files....exiting"
    exit 1
  fi
}

#####################################################################################################################
# Globals:
# RUN_dir, ini_data_dir
# Arguments:
#   None
# Returns:
#   None
# Purpose: copy the initialization files to inidata directory in the scratch directory of the experiment, the command
#   creates links to the files instead of copying these physically for optimizing storage
#
#####################################################################################################################
function copy_inidata() {
  cp --symbolic-link --recursive --force --update ${ini_data_dir} ${RUN_dir}
}

#####################################################################################################################
# Globals:
#
# Arguments:
#   src: source folder for the needed restarts
#   dst: target location where these will be copied
# Returns:
#   None
# Purpose: prepare restarts to be linked in to the inidata folder
#####################################################################################################################
function copy_default_restarts() {
  local src=$1
  local dst=$2
  cp --symbolic-link --recursive --force --update ${src} ${dst}
}

#####################################################################################################################
# Globals:
# RUN_dir, SCRATCH_TMP_DIR, MEMBER, ROOTDIR
# Arguments:
#   None
# Returns:
#   None
# Purpose: clean up the temporary directories used and set run dir, used in almost all platforms
#
#####################################################################################################################
function setup_paths_transfer() {
  rm -rf ${SCRATCH_TMP_DIR}
  mkdir -p ${SCRATCH_TMP_DIR}
  RUN_dir=${ROOTDIR}/${START_date}/${MEMBER}/runtime
}

#####################################################################################################################
# Globals:
# SCRATCH_DIR, HPCPROJ, HPCUSER, EXPID,PROJDEST
# Arguments:
#   None
# Returns:
#   None
# Purpose:  check the existence of the auto-ecearth folder in the experiment folder in the cluster
#
#####################################################################################################################
function check_model_existence() {
  SCRATCH_MODEL_PATH=${SCRATCH_DIR}/${HPCPROJ}/${HPCUSER}/${EXPID}/${PROJDEST}
  MODEL_EXISTS="FALSE"
  if [[ -d ${SCRATCH_MODEL_PATH} ]]; then
    MODEL_EXISTS="TRUE"
    export MODEL_EXISTS
  fi
}

#####################################################################################################################
# Globals:
# PROJDIR, SCRATCH_DIR, HPCPROJ, HPCUSER, EXPID, HPCARCH
# Arguments:
#   None
# Returns:
#   None
# Purpose: prepare/load needed libraries in order to execute the synchronize process
#
#####################################################################################################################
function setup_synchronize() {
  export target_destination=$SCRATCH_DIR/$HPCPROJ/$HPCUSER/$EXPID/
  export REMOTE_PATH=' '
  # import custom architecture
  . ${PROJDIR}/platforms/${HPCARCH}/filesystem.sh
}

#####################################################################################################################
# Globals:
# RUN_dir, USE_REMOTE_HOST, HPCUSER,  HPCHOST
# Arguments:
#    source_cmor_dir= source location for the cmorized files , usually the folder that corresponds to the chunk folder
#    target_cmor_dir= target destination where the files will be moved
#    is_transfer= indicates if the temporary storage will be used or not
# Returns:
#   None
# Purpose: upper level function to transfer the cmorized outputs either in CLEAN ( when USE_INTERMEDIATE_STORAGE=FALSE )
#    or the standard TRANSFER, this is the base function and other platforms may have different implementations such as
#    ecmwf-xc40
#####################################################################################################################
function move_cmor_files() {
  local source_cmor_dir=$1
  local target_cmor_dir=$2
  local is_transfer=$3
  local remote_host=' '

  # used in bscearth000 machine, maybe wise to create a function in platforms for this particular case
  if [[ "${USE_REMOTE_HOST-}" == "TRUE" ]]; then
    cmor_folders=$(ssh ${HPCUSER}@${HPCHOST} "ls -d ${source_cmor_dir}/cmor_*_${CHUNK}/" 2>/dev/null) && returncode=$? || returncode=$?
    if [[ ! -z "${cmor_folders-}" && ${returncode} -eq 0 ]]; then
      for cmor_remote_dir in ${cmor_folders}; do
        list_of_cmor_files=$(ssh ${HPCUSER}@${HPCHOST} "ls ${cmor_remote_dir}" 2>/dev/null) && returncode=$? || returncode=$?
        if [[ ! -z "${list_of_cmor_files-}" && ${returncode} -eq 0 ]]; then
          echo "moving cmor files ....."
          move_files ${HPCUSER}@${HPCHOST}:${cmor_remote_dir} ${target_cmor_dir} ${is_transfer}
          # remove cmor folder after moving the files
          # if there where some files left behind, this method will throw an exception making the job to crash
          ssh ${HPCUSER}@${HPCHOST} "find ${cmor_remote_dir} -type d -delete"
        fi
      done
    fi
  else
    if [[ -d ${source_cmor_dir} ]]; then
      echo "moving cmor files ....."
      cmor_folders=$(ls -d ${source_cmor_dir}/cmor_*_${CHUNK}/ 2>/dev/null) && returncode=$? || returncode=$?
      if [[ ! -z "${cmor_folders-}" && ${returncode} -eq 0 ]]; then
        base_storage_cmor_target_dir=${target_cmor_dir}
        for cmor_remote_dir in ${cmor_folders}; do
          #defaults when we use intermediate storage structure must be the same in the intermediate storage
          if [[ "${is_transfer-}" != "TRUE" ]]; then
            target_cmor_dir=${base_storage_cmor_target_dir}/$(basename ${cmor_remote_dir})
          fi
          move_files ${cmor_remote_dir} ${target_cmor_dir} ${is_transfer}
          # delete the folder, if there where some files left behind, this method will throw an exception making the job to crash
          find ${cmor_remote_dir} -type d -delete
        done
      fi
    fi
  fi
}

#####################################################################################################################
# Globals:
# RUN_dir, ROOTDIR, EXPID
# Arguments:
#   source_pattern: is the destination path for the output
#   destination_path: is the name of the final folder of the destination path /
#   list_of_files_to_copy: list of files to copy
# Returns:
#   None
# Purpose: Local transfer of model output, the function first retrieve the list of files to be transferred and then
#     if such file is not empty proceed to transfer them
#####################################################################################################################
function do_transfer() {

  #  to leave the dt command output in log folder
  cd ${ROOTDIR}/LOG_${EXPID}

  source_pattern=$1
  destination_path=$2
  source_path=$(dirname ${source_pattern[0]})
  local remote_host=' '

  get_list_of_files_to_copy && returncode=$? || returncode=$?
  #apply mkdir and let the rsync apply the chmod
  if [[ ! -z "${list_of_files_to_copy-}" ]]; then
    mkdir -p ${destination_path}
    for source_file_name in $list_of_files_to_copy; do
      echo "transferring file ... "${source_file_name} ${destination_path}
      blocking_rsync '--no-perms --links --times --recursive --remove-source-files --no-group  --omit-dir-times --chmod=g+rwxs,o-rwx' ${remote_host}${source_file_name} ${destination_path} Earth
    done
  fi
}

#####################################################################################################################
# Globals:
# USE_REMOTE_HOST, HPCUSER, HPCHOST, source_pattern
# Arguments:
#   None
# Returns:
#   None
# Purpose: get the list of files to be copied to the cluster, used by transfer/clean templates
#
#####################################################################################################################
function get_list_of_files_to_copy() {
  if [[ "${USE_REMOTE_HOST-}" == "TRUE" ]]; then
    list_of_files_to_copy=$(ssh ${HPCUSER}@${HPCHOST} "ls ${source_pattern}" 2>/dev/null)
    remote_host=${HPCUSER}@${HPCHOST}:
  else
    list_of_files_to_copy=$(ls ${source_pattern} 2>/dev/null)
  fi
}

#####################################################################################################################
# Globals:
# RUN_dir,
# Arguments:
#   None
# Returns:
#   None
# Purpose: performs the synchronization by transferring the model files to the cluster for preparation before running
# the experiment, is the common interface for the synchronization template, implementation may be different depending
# on the underlying architecture where the sync needs to be executed
#
#####################################################################################################################
function do_synchronize() {
  folders_that_will_not_be_transferred=""
  if [[ "$#" -eq 1 ]] ; then
    folders_that_will_not_be_transferred="$1"
  fi
  echo "Calling function transfer_model"
  transfer_model ${folders_that_will_not_be_transferred}
  echo "common.synchronize Done"
}

#####################################################################################################################
# Globals:
# RUN_dir, INTERMEDIATE_EXP_DIR, MODEL, EXPID
# Arguments:
#   None
# Returns:
#   None
# Purpose: Prepare source and target paths for CLEAN/TRANSFER
#####################################################################################################################
function setup_paths_localtrans() {
  src=${INTERMEDIATE_EXP_DIR}
  local_INTERMEDIATE_EXP_DIR=/gpfs/archive/bsc32/esarchive/exp/${MODEL}
  destination_folder=${local_INTERMEDIATE_EXP_DIR}/${EXPID}/original_files/
}

# copy to scratch the platforms files
#####################################################################################################################
# Globals:
# ROOTDIR, PROJDEST, PROJDIR
# Arguments:
#   None
# Returns:
#   None
# Purpose: send the initial set of libraries used by the synchronize job in order to be able to run
#
#####################################################################################################################
function copy_platform_environment_to_host() {
  mkdir -p ${ROOTDIR}/${PROJDEST}/platforms
  rsync --recursive --links --perms --times --owner --devices --specials --verbose --omit-dir-times ${PROJDIR}/platforms/ ${ROOTDIR}/${PROJDEST}/platforms
  rsync --recursive --links --perms --times --owner --devices --specials --verbose --omit-dir-times ${PROJDIR}/plugins/ ${ROOTDIR}/${PROJDEST}/plugins
}

#####################################################################################################################
# Globals:
# RUN_dir,  LOGDIR, CHUNK, START_date, MEMBER, STAMP, PATHOUT
# Arguments:
#   None
# Returns:
#   None
# Purpose: copy logs to the proper storage location before wiping out the experiment folder ( used by CLEAN_EXPERIMENT )
#####################################################################################################################
function save_logs() {
  echo "We are now storing log files"
  cd ${LOGDIR}
  logs_tarfile="logfiles_${EXPID}_${STAMP}.tar"
  IS_TRANSFER=TRUE
  prepare_logs && returncode=$? || returncode=$?
  if [[ ${returncode} -eq 0 ]] || [[ ${returncode} -eq 123 ]]; then
    [[ ! -f ../${logs_tarfile}.gz ]] && gzip -9 ../${logs_tarfile}
    local pathout_logfiles="${PATHOUT}/logfiles/"
    move_files ../${logs_tarfile}.gz ${pathout_logfiles} ${IS_TRANSFER}
    rm -f ../${logs_tarfile}.gz
  fi
}

#####################################################################################################################
# Globals:
# INTERMEDIATE_EXP_DIR, EXPID, ROOTDIR, RM ( alias for deleting files )
# Arguments:
#   None
# Returns:
#   None
# Purpose: wipe out the experiment folders (used intermediate storage, scratch)
#####################################################################################################################
function remove_experiment_folders() {
  #remove the folder in intermediate storage (if it was used)
  delete_intermediate_storage ${INTERMEDIATE_EXP_DIR}/${EXPID}
  #delete scratch folder
  rm -rf ${ROOTDIR}/*
  # needed for indicating that the job finished successfully by autosubmit
  mkdir -p ${ROOTDIR}/LOG_${EXPID}/
}

#####################################################################################################################
# any override from the common libraries defined in filesystem.sh or new function goes here
#####################################################################################################################

#####################################################################################################################
# Globals:
#   EXPID, MODEL, INTERMEDIATE_EXP_DIR, EXPID, USE_INTERMEDIATE_STORAGE
# Arguments:
#   None
# Returns:
#   None
# Purpose: template function for Marenostrum4 for setting up common paths ( used as setup_paths_transfer_${HPCARCH} )
#
#####################################################################################################################
function setup_paths_transfer_marenostrum4() {
  #base function
  setup_paths_transfer
  #custom settings
  if [[ "${USE_INTERMEDIATE_STORAGE-}" == "TRUE" ]]; then
    PATHOUT="${INTERMEDIATE_EXP_DIR}/${EXPID}/"
  else
    local_INTERMEDIATE_EXP_DIR=/gpfs/archive/bsc32/esarchive/exp/${MODEL}
    PATHOUT="${local_INTERMEDIATE_EXP_DIR}/${EXPID}/original_files/"
  fi
}
+ . /gpfs/scratch/bsc32/bsc32627/t0ni/auto-ecearth3/platforms/marenostrum4/utils.sh
#####################################################################################################################
# Functions for handling filesystem operations for marenostrum4 platform
# Author: J.R.Berlin
#####################################################################################################################

#. ${PROJDIR}/platforms/common/common.utils.sh

#####################################################################################################################
# any override from the common libraries defined in filesystem.sh or new function goes here
#####################################################################################################################

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: load the common modules required by standard jobs in Marenostrum4
#####################################################################################################################
function setup_modules() {
  set +xuve
  module purge
  module load intel/2018.3
  module load mkl/2018.3
  module load impi/2018.3
  module load netcdf/4.2
  module load hdf5/1.8.19
  module load perl/5.26
  module load fftw/3.3.6
  module load python/2.7.13
  module load udunits/2.2.25
  module load gsl/2.4
  module load nco/4.2.3_netcdf-4.2
  module load CDO/1.7.2
  module list
  set -xuve
}

#####################################################################################################################
# Globals:
#   USER
# Arguments:
#   None
# Returns:
#   None
# Purpose: Setup common paths as global variables, required by the different jobs running un the underlying platform
#           used in nord3, transfer_node and bscearth000 as well
####################################################################################################################
function setup_common_paths() {
  # common globals
  MODELS_DIR=/gpfs/projects/bsc32/models
  export SCRATCH=/gpfs/scratch/$(id -gn)/${USER}
  GROUP=bsc32
  # check if the machine has the environment tmp var defined
  SCRATCH_TMP_DIR=${SCRATCH}/${EXPID}/${START_date}/${MEMBER}/tmp/${JOBNAME}
  #initial conditions dirr for the platform, in order to have this working properly we need
  #all IC/rst files in place synchronized from bsc32/projects
  INTERMEDIATE_EXP_DIR=/gpfs/projects/${GROUP}/repository/exp
  EXTRA_DIR=/gpfs/projects/${GROUP}/repository
  IC_DIR=/gpfs/projects/${GROUP}/repository/ic
}

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: set the global variables and functions to be used by the jobs running in the underlying architecture
#          used in nord3, transfer_node and bscearth000 as well
#####################################################################################################################
function setup_common_environment_vars() {
  # common globals
  MODELS_DIR=/gpfs/projects/bsc32/models
  GROUP=bsc32
}

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: set the common set of global variables to be used for the jobs running on the underlying archictecture
#####################################################################################################################
function setup_environment_vars() {
  # common globals
  setup_common_environment_vars
  export ECE3_POSTPROC_MACHINE=mn4
}

#####################################################################################################################
# Globals:
#   CURRENT_ARCH, HPCARCH
# Arguments:
#   None
# Returns:
#   None
# Purpose: set the global variables refering to common paths to be used by the jobs running in the
#         underlying architecture, used in nord3, transfer_node and bscearth000 as well
#####################################################################################################################
function setup_paths() {
  setup_common_paths
  if [[ "${CURRENT_ARCH}" == "${HPCARCH}" ]]; then
    setup_paths_libs
  fi
}

#####################################################################################################################
# Globals:
#   USER, PATH
# Arguments:
#   None
# Returns:
#   None
# Purpose: auxiliary function to load paths to the different modules that will be load when the job
#         executes in the cluster
#####################################################################################################################
function setup_paths_libs() {
  grib_api_path=/apps/GRIB/1.14.0/INTEL
  PATH=${grib_api_path}/bin:${PATH}

  export SCRATCH=/gpfs/scratch/$(id -gn)/${USER}
  export PATH=/gpfs/projects/bsc32/repository/apps/rebuild_nemo_marenostrum4:${PATH}
  export PYTHONPATH=${grib_api_path}/lib/python2.7/site-packages/grib_api:${PYTHONPATH-}
  export GRIB_BIN_PATH=${grib_api_path}/bin
  export GRIB_DEFINITION_PATH=${grib_api_path}/share/grib_api/definitions
  export GRIB_SAMPLES_PATH=${grib_api_path}/share/grib_api/ifs_samples/grib1
}

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: auxiliary function to load paths to the different modules that will be load when the job
#          executes in the cluster, used in transfer machines!
#####################################################################################################################
function setup_environment_vars_local() {
  setup_common_environment_vars
  export ECE3_POSTPROC_MACHINE=mn4
}

#used in transfer machines !

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: auxiliary function to set the intermediate storage root path, used in transfer machines, this function
# overrides the base function in common.filesystem.sh
#####################################################################################################################
function setup_paths_local() {
  echo "MN4 Paths"
  GROUP=bsc32
  export INTERMEDIATE_EXP_DIR=/gpfs/projects/${GROUP}/repository/exp
}
+ setup_modules
+ set +xuve
remove mkl/2017.4 (LD_LIBRARY_PATH) 
remove impi/2017.4 (PATH, MANPATH, LD_LIBRARY_PATH) 
Set INTEL compilers as MPI wrappers backend 
load mkl/2018.3 (LD_LIBRARY_PATH) 
load impi/2018.3 (PATH, MANPATH, LD_LIBRARY_PATH) 
load netcdf/4.2 (PATH, LD_LIBRARY_PATH, NETCDF, NETCDF_PREFIX, NETCDF_HOME,
NETCDF_INC, NETCDF_LIB, NETCDF_ROOT) 
load hdf5/1.8.19 (PATH, LD_LIBRARY_PATH, LIBRARY_PATH, C_INCLUDE_PATH,
CPLUS_INCLUDE_PATH) 
load perl/5.26 (PATH, LD_LIBRARY_PATH, MANPATH, PERL5LIB) 
load fftw/3.3.6 (FFTW_CFLAGS, FFTW_FFLAGS, FFTW_LDFLAGS, FFTW_INCL, FFTW_LIBS,
LIBRARY_PATH, LD_LIBRARY_PATH, C_INCLUDE_PATH) 
load PYTHON/2.7.13 (PATH, MANPATH, LD_LIBRARY_PATH, LIBRARY_PATH,
PKG_CONFIG_PATH, C_INCLUDE_PATH, CPLUS_INCLUDE_PATH, PYTHONHOME) 
load lzo/2.10 (LD_LIBRARY_PATH,PKG_CONFIG_PATH,CFLAGS,CXXFLAGS,LDFLAGS) 
load udunits/2.2.25 (PATH, LD_LIBRARY_PATH, LIBRARY_PATH, INFOPATH,
C_INCLUDE_PATH, CPLUS_INCLUDE_PATH, UDUNITS2_PATH, UDUNITS2_XML_PATH) 
load gsl/2.4 (PATH, LD_LIBRARY_PATH, LIBRARY_PATH, MANPATH, INFOPATH,
C_INCLUDE_PATH, CPLUS_INCLUDE_PATH, PKG_CONFIG_PATH, GSL_DIR) 
load nco/4.2.3_netcdf-4.2 (PATH, LD_LIBRARY_PATH, LIBRARY_PATH, MANPATH,
INFOPATH, C_INCLUDE_PATH, CPLUS_INCLUDE_PATH) 
load CDO/1.7.2 (PATH) 

Currently Loaded Modules:
  1) intel/2018.3   5) hdf5/1.8.19   9) python/2.7.13         13) CDO/1.7.2
  2) mkl/2018.3     6) perl/5.26    10) udunits/2.2.25
  3) impi/2018.3    7) fftw/3.3.6   11) gsl/2.4
  4) netcdf/4.2     8) lzo/2.10     12) nco/4.2.3_netcdf-4.2

 

+ setup_environment_vars
+ setup_common_environment_vars
+ MODELS_DIR=/gpfs/projects/bsc32/models
+ GROUP=bsc32
+ export ECE3_POSTPROC_MACHINE=mn4
+ ECE3_POSTPROC_MACHINE=mn4
+ setup_paths
+ setup_common_paths
+ MODELS_DIR=/gpfs/projects/bsc32/models
id -gn
++ id -gn
+ export SCRATCH=/gpfs/scratch/bsc32/bsc32627
+ SCRATCH=/gpfs/scratch/bsc32/bsc32627
+ GROUP=bsc32
+ SCRATCH_TMP_DIR=/gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/tmp/t0ni_19931101_fc0_INI
+ INTERMEDIATE_EXP_DIR=/gpfs/projects/bsc32/repository/exp
+ EXTRA_DIR=/gpfs/projects/bsc32/repository
+ IC_DIR=/gpfs/projects/bsc32/repository/ic
+ [[ marenostrum4 == \m\a\r\e\n\o\s\t\r\u\m\4 ]]
+ setup_paths_libs
+ grib_api_path=/apps/GRIB/1.14.0/INTEL
+ PATH=/apps/GRIB/1.14.0/INTEL/bin:/apps/CDO/1.7.2/INTEL/IMPI/bin:/apps/NCO/4.2.3_netcdf-4.2/INTEL/bin:/apps/GSL/2.4/INTEL/bin:/apps/UDUNITS/2.2.25/INTEL/bin:/apps/PYTHON/2.7.13/INTEL/bin:/apps/FFTW/3.3.6/INTEL/IMPI/bin:/apps/PERL/5.26/bin:/apps/HDF5/1.8.19/INTEL/IMPI/bin:/apps/NETCDF/4.2/INTEL/IMPI/bin:/apps/INTEL/2018.3.051/impi/2018.3.222/bin64:/apps/INTEL/2018.3.051/bin:/usr/local/bin:/usr/bin:/bin:/usr/games:/usr/lpp/mmfs/bin
id -gn
++ id -gn
+ export SCRATCH=/gpfs/scratch/bsc32/bsc32627
+ SCRATCH=/gpfs/scratch/bsc32/bsc32627
+ export PATH=/gpfs/projects/bsc32/repository/apps/rebuild_nemo_marenostrum4:/apps/GRIB/1.14.0/INTEL/bin:/apps/CDO/1.7.2/INTEL/IMPI/bin:/apps/NCO/4.2.3_netcdf-4.2/INTEL/bin:/apps/GSL/2.4/INTEL/bin:/apps/UDUNITS/2.2.25/INTEL/bin:/apps/PYTHON/2.7.13/INTEL/bin:/apps/FFTW/3.3.6/INTEL/IMPI/bin:/apps/PERL/5.26/bin:/apps/HDF5/1.8.19/INTEL/IMPI/bin:/apps/NETCDF/4.2/INTEL/IMPI/bin:/apps/INTEL/2018.3.051/impi/2018.3.222/bin64:/apps/INTEL/2018.3.051/bin:/usr/local/bin:/usr/bin:/bin:/usr/games:/usr/lpp/mmfs/bin
+ PATH=/gpfs/projects/bsc32/repository/apps/rebuild_nemo_marenostrum4:/apps/GRIB/1.14.0/INTEL/bin:/apps/CDO/1.7.2/INTEL/IMPI/bin:/apps/NCO/4.2.3_netcdf-4.2/INTEL/bin:/apps/GSL/2.4/INTEL/bin:/apps/UDUNITS/2.2.25/INTEL/bin:/apps/PYTHON/2.7.13/INTEL/bin:/apps/FFTW/3.3.6/INTEL/IMPI/bin:/apps/PERL/5.26/bin:/apps/HDF5/1.8.19/INTEL/IMPI/bin:/apps/NETCDF/4.2/INTEL/IMPI/bin:/apps/INTEL/2018.3.051/impi/2018.3.222/bin64:/apps/INTEL/2018.3.051/bin:/usr/local/bin:/usr/bin:/bin:/usr/games:/usr/lpp/mmfs/bin
+ export PYTHONPATH=/apps/GRIB/1.14.0/INTEL/lib/python2.7/site-packages/grib_api:/apps/PYTHON/2.7.13/INTEL/lib/python2.7/site-packages
+ PYTHONPATH=/apps/GRIB/1.14.0/INTEL/lib/python2.7/site-packages/grib_api:/apps/PYTHON/2.7.13/INTEL/lib/python2.7/site-packages
+ export GRIB_BIN_PATH=/apps/GRIB/1.14.0/INTEL/bin
+ GRIB_BIN_PATH=/apps/GRIB/1.14.0/INTEL/bin
+ export GRIB_DEFINITION_PATH=/apps/GRIB/1.14.0/INTEL/share/grib_api/definitions
+ GRIB_DEFINITION_PATH=/apps/GRIB/1.14.0/INTEL/share/grib_api/definitions
+ export GRIB_SAMPLES_PATH=/apps/GRIB/1.14.0/INTEL/share/grib_api/ifs_samples/grib1
+ GRIB_SAMPLES_PATH=/apps/GRIB/1.14.0/INTEL/share/grib_api/ifs_samples/grib1

#
# General Paths and Conf.(these lines must be ALWAYS after the load_platform_environment invocation)
#
MODEL_DIR=${MODELS_DIR}/${MODEL}/${VERSION}
+ MODEL_DIR=/gpfs/projects/bsc32/models/ecearth/trunk
ini_data_dir=${MODEL_DIR}/inidata
+ ini_data_dir=/gpfs/projects/bsc32/models/ecearth/trunk/inidata
RUN_dir=${ROOTDIR}/${START_date}/${MEMBER}
+ RUN_dir=/gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0
INIPATH=${RUN_dir}/inidata
+ INIPATH=/gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/inidata

#####################################################################################################################
# Globals:
# INTERMEDIATE_EXP_DIR, EXPID, START_date, MEMBER, RUN_dir, INIPATH, DELETE_INI_DIR_ON_INI
# HPCUSER, JOBNAME, SCRATCH_TMP_DIR
# Arguments:
#   None
# Returns:
#   None
# Purpose: Prepare the folders and paths for the initialization process of the experiment
#
#####################################################################################################################
function setup_ini_folders() {
  #Create intermediate storage
  local storage_dir=${INTERMEDIATE_EXP_DIR}/${EXPID}/${START_date}/${MEMBER}
  if ! test_intermediate_storage "-d ${storage_dir}"; then
    mkdir_intermediate_storage ${storage_dir} Earth
    if ! test_intermediate_storage "-d ${storage_dir}"; then
      echo " OMG we have a problem!"
      echo " this directory should exist! ${storage_dir}"
      exit
    fi
  fi

  #Check if ini/runtime folders should be deleted and regenerated
  if [ "${DELETE_INI_DIR_ON_INI}" == "TRUE" ]; then
    rm -rf ${INIPATH}
    if [[ $(ls runtime/* >&/dev/null) ]] && [[ -z "${FORCE_RUN_FROM_SCRATCH}" ]]; then
      # let user know user of potential issues of leaving runtime folder
      echo "Runtime '${RUN_dir}/runtime' directory already existing, please remove it manually or set DELETE_RUN_DIR_ON_INI to true in [common] section in prog_EXPID.conf"
    fi
  fi

  mkdir -p ${INIPATH}

  #in DT machine this can be empty, in that case tmp folder will be the root of the experiment
  rm -rf ${SCRATCH_TMP_DIR}
  mkdir -p ${SCRATCH_TMP_DIR}

  #
  # Model Paths and Conf.
  #
  ini_data_dir=${MODEL_DIR}/inidata
}

#####################################################################################################################
# Globals:  out_member: id of the member for the component,  out_member_perturb,
# Arguments:
#   ini_member: id or list of id for the initialization of one component
# Returns:
#   None
# Purpose: Choose the member id for one given component
#
#####################################################################################################################
function calc_member() {
  ini_member=${1}
  membid=${2}
  membarr=($ini_member)
  nbmemb=${#membarr[@]}
  out_member_perturb=false
  if [[ $nbmemb == 1 ]]; then
    if [[ $ini_member == "all_members" ]]; then
      out_member=$membid
    else
      out_member=$ini_member
    fi
  else
    imemb=10#${membid/fc/}
    out_member=${membarr[$imemb]}
  fi
}

#####################################################################################################################
# Globals:
#  EXPID, ini_data_dir, START_date, IC_DIR
#  ATM_ini, ATM_ini_member, ifs_grid, ATM_ini_member_perturb
# Arguments:
#  membid: id of the member
# Returns:
#   None
# Purpose: IC's for Atmosphere Component (EC-Earth3)
#
#####################################################################################################################
function atm_ini() {
  local membid=${1}
  # use this grib_set command to make sure the date is correct
  local grib_cmd="grib_set -s dataDate=${START_date},dataTime=0"
  if [[ $ATM_ini != '' ]]; then
    # user can specify alternate date of the ATM_ini for e.g. starting from a spinup
    local ini_start_date=$([[ -z "${ATM_ini_date}" ]] && echo ${START_date} || echo ${ATM_ini_date})
    # copy IC's
    local ifile=${ATM_ini}_${ATM_ini_member}_${ini_start_date}00.tar.gz
    # ICś always are copied from intermediate storage independently of the CLEAN+TRANSFER feature
    copy_intermediate_storage ${IC_DIR}/atmos/${ifs_grid}/${ATM_ini}/${ifile} . "TRUE"
    tar -xzvf ${ifile}
    rm -f ${ifile}
    # rename IC's
    mkdir -p ifs/${ifs_grid}/${START_date}
    ${grib_cmd} ICMGG*INIT ifs/${ifs_grid}/${START_date}/ICMGGECE3INIT   #ICMGG${EXPID}INIT
    ${grib_cmd} ICMSH*INIT ifs/${ifs_grid}/${START_date}/ICMSHECE3INIT   #ICMSH${EXPID}INIT
    ${grib_cmd} ICMGG*INIUA ifs/${ifs_grid}/${START_date}/ICMGGECE3INIUA #ICMGG${EXPID}INIUA
    rm -f ICMGG*INIT ICMSH*INIT ICMGG*INIUA
  else
    mkdir -p ifs/${ifs_grid}/${START_date}/
    ${grib_cmd} ${ini_data_dir}/ifs/${ifs_grid}/${START_date}/ICMGGECE3INIT ifs/${ifs_grid}/${START_date}/ICMGGECE3INIT
    ${grib_cmd} ${ini_data_dir}/ifs/${ifs_grid}/${START_date}/ICMSHECE3INIT ifs/${ifs_grid}/${START_date}/ICMSHECE3INIT
    ${grib_cmd} ${ini_data_dir}/ifs/${ifs_grid}/${START_date}/ICMGGECE3INIUA ifs/${ifs_grid}/${START_date}/ICMGGECE3INIUA
  fi

  #if requested, perturb t field in IFS IC using the member # as seed
  if [[ "${ATM_ini_member_perturb}" == "TRUE" ]]; then
    param="t"
    seed=${membid//[!0-9]/}
    mv ifs/${ifs_grid}/${START_date}/ICMSHECE3INIT ifs/${ifs_grid}/${START_date}/ICMSHECE3INIT_ORIG
    python $PROJDIR/plugins/perturb_var.py -s $param --seed $seed ifs/${ifs_grid}/${START_date}/ICMSHECE3INIT_ORIG ifs/${ifs_grid}/${START_date}/ICMSHECE3INIT
  fi
}

#####################################################################################################################
# Globals:
#   START_date_1,  OCEAN_ini, OCEAN_ini_member,
#   NEMO_resolution, OCEAN_ini_date, OCEAN_ini_date, IC_DIR
# Arguments:
#   None
# Returns:
#   None
# Purpose: IC's for Ocean Component (EC-Earth3)
#
#####################################################################################################################
function oce_ini() {
  local ini_start_date=$([[ -z "${OCEAN_ini_date}" ]] && echo ${START_date_1} || echo $(date -d "${OCEAN_ini_date} - 1 day" +%Y%m%d))
  nemo_components_ini_ic $([[ -z "${OCEAN_ini}" ]] && echo "FALSE" || echo "${OCEAN_ini}") ${IC_DIR}/ocean/${NEMO_resolution}/${OCEAN_ini} ${OCEAN_ini}_${OCEAN_ini_member}_${ini_start_date}_restart.nc restart_oce.nc "OCE_ini"
}

#####################################################################################################################
# Globals:
#   START_date_1, ICE_ini_date, ICE_ini, ICE_ini_member, nem_grid_wol, IC_DIR
# Arguments:
#   None
# Returns:
#   None
# Purpose: IC's for Sea Ice Component (EC-Earth3)
#
#####################################################################################################################
function ice_ini() {
  local ini_start_date=$([[ -z "${ICE_ini_date}" ]] && echo ${START_date_1} || echo $(date -d "${ICE_ini_date} - 1 day" +%Y%m%d))
  nemo_components_ini_ic $([[ -z "${ICE_ini}" ]] && echo "FALSE" || echo "${ICE_ini}") ${IC_DIR}/ice/${nem_grid_wol}_LIM3/${ICE_ini} ${ICE_ini}_${ICE_ini_member}_${ini_start_date}_restart_ice.nc restart_ice.nc "ICE_ini"
}

#####################################################################################################################
# Globals:
# PISCES_ini, PISCES_ini_member, START_date_1, NEMO_resolution, ini_start_date
# Arguments:
#   None
# Returns:
#   None
# Purpose: IC's for PISCES component (EC-Earth3)
#
#####################################################################################################################
function pisces_ini() {
  local ini_start_date=$([[ -z "${PISCES_ini_date}" ]] && echo ${START_date_1} || echo $(date -d "${PISCES_ini_date} - 1 day" +%Y%m%d))
  nemo_components_ini_ic $([[ -z "${PISCES_ini}" ]] && echo "FALSE" || echo "${PISCES_ini}") ${IC_DIR}/ocnBgChem/${NEMO_resolution}/${PISCES_ini} ${PISCES_ini}_${PISCES_ini_member}_${ini_start_date}_restart_trc.nc restart_trc.nc "PISCES_ini"
}

#####################################################################################################################
# Globals:
# PISCES_OFF, PISCES_OFF_DYN, EXTRA_DIR, Chunk_start_date, INIPATH
# Arguments:
#   None
# Returns:
#   None
# Purpose:use the right forcing fields for PISCES offline - PISCES component (EC-Earth3)
#
#####################################################################################################################
function pisces_offline_forcing() {

  if [[ ! -z ${PISCES_OFF} ]]; then
    year=$(echo ${Chunk_start_date} | cut -c1-4)
  fi

  mkdir -p ${INIPATH}/nemo/pisces/pisces_dyn
  if [[ ${PISCES_OFF_DYN} == '' ]]; then
    echo "ERROR Data used to force PISCES OFFLINE not specified, precise the data with PISCES_OFF_DYN in the [pisces] section of proj_EXPID.conf"
  fi
 if [[ ${PISCES_OFF_DYN} != '' ]]; then
    for offline_pisces_forcing_file in $(ls ${EXTRA_DIR}/dynamical_fields/${PISCES_OFF_DYN}/*); do
      filename="${offline_pisces_forcing_file%.*}"
      file_postfix="${filename: -1}"
      if [[ ! -f ${INIPATH}/nemo/pisces/pisces_dyn/OfflineForcing_grid_${file_postfix}.nc ]]; then
        ln -s ${offline_pisces_forcing_file} ${INIPATH}/nemo/pisces/pisces_dyn/OfflineForcing_grid_${file_postfix}.nc
      fi
    done
  else
    echo "ERROR dynamical fields do not exist "
  fi

}

#####################################################################################################################
# Globals:
#   EXPID, ini_data_dir, START_date_1, IC_DIR
#   ICE_ini, ICE_ini_member, nem_grid_wol, ICE
# Arguments:
#   #$1 - ICs_Component, $2 - ICs_Path, $3 - ICs_Filename, $4 - ICs_output, $5 - string for msg out
# Returns:
#   None
# Purpose: Generic function to copy the IC's for the different NEMO components( Ocean, ICE amd PISCES (EC-Earth3))
#
#####################################################################################################################
function nemo_components_ini_ic() {

  #$1 - ICs_Component
  #$2 - ICs_Path
  #$3 - ICs_Filename
  #$4 - ICs_output
  #$5 - string for msg out

  if [[ "${1}" != "FALSE" ]]; then
    copy_intermediate_storage ${2}/${3}.gz . "TRUE"
    gunzip ${3}.gz
    mv ${3} ${4}
  else
    echo "Start ${5} component at rest"
  fi
}

######################################################
########################IFS3##########################
######################################################

#####################################################################################################################
# Globals:
# RUN_dir,  ATM_NUDGING, CHUNK, Chunk_start_date, CHUNKSIZE ,EXTRA_DIR
# Arguments:
#   None
# Returns:
#   None
# Purpose: Prepare the Nudging for the atmosphere component by taking into account the IFS resolution
#
#####################################################################################################################
function ifs_setup_nudging() {
  # Atmospheric nudging - Lauriane, Omar
  # Preperation of refence files for Chunk 1+2
  # Ref: e007, i05g

  PATHNUDA=${EXTRA_DIR}/nudging/atmos/T511L91
  rm -f *rlxmlgg* *rlxmlsh*

  start_date_mm=$(echo $Chunk_start_date | cut -c5-6)
  start_date_mm=${start_date_mm#0} # Cut leading '0'
  start_date_yyyy=$(echo $Chunk_start_date | cut -c1-4)
  RUN_months=${CHUNKSIZE}
  . $PROJDIR/plugins/atmospheric_nudging.sh

  # Prepare first three chunks
  atm_nudg_chunk_lnk 0
  atm_nudg_chunk_lnk 1
  atm_nudg_chunk_lnk 2

}

#####################################################################################################################
# Globals:
# RUN_dir,  ATM_ini_member, MEMBER, out_member
# Arguments:
#   None
# Returns:
#   None
# Purpose: set the member for the atmosphere component
#
#####################################################################################################################
function ifs_setup_members() {
  #
  # IFS
  #
  if [[ ${ATM_ini_member} != '' ]]; then
    calc_member "${ATM_ini_member}" ${MEMBER}
    ATM_ini_member=${out_member}
  fi
}

#####################################################################################################################
# Globals:
# RUN_dir,  SCRATCH_TMP_DIR, MEMBER, out_member
# Arguments:
#   None
# Returns:
#   None
# Purpose: Main function for initialization - ifs3
#
#####################################################################################################################
function ifs3_init() {

  #
  # IFS
  #
  ifs_setup_members

  #
  # Prepare stuff for experiment
  #
  # ifs-only experiment, atm_ini is required, but ifs_setup_nudging can be disabled, check if activated or not to load the IC's

  cd ${SCRATCH_TMP_DIR}

  atm_ini $MEMBER

  [[ "$ATM_NUDGING" == "TRUE" ]] && ifs_setup_nudging

  cd ${INIPATH}

}

#####################################################################################################################
# Globals:
# LPJG,  ifs_veg_source, EXTRA_DIR, ifs_grid, INIPATH
# Arguments:
#   None
# Returns:
#   None
# Purpose: initialization of the lpjg component ( linkinh of icmcl files in inidata folder
#
#####################################################################################################################
function ifs_veg_source_init() {

  case ${ifs_veg_source} in

  "custom_"*)
    # custom not allowed when running with LPJG, since when config=lpjg:fdbck the ifs_veg_source is not used
    if [[ $LPJG == "TRUE" ]]; then
      echo "requested IFS_VEG_SOURCE = ${ifs_veg_source} but this is not allowed when LPJG=TRUE!"
      exit 1
    fi
    # link icmcl folder
    veg_version=${ifs_veg_source:7}
    dir_veg_source=${EXTRA_DIR}/icmcl/${ifs_grid}/icmcl_${veg_version}
    if [ ! -d ${dir_veg_source} ]; then
      echo "requested IFS_VEG_SOURCE = ${ifs_veg_source} but not found in ${dir_veg_source}"
      exit 1
    else
      ln -sf ${dir_veg_source} ${INIPATH}/ifs/${ifs_grid}
    fi
    ;;

  *)
    echo "Nothing to do for ifs veg source ${ifs_veg_source}"
    ;;

  esac

}

######################################################
########################NEMO3#########################
######################################################

#####################################################################################################################
# Globals: MEMBER, EXTRA_DIR, NEMO_resolution, INIPATH,
#          OCE_NUDG_DATA, OCE_NUDG_DATA_member, OCE_NUDG_COEFF
# Arguments:
#   None
# Returns:
#   None
# Purpose: 03/2018 - Yohan Ruprich-Robert, Valentina Sicardi; 08/2021 - Vladimir Lapin
# Link ocean nudging files with salinity and temperature 3D and nudging coefficients (resto.nc).
# Note that the files to be linked are outside the inidata folder, in the repository.
#
#####################################################################################################################
function nemo_setup_nudging() {

  mkdir -p ${INIPATH}/nemo/oce_nudg

  # if ${OCE_NUDG_DATA_member} is undefined or "all_members", use the same as ${MEMBER}
  if [[ -z "${OCE_NUDG_DATA_member}" ]] || [[ "${OCE_NUDG_DATA_member}" == "all_members" ]]; then
    OCE_NUDG_DATA_member=${MEMBER}
  fi

  # if ${OCE_NUDG_COEFF} is undefined, use resto.nc from RESTO_DEFAULT
  if [[ -z "${OCE_NUDG_COEFF}" ]]; then
    OCE_NUDG_COEFF=RESTO_DEFAULT
  fi

  PATH_NUDG_FILE=${EXTRA_DIR}/nudging/ocean/${OCE_NUDG_DATA}/${NEMO_resolution}/${OCE_NUDG_DATA_member}
  PATH_NUDG_RESTO_COEFF=${EXTRA_DIR}/nudging/ocean/${OCE_NUDG_COEFF}/${NEMO_resolution}

  if [[ "${OCE_NUDG_DATA}" != '' ]] && [[ -d ${PATH_NUDG_FILE} ]]; then
    ln -fs ${PATH_NUDG_FILE}/temp_sal_*.nc ${INIPATH}/nemo/oce_nudg/
  else
    echo "ERROR The specified path for ocean nudging data is invalid: ${PATH_NUDG_FILE}"
    get_out="true"
  fi

  if [[ -d ${PATH_NUDG_RESTO_COEFF} ]]; then
    ln -fs ${PATH_NUDG_RESTO_COEFF}/resto.nc ${INIPATH}/nemo/oce_nudg/resto.nc
  else
    echo "ERROR The specified path for resto.nc is invalid: ${PATH_NUDG_RESTO_COEFF}"
    get_out="true"
  fi

}

#####################################################################################################################
# Globals: MEMBER, EXTRA_DIR, NEMO_resolution, INIPATH,
#   SURF_RESTO, SURF_RESTO_DATA, SURF_RESTO_DATA_member, SURF_RESTO_MASK
# Arguments:
#   None
# Returns:
#   None
# Purpose: 03/2018 - Yohan Ruprich-Robert, Valentina Sicardi; 08/2021 - Vladimir Lapin
# Link surface restoring files for salinity and temperature and an optional mask.
# Note that the files to be linked are outside the inidata folder, in the repository.
#
#####################################################################################################################
function nemo_surface_restoring() {

  mkdir -p ${INIPATH}/nemo/surface_restoring

  # if ${SURF_RESTO_DATA_member} is undefined or "all_members", use the same as ${MEMBER}
  if [[ -z "${SURF_RESTO_DATA_member}" ]] || [[ "${SURF_RESTO_DATA_member}" == "all_members" ]]; then
    SURF_RESTO_DATA_member=${MEMBER}
  fi

  # if ${SURF_RESTO_MASK} is undefined, use mask_restore.nc from RESTO_DEFAULT
  if [[ -z "${SURF_RESTO_MASK}" ]]; then
    SURF_RESTO_MASK=MASK_DEFAULT
  fi

  PATH_SURF_FILE=${EXTRA_DIR}/surface_restoring/ocean/${SURF_RESTO_DATA}/${NEMO_resolution}/${SURF_RESTO_DATA_member}
  PATH_SURF_MASK=${EXTRA_DIR}/surface_restoring/ocean/masks/${SURF_RESTO_MASK}/${NEMO_resolution}

  if [[ "${SURF_RESTO_DATA}" != '' ]] && [[ -d ${PATH_SURF_FILE} ]]; then
    ln -fs ${PATH_SURF_FILE}/sss_restore_data*.nc ${INIPATH}/nemo/surface_restoring/
    ln -fs ${PATH_SURF_FILE}/sst_restore_data*.nc ${INIPATH}/nemo/surface_restoring/
  else
    echo "ERROR The specified path for surface restoring data is invalid: ${PATH_SURF_FILE}"
    get_out="true"
  fi

  if [[ -d ${PATH_SURF_MASK} ]]; then
    ln -fs ${PATH_SURF_MASK}/mask_restore*.nc ${INIPATH}/nemo/surface_restoring/
  else
    echo "ERROR The specified path for surface restoring data is invalid: ${PATH_SURF_MASK}"
    get_out="true"
  fi

}

#####################################################################################################################
# Globals: OCEAN_ini_member, ICE_ini_member, PISCES_ini_member, MEMBER
#
# Arguments:
#   None
# Returns:
#   OCEAN_ini_member
# Purpose: Ocean/LIM member setup, Prepare member names and other needed ini data for members
#
#####################################################################################################################
function nemo_setup_members() {
  #
  # NEMO
  #

  if [[ "${OCEAN_ini_member}" != "" ]]; then
    calc_member "$OCEAN_ini_member" ${MEMBER}
    OCEAN_ini_member=${out_member}
  fi

  #
  # ICE
  #

  if [[ ${ICE_ini_member} != "" ]]; then
    calc_member "$ICE_ini_member" ${MEMBER}
    ICE_ini_member=${out_member}
  fi

  #
  # PISCES
  #

  if [[ ${PISCES_ini_member} != "" ]]; then
    calc_member "$PISCES_ini_member" ${MEMBER}
    PISCES_ini_member=${out_member}
  else
    PISCES_ini_member=fc0 # Warning: uses only one initial condition
  fi

}

#####################################################################################################################
# Globals: MEMBER, CHUNKSIZE, nem_forcing_set, INIPATH, ini_data_dir, PROJDIR, START_date
#
# Arguments:
#   None
# Returns:
#   None
# Purpose:  Ocean forcing setup,  Prepare the needed files for the setup for the atm forcings
# to be used only in NEMO only experiments
#
#####################################################################################################################
function nemo_setup_forcing() {

  ##############################################
  #        NEMO FORCING  WEIGHTS               #
  ##############################################
  MEMBER=${MEMBER}
  chunk_size=${CHUNKSIZE}
  if [[ -z ${nem_forcing_set-} ]]; then
    nem_forcing_set="CoreII"
  fi

  # Forcing files for salinity and temperature interpolation
  mkdir -p ${INIPATH}/nemo/forcing/weights
  ln -sf ${ini_data_dir}/nemo/forcing/weights/* ${INIPATH}/nemo/forcing/weights/

  source ${PROJDIR}/plugins/nemo_forcing.sh

  link_nemo_forcing ${START_date} ${chunk_size} ${num_chunks} ${chunk_size_unit}

}

#####################################################################################################################
# Globals: SCRATCH_TMP_DIR, OCE_NUDG
#
# Arguments:
#   None
# Returns:
#   None
# Purpose: Main function for initialization - Nemo3
#
#####################################################################################################################
function nemo3_init() {

  #
  # Prepare stuff for experiment
  # nemo-only template, nemo components are required but the rest of the components can be disabled and with this checks it is avoided to load the IC's for them when not necessary
  #

  cd ${SCRATCH_TMP_DIR}

  nemo_setup_members

  oce_ini

  [[ "${PISCES_OFF}" == "TRUE" ]] && pisces_offline_forcing

  [[ "${ICE}" == "LIM3" || "${ICE}" == "LIM2" ]] && ice_ini

  [[ "${PISCES}" == "TRUE" ]] &&  pisces_ini

  [[ "${OCE_NUDG}" == "TRUE" ]] && nemo_setup_nudging

  [[ "${SURF_RESTO}" == "TRUE" ]] && nemo_surface_restoring

  nemo_setup_forcing

  cd ${INIPATH}

}

######################################################
#######OASIS##########################################
######################################################

#####################################################################################################################
# Globals: OCEAN_ini, SCRATCH_TMP_DIR, PROJDIR
#
# Arguments:
#   None
# Returns:
#   None:
# Purpose:
# Function to execute the plugin that generates the proper OASIS restarts for IFS from the NEMO restarts when those restarts exist
####################################################################################################################
function generate_oasis_restarts() {

  . ${PROJDIR}/platforms/${CURRENT_ARCH}/configure.sh
  load_modules_oas_rsts_gen
  
  . ${PROJDIR}/platforms/common/common.ini.sh
  . ${PROJDIR}/platforms/${CURRENT_ARCH}/ini.sh
  gen_oasis_rsts
}

#####################################################################################################################
# Globals: OASIS_ini, ATM_ini, OCEAN_ini, ATM_ini_member_perturb, OASIS_ini_member, ATM_ini_member, MEMBER,
#   SCRATCH_TMP_DIR, IC_DIR, START_date, ifs_res_hor, nem_res_hor
#
# Arguments:
#   None
# Returns:
#   None
# Purpose:
#  only for coupled experiments: check which set of IC to use based in the following rules:
#  if OASIS_ini is not empty, look for the coupler restarts for that EXPID, if not found crash (IC files,
#  both flags mentioned below will be false)
#  if OASIS_ini is empty and ATM_ini = OCEAN_ini, look for the coupler restarts for ATM_ini,
#  if not found use the default ones (rst files) (use_rst_default_if_ic_not_found)
#  if OASIS_ini is empty and ATM_ini != OCEAN_ini, always use the default ones (rst files) (use_default_rst)
#  the case in which OASIS_ini has a defined value and ATM_ini = OCEAN_ini = null is abnormal thus not contemplated
#  here
#
#####################################################################################################################
function oasis_setup_ifs_nemo() {

  use_rst_default_if_ic_not_found="FALSE"
  use_default_rst="FALSE"
  if [[ -z ${OASIS_ini} ]]; then
    if [[ ${ATM_ini} == ${OCEAN_ini} ]] && [[ ! -z ${ATM_ini} ]]; then
      use_rst_default_if_ic_not_found="TRUE"
    else
      use_default_rst="TRUE"
    fi
  fi

  #for the OASIS member ini, is defined based on the following rules:
  #if OASIS_ini_member is empty -->  should match the ATM_ini_member/OCE_ini_member
  #if ATM_ini_perturb=true --> OASIS_ini_member should be ATM_ini_member

  if [[ "${ATM_ini_member_perturb}" == "TRUE" ]]; then
    OASIS_ini_member=${ATM_ini_member}
  fi

  if [[ -z "${OASIS_ini_member}" ]] && [[ ${ATM_ini_member} == ${OCEAN_ini_member} ]]; then
    #if both members are undefined , we use the default one defined in the experiment
    if [[ -z "${ATM_ini_member}" ]]; then
      OASIS_ini_member=${MEMBER}
    else
      OASIS_ini_member=${ATM_ini_member}
    fi
  fi

  #Point to the inidata folder
  cd ${SCRATCH_TMP_DIR}

  case ${nem_grid_wol} in
  ORCA1) oas_ogrd=O1t0 ;;
  ORCA025) oas_ogrd=Ot25 ;;
  *)
    echo "Unsupported horizontal resolution (NEMO): ${nem_grid_wol}"
    get_out="true"
    ;;
  esac

  # Grid definition files
  # destination path for the experiment where the ICs will stored
  # (also the sub-directory under which to find the default files in ${ini_data_dir})
  oas_sub_dir=oasis/T${ifs_res_hor}-ORCA${nem_res_hor}/rst
  # Restart files
  oasis_restart_files="rstas.nc rstos.nc"

  # location if the oasis ics
  oas_ics_dir=${IC_DIR}/coupler/T${ifs_res_hor}-ORCA${nem_res_hor}/${OASIS_ini}
  local ini_start_date=$([[ -z "${OASIS_ini_date}" ]] && echo ${START_date} || echo ${OASIS_ini_date})
  oas_ics_prefix=${OASIS_ini}_${OASIS_ini_member}_${ini_start_date}00_
  # get them from the IC folder if found ( we use these only when use_rst_default_if_ic_not_found =="FALSE" )
  if [[ "${use_rst_default_if_ic_not_found}" == "TRUE" ]]; then
    oas_ics_dir=${IC_DIR}/coupler/T${ifs_res_hor}-ORCA${nem_res_hor}/${ATM_ini}
    oas_ics_prefix=${ATM_ini}_${ATM_ini_member}_${START_date}00_
    echo "getting oasis restarts from ${oas_ics_dir}/${oas_ics_prefix}*"
  fi

  if [[ ! -d ${oas_sub_dir} ]]; then
    mkdir -p ${oas_sub_dir}
  fi

  if [[ ! -d xios/ORCA${nem_res_hor} ]]; then
    mkdir -p xios/ORCA${nem_res_hor}
  fi

  #check the condition by which the job should be aborted
  abort_if_files_not_found="FALSE"
  if [[ "${use_rst_default_if_ic_not_found}" == "FALSE" ]] && [[ "${use_default_rst}" == "FALSE" ]]; then
    abort_if_files_not_found="TRUE"
  fi

  # first iteration, check existence of OASIS_ini is defined in the proj.conf means that it should be used or crash if the files doesnt exists
  if [[ "${use_default_rst-}" == "FALSE" ]]; then
    for f in ${oasis_restart_files}; do
      #check the existence of the file
      if ! test_intermediate_storage "-f ${oas_ics_dir}/${oas_ics_prefix}${f}.gz"; then
        # Issue 811, if there are OASIS no restarts, we abort and not continue with the INI
        if [[ "${abort_if_files_not_found}" == "TRUE" ]]; then
          echo "ERROR: Oasis initial conditions not present for ${ATM_ini}, ORCA${nem_res_hor} and  ${OASIS_ini}, please check IC repository for oasis"
          echo "to use default oasis restarts please set OASIS_ini=  in proj.conf"
          get_out="true"
        else
          # at this point we know that the Oasis files for given configuration don't exist, then we use default ones in the case that
          # the job must continue and use default ones
          use_default_rst="TRUE"
        fi
      fi
    done
  fi

  # from the previous step we gather information about existence of IC for Oasis, in case they exists we use them
  # cases contemplated: (OCE_ini = ATM_ini) and OASIS_ini= empty and OASIS_ini!=empty
  for f in ${oasis_restart_files}; do
    if [[ "${use_default_rst}" == "FALSE" ]]; then
      rm -f ${oas_sub_dir}/${f}
      copy_intermediate_storage ${oas_ics_dir}/${oas_ics_prefix}${f}.gz . "TRUE"
      gunzip -c ${oas_ics_prefix}${f}.gz >${oas_sub_dir}/${f}
      rm -f ${oas_ics_prefix}${f}.gz
    else
      if [ -f ${ini_data_dir}/${oas_sub_dir}/${f} ]; then
        # only for the default case
        copy_default_restarts ${ini_data_dir}/${oas_sub_dir}/${f} ${oas_sub_dir}/${f}
      fi
    fi
  done

  #generate proper OASIS restart for IFS from the NEMO restart
  if [[ "${use_default_rst}" == "TRUE" && -f ${oas_sub_dir}/rstos.nc && ! -z ${OCEAN_ini} ]]; then
    generate_oasis_restarts
  fi
}

#####################################################################################################################
# Globals:
#   OASIS_ini, SCRATCH_TMP_DIR, ifs_res_hor, ifs_grid_wol, IC_DIR, OASIS_ini_date, START_date, ini_start_date,
#   OASIS_ini_member
# Arguments:
#   None
# Returns:
#   None
# Purpose: Setup LPJG oasis restart.
#          we handle them in a simple manner, considering OASIS_ini and OASIS_ini_member without testing
#          if the restart files exist (copy_intermediate_storage will raise an error)
#          and not considering ATM_ini/ATM_ini_member/ATM_ini_member_perturb
#
#####################################################################################################################
function oasis_setup_ifs_lpjg() {
  if [[ ! -z ${OASIS_ini} ]]; then

    #Point to the inidata folder
    cd ${SCRATCH_TMP_DIR}

    # Grid definition files

    # LPJG oasis restarts are in lpjg/oasismct/T255
    oas_sub_dir=lpjg/oasismct/T${ifs_res_hor}

    # location if the oasis ics
    oas_ics_dir=${IC_DIR}/coupler/${ifs_grid_wol}/${OASIS_ini}
    local ini_start_date=$([[ -z "${OASIS_ini_date}" ]] && echo ${START_date} || echo ${OASIS_ini_date})
    oas_ics_prefix=${OASIS_ini}_${OASIS_ini_member}_${ini_start_date}00_

    if [[ ! -d ${oas_sub_dir} ]]; then
      mkdir -p ${oas_sub_dir}
    fi

    oas_rst_ifs_lpjg="vegin.nc lpjgv.nc"

    for f in ${oas_rst_ifs_lpjg}; do
      rm -f ${oas_sub_dir}/${f}
      copy_intermediate_storage ${oas_ics_dir}/${oas_ics_prefix}${f}.gz . "TRUE"
      gunzip -c ${oas_ics_prefix}${f}.gz >${oas_sub_dir}/${f}
      rm -f ${oas_ics_prefix}${f}.gz
    done

  fi
}

#####################################################################################################################
# Globals: OASIS_ini, IC_DIR, ifs_res_hor, TM5_NLEVS, OASIS_ini_date, START_date, ini_start_date
#
# Arguments:
#   None
# Returns:
#   None
# Purpose: Setup TM5 oasis restart. Function divided to handle the case where only IFS+TM5 is run.
#
#####################################################################################################################
function oasis_setup_ifs_tm5() {
  if [[ ! -z ${OASIS_ini} ]]; then

    #Point to the inidata folder
    cd ${SCRATCH_TMP_DIR}

    # Grid definition files

    # destination path for the experiment where the ICs will stored
    # (also the sub-directory under which to find the default files in ${ini_data_dir})
    oas_sub_dir=oasis/T${ifs_res_hor}-TM5-LPJG/rst/${TM5_NLEVS}-levels

    # location if the oasis ics
    oas_ics_dir=${IC_DIR}/coupler/T${ifs_res_hor}-TM5-${TM5_NLEVS}-levels/${OASIS_ini}
    local ini_start_date=$([[ -z "${OASIS_ini_date}" ]] && echo ${START_date} || echo ${OASIS_ini_date})
    oas_ics_prefix=${OASIS_ini}_${OASIS_ini_member}_${ini_start_date}00_

    if [[ ! -d ${oas_sub_dir} ]]; then
      mkdir -p ${oas_sub_dir}
    fi

    # Get all oasis restart files in the TM5 oasis restart directory
    # Get only the nc filename (remove prefix and gz extension)
    oas_rst_ifs_tm5=$(ls ${oas_ics_dir}/${oas_ics_prefix}* | xargs -n 1 basename | sed -e 's/\.gz$//' | sed -e "s/^${oas_ics_prefix}//")

    for f in ${oas_rst_ifs_tm5}; do
      rm -f ${oas_sub_dir}/${f}
      copy_intermediate_storage ${oas_ics_dir}/${oas_ics_prefix}${f}.gz . "TRUE"
      gunzip -c ${oas_ics_prefix}${f}.gz >${oas_sub_dir}/${f}
      rm -f ${oas_ics_prefix}${f}.gz
    done
  fi
}

#####################################################################################################################
# Globals: INIPATH, START_year, ifs_grid_wol LPJG_STATE_DIR, LPJG_ini_member, SCRATCH_TMP_DIR, LPJG_ini,
#
# Arguments:
#   None
# Returns:
#   LPJG_ini_member
# Purpose: IC's for LPJG Component (EC-Earth3/lsm)
#
#####################################################################################################################
function lpjg_init() {

  if [[ $LPJG_ini != "" || $LPJG_STATE_DIR != "" ]]; then

    START_year=${START_date:0:4}
    lpjg_state_dir=lpjg/ini_state/${ifs_grid_wol}
    [[ "${START_date:4:4}" == "0101" ]] && lpjg_rst_dir=lpjg_state_${START_year} || lpjg_rst_dir=lpjg_state_${START_date}

    # first remove any existing lpjg_state folder for the START_year
    cd ${INIPATH}
    [[ -d ${lpjg_state_dir}/${lpjg_rst_dir} ]] && rm -rf ${lpjg_state_dir}/${lpjg_rst_dir}
    # make sure top-level ini_state folder exists
    mkdir -p ${lpjg_state_dir}

    if [[ $LPJG_ini != "" ]]; then

      if [[ ${LPJG_ini_member} != '' ]]; then
        calc_member "${LPJG_ini_member}" ${MEMBER}
        LPJG_ini_member=${out_member}
      fi

      lpjg_ics_dir=${IC_DIR}/lpjg/T${ifs_res_hor}/${LPJG_ini}
      local ini_start_date=$([[ -z "${LPJG_ini_date}" ]] && echo ${START_date} || echo ${LPJG_ini_date})
      lpjg_ics_file=${LPJG_ini}_${LPJG_ini_member}_${ini_start_date}00_lpjg.tgz
      copy_intermediate_storage ${lpjg_ics_dir}/${lpjg_ics_file} ${SCRATCH_TMP_DIR} "TRUE"

      cd ${lpjg_state_dir}
      # untar removing any folder structure (needed when untarring RESTV files)
      # TODO adapt this code for the filenames of restarts generated by other experiments (e.g. RESTV_a1tf_19800101_fc0_1_19800101-19801231.tar.gz)
      mkdir -p ${lpjg_rst_dir}
      tar xzvf ${SCRATCH_TMP_DIR}/${lpjg_ics_file} --transform='s/.*\///' -C ${lpjg_rst_dir}
      rm -rf ${SCRATCH_TMP_DIR}/${lpjg_ics_file}
      cd -

    # special treatment for the lpjg_state directories, maybe user has specified their location in LPJG_STATE_DIR
    elif [[ $LPJG_STATE_DIR != "" ]]; then
      # link to the folder if it already exists in LPJG_STATE_DIR
      if [[ -d ${LPJG_STATE_DIR}/${lpjg_rst_dir} ]] ; then
        ln -sf ${LPJG_STATE_DIR}/${lpjg_rst_dir} ${lpjg_state_dir}
      # untar .tgz file if necessary
      elif [[ -f ${LPJG_STATE_DIR}/${lpjg_rst_dir}.tgz ]] ; then
        # should we untar in the experiment's inidata folder to save space in scratch/projects?
        [[ $LPJG_STATE_DIR_INI == "TRUE" ]] && cd ${lpjg_state_dir} || cd ${LPJG_STATE_DIR}
        # untar removing any folder structure (needed when untarring RESTV files)
        # TODO adapt this code for the filenames of restarts generated by other experiments (e.g. RESTV_a1tf_19800101_fc0_1_19800101-19801231.tar.gz)
        mkdir -p ${lpjg_rst_dir}
        tar xzvf ${LPJG_STATE_DIR}/${lpjg_rst_dir}.tgz --transform='s/.*\///' -C ${lpjg_rst_dir}
        cd -
        [[ $LPJG_STATE_DIR_INI = "TRUE" ]] || ln -sf ${LPJG_STATE_DIR}/${lpjg_rst_dir} ${lpjg_state_dir}
      # if not found, link to the folder (which might be created during the workflow, if not SIM will fail)
      else
        echo "ERROR! cannot find ${lpjg_rst_dir} in ${LPJG_STATE_DIR}"
        [[ "${TEMPLATE_NAME}" != "lsm" ]] && exit 1
        mkdir -p ${LPJG_STATE_DIR}
        ln -sf ${LPJG_STATE_DIR}/${lpjg_rst_dir} ${lpjg_state_dir}
      fi
    fi
  fi
}

#####################################################################################################################
# Globals: TM5_ini, TM5_ini_member, IC_DIR, MEMBER, out_member, TM5_ini_date, ini_start_date, TM5_CONFIG,
# SCRATCH_TMP_DIR
#
# Arguments:
#   None
# Returns:
#   None
# Purpose: IC's for TM5 Component
#
#####################################################################################################################
function tm5_init() {
  if [[ ${TM5_ini} != "" ]]; then
    if [[ ${TM5_ini_member} != '' ]]; then
      calc_member "${TM5_ini_member}" ${MEMBER}
      TM5_ini_member=${out_member}
    fi

    tm5_ics_dir=${IC_DIR}/tm5/${TM5_ini}
    local ini_start_date=$([[ -z "${TM5_ini_date}" ]] && echo ${START_date} || echo ${TM5_ini_date})
    tm5_ics_prefix=${TM5_ini}_${TM5_ini_member}_${ini_start_date}00_
    cd ${tm5_ics_dir}
    # get available tm5 restart files
    tm5_ics_files=$(ls ${tm5_ics_prefix}TM5_restart_${ini_start_date}_0000_glb300x200.nc.gz ${tm5_ics_prefix}save_${ini_start_date}00_glb300x200.hdf.gz 2>/dev/null || true)

    [[ ${tm5_ics_files} = "" ]] && echo "ERROR cannot find TM5 IC files for ${ini_start_date} in ${tm5_ics_dir}" && exit 1

    [[ $TM5_CONFIG =~ "co2" ]] && tmversion="co2" || tmversion="cb05"
    tm5_sub_dir=tm5/restart/${tmversion}-ml${TM5_NLEVS}
    cd ${SCRATCH_TMP_DIR}
    mkdir -p ${tm5_sub_dir}

    for file in ${tm5_ics_files}; do
      copy_intermediate_storage ${tm5_ics_dir}/${file} . "TRUE"
      # Unzip into tm5_sub_dir removing prefix and gz extension from restart file
      gunzip -c ${file} >${tm5_sub_dir}/$(echo "${file%.*}" | sed -e "s/^${tm5_ics_prefix}//")
      rm -f ${file}
    done
  fi
}

######################################################
########################EC-Earth3######################
######################################################

#####################################################################################################################
# Globals: OCE_NUDG, MEMBER
#
# Arguments:
#   None
# Returns:
#   None
# Purpose:  Main function for initialization - EC-Earth3
#
#####################################################################################################################
function ecearth3_init() {

  #
  # IFS
  #

  ifs_setup_members

  #
  # NEMO
  #
  nemo_setup_members

  #
  # OASIS
  #

  #
  # COMMON - Prepare stuff for experiment
  #
  cd ${SCRATCH_TMP_DIR}

  atm_ini $MEMBER

  oce_ini

  [[ "${PISCES_OFF}" == "TRUE" ]] && pisces_offline_forcing

  [[ "${ICE}" == "LIM3" || "${ICE}" == "LIM2" ]] && ice_ini

  [[ "${PISCES}" == "TRUE" ]] &&  pisces_ini

  [[ "$ATM_NUDGING" == "TRUE" ]] && ifs_setup_nudging

  [[ "${OCE_NUDG}" == "TRUE" ]] && nemo_setup_nudging

  [[ "${SURF_RESTO}" == "TRUE" ]] && nemo_surface_restoring

  cd ${INIPATH}

}

######################################################
##########################LSM#########################
######################################################

#####################################################################################################################
# Globals: SCRATCH_TMP_DIR, OSM, MEMBER
#
# Arguments:
#   None
# Returns:
#   None
# Purpose: Main function for initialization - LSM
#
#####################################################################################################################
function lsm_init() {

  cd ${SCRATCH_TMP_DIR}

  if [[ "$OSM" == TRUE ]]; then
    ifs_setup_members
    atm_ini $MEMBER
  fi
}

if [[ "${DEBUG_MODE-}" == "FALSE" ]]; then

  ######################################################
  #
  # MAIN
  #
  ######################################################

  #error handling
  export get_out="false"

  #prepare main inidata folder
  setup_ini_folders

  #init data for the experiment
  ${TEMPLATE_NAME}_init
  cd ${INIPATH}

  # generate the inidata folder - we will make a copy (symbolic links) of the folder
  copy_inidata

  # custom ifs_veg_source
  ifs_veg_source_init

  #if it is a coupled experiment, we setup OASIS coupler
  if [[ "${TEMPLATE_NAME}" == "ecearth3" ]]; then
    oasis_setup_ifs_nemo
  fi

  # IFS-LPJG restarts
  [[ "${TEMPLATE_NAME}" != "lsm" ]] && [[ $LPJG == "TRUE" ]] && oasis_setup_ifs_lpjg
  # special treatment of LPJG ini_state, do it after copying all links
  [[ "${TEMPLATE_NAME}" == "ecearth3" ]] && [[ $LPJG == "TRUE" ]] && lpjg_init
  # if it a lsm run always copy/link LPJG files even if LPJG!=TRUE
  [[ "${TEMPLATE_NAME}" == "lsm" ]] && lpjg_init

  # IFS-TM5 restarts
  if [[ ${TM5} == "TRUE" ]]; then
    oasis_setup_ifs_tm5
    tm5_init
  fi

  #restore path
  cd ${INIPATH}
  if [[ -e inidata/rcf ]]; then
    rm -f rcf
    cp inidata/rcf .
  fi
  #copy ic/restarts to inidata
  [ ! -z "$(ls -A ${SCRATCH_TMP_DIR})" ] && cp -rf --remove-destination ${SCRATCH_TMP_DIR}/* .
  #remove temporary folder used to unpack restarts and ICs
  cd ${ROOTDIR}
  #remove temporary user folder used
  rm -rf ${SCRATCH_TMP_DIR}
  ls -l

  echo "Generating checksum file to check if the inidata is modified during the experiment"

  cd ${ROOTDIR}/${START_date}/${MEMBER}
  inidata_checksum_generate ini

  echo "common.ini Done"

  if [[ "${get_out}" == "true" ]]; then exit 1; fi

fi
+ [[ FALSE == \F\A\L\S\E ]]
+ export get_out=false
+ get_out=false
+ setup_ini_folders
+ local storage_dir=/gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0
+ test_intermediate_storage '-d /gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0'
+ test -d /gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0
+ mkdir_intermediate_storage /gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0 Earth
+ target_group=bsc32
+ target_dir=/gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0
+ [[ 2 -eq 2 ]]
+ target_group=Earth
+ [[ ! -d /gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0 ]]
+ sg Earth 'mkdir -p /gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0'
+ chmod g+w /gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0
+ test_intermediate_storage '-d /gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0'
+ test -d /gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0
+ '[' TRUE == TRUE ']'
+ rm -rf /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/inidata
ls runtime/* >&/dev/null
++ ls 'runtime/*'
+ [[ -n '' ]]
+ mkdir -p /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/inidata
+ rm -rf /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/tmp/t0ni_19931101_fc0_INI
+ mkdir -p /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/tmp/t0ni_19931101_fc0_INI
+ ini_data_dir=/gpfs/projects/bsc32/models/ecearth/trunk/inidata
+ ecearth3_init
+ ifs_setup_members
+ [[ fc0 != '' ]]
+ calc_member fc0 fc0
+ ini_member=fc0
+ membid=fc0
+ membarr=($ini_member)
+ nbmemb=1
+ out_member_perturb=false
+ [[ 1 == 1 ]]
+ [[ fc0 == \a\l\l\_\m\e\m\b\e\r\s ]]
+ out_member=fc0
+ ATM_ini_member=fc0
+ nemo_setup_members
+ [[ fc0 != '' ]]
+ calc_member fc0 fc0
+ ini_member=fc0
+ membid=fc0
+ membarr=($ini_member)
+ nbmemb=1
+ out_member_perturb=false
+ [[ 1 == 1 ]]
+ [[ fc0 == \a\l\l\_\m\e\m\b\e\r\s ]]
+ out_member=fc0
+ OCEAN_ini_member=fc0
+ [[ fc0 != '' ]]
+ calc_member fc0 fc0
+ ini_member=fc0
+ membid=fc0
+ membarr=($ini_member)
+ nbmemb=1
+ out_member_perturb=false
+ [[ 1 == 1 ]]
+ [[ fc0 == \a\l\l\_\m\e\m\b\e\r\s ]]
+ out_member=fc0
+ ICE_ini_member=fc0
+ [[ '' != '' ]]
+ PISCES_ini_member=fc0
+ cd /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/tmp/t0ni_19931101_fc0_INI
+ atm_ini fc0
+ local membid=fc0
+ local 'grib_cmd=grib_set -s dataDate=19931101,dataTime=0'
+ [[ a2s5 != '' ]]
[[ -z "${ATM_ini_date}" ]] && echo ${START_date} || echo ${ATM_ini_date}
++ [[ -z 22990101 ]]
++ echo 22990101
+ local ini_start_date=22990101
+ local ifile=a2s5_fc0_2299010100.tar.gz
+ copy_intermediate_storage /gpfs/projects/bsc32/repository/ic/atmos/T511L91/a2s5/a2s5_fc0_2299010100.tar.gz . TRUE
+ cp -f /gpfs/projects/bsc32/repository/ic/atmos/T511L91/a2s5/a2s5_fc0_2299010100.tar.gz .
+ tar -xzvf a2s5_fc0_2299010100.tar.gz
+ rm -f a2s5_fc0_2299010100.tar.gz
+ mkdir -p ifs/T511L91/19931101
+ grib_set -s dataDate=19931101,dataTime=0 ICMGGa2s5INIT ifs/T511L91/19931101/ICMGGECE3INIT
+ grib_set -s dataDate=19931101,dataTime=0 ICMSHa2s5INIT ifs/T511L91/19931101/ICMSHECE3INIT
+ grib_set -s dataDate=19931101,dataTime=0 ICMGGa2s5INIUA ifs/T511L91/19931101/ICMGGECE3INIUA
+ rm -f ICMGGa2s5INIT ICMSHa2s5INIT ICMGGa2s5INIUA
+ [[ FALSE == \T\R\U\E ]]
+ oce_ini
[[ -z "${OCEAN_ini_date}" ]] && echo ${START_date_1} || echo $(date -d "${OCEAN_ini_date} - 1 day" +%Y%m%d)
++ [[ -z 22990101 ]]
date -d "${OCEAN_ini_date} - 1 day" +%Y%m%d
+++ date -d '22990101 - 1 day' +%Y%m%d
++ echo 22981231
+ local ini_start_date=22981231
[[ -z "${OCEAN_ini}" ]] && echo "FALSE" || echo "${OCEAN_ini}"
++ [[ -z a2s5 ]]
++ echo a2s5
+ nemo_components_ini_ic a2s5 /gpfs/projects/bsc32/repository/ic/ocean/ORCA025L75/a2s5 a2s5_fc0_22981231_restart.nc restart_oce.nc OCE_ini
+ [[ a2s5 != \F\A\L\S\E ]]
+ copy_intermediate_storage /gpfs/projects/bsc32/repository/ic/ocean/ORCA025L75/a2s5/a2s5_fc0_22981231_restart.nc.gz . TRUE
+ cp -f /gpfs/projects/bsc32/repository/ic/ocean/ORCA025L75/a2s5/a2s5_fc0_22981231_restart.nc.gz .
+ gunzip a2s5_fc0_22981231_restart.nc.gz
+ mv a2s5_fc0_22981231_restart.nc restart_oce.nc
+ [[ FALSE == \T\R\U\E ]]
+ [[ LIM3 == \L\I\M\3 ]]
+ ice_ini
[[ -z "${ICE_ini_date}" ]] && echo ${START_date_1} || echo $(date -d "${ICE_ini_date} - 1 day" +%Y%m%d)
++ [[ -z 22990101 ]]
date -d "${ICE_ini_date} - 1 day" +%Y%m%d
+++ date -d '22990101 - 1 day' +%Y%m%d
++ echo 22981231
+ local ini_start_date=22981231
[[ -z "${ICE_ini}" ]] && echo "FALSE" || echo "${ICE_ini}"
++ [[ -z a2s5 ]]
++ echo a2s5
+ nemo_components_ini_ic a2s5 /gpfs/projects/bsc32/repository/ic/ice/ORCA025_LIM3/a2s5 a2s5_fc0_22981231_restart_ice.nc restart_ice.nc ICE_ini
+ [[ a2s5 != \F\A\L\S\E ]]
+ copy_intermediate_storage /gpfs/projects/bsc32/repository/ic/ice/ORCA025_LIM3/a2s5/a2s5_fc0_22981231_restart_ice.nc.gz . TRUE
+ cp -f /gpfs/projects/bsc32/repository/ic/ice/ORCA025_LIM3/a2s5/a2s5_fc0_22981231_restart_ice.nc.gz .
+ gunzip a2s5_fc0_22981231_restart_ice.nc.gz
+ mv a2s5_fc0_22981231_restart_ice.nc restart_ice.nc
+ [[ FALSE == \T\R\U\E ]]
+ [[ FALSE == \T\R\U\E ]]
+ [[ FALSE == \T\R\U\E ]]
+ [[ FALSE == \T\R\U\E ]]
+ cd /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/inidata
+ cd /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/inidata
+ copy_inidata
+ cp --symbolic-link --recursive --force --update /gpfs/projects/bsc32/models/ecearth/trunk/inidata /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0
+ ifs_veg_source_init
+ case ${ifs_veg_source} in
+ echo 'Nothing to do for ifs veg source ifs'
+ [[ ecearth3 == \e\c\e\a\r\t\h\3 ]]
+ oasis_setup_ifs_nemo
+ use_rst_default_if_ic_not_found=FALSE
+ use_default_rst=FALSE
+ [[ -z a2s5 ]]
+ [[ FALSE == \T\R\U\E ]]
+ [[ -z fc0 ]]
+ cd /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/tmp/t0ni_19931101_fc0_INI
+ case ${nem_grid_wol} in
+ oas_ogrd=Ot25
+ oas_sub_dir=oasis/T511-ORCA025/rst
+ oasis_restart_files='rstas.nc rstos.nc'
+ oas_ics_dir=/gpfs/projects/bsc32/repository/ic/coupler/T511-ORCA025/a2s5
[[ -z "${OASIS_ini_date}" ]] && echo ${START_date} || echo ${OASIS_ini_date}
++ [[ -z 22990101 ]]
++ echo 22990101
+ local ini_start_date=22990101
+ oas_ics_prefix=a2s5_fc0_2299010100_
+ [[ FALSE == \T\R\U\E ]]
+ [[ ! -d oasis/T511-ORCA025/rst ]]
+ mkdir -p oasis/T511-ORCA025/rst
+ [[ ! -d xios/ORCA025 ]]
+ mkdir -p xios/ORCA025
+ abort_if_files_not_found=FALSE
+ [[ FALSE == \F\A\L\S\E ]]
+ [[ FALSE == \F\A\L\S\E ]]
+ abort_if_files_not_found=TRUE
+ [[ FALSE == \F\A\L\S\E ]]
+ for f in '${oasis_restart_files}'
+ test_intermediate_storage '-f /gpfs/projects/bsc32/repository/ic/coupler/T511-ORCA025/a2s5/a2s5_fc0_2299010100_rstas.nc.gz'
+ test -f /gpfs/projects/bsc32/repository/ic/coupler/T511-ORCA025/a2s5/a2s5_fc0_2299010100_rstas.nc.gz
+ for f in '${oasis_restart_files}'
+ test_intermediate_storage '-f /gpfs/projects/bsc32/repository/ic/coupler/T511-ORCA025/a2s5/a2s5_fc0_2299010100_rstos.nc.gz'
+ test -f /gpfs/projects/bsc32/repository/ic/coupler/T511-ORCA025/a2s5/a2s5_fc0_2299010100_rstos.nc.gz
+ for f in '${oasis_restart_files}'
+ [[ FALSE == \F\A\L\S\E ]]
+ rm -f oasis/T511-ORCA025/rst/rstas.nc
+ copy_intermediate_storage /gpfs/projects/bsc32/repository/ic/coupler/T511-ORCA025/a2s5/a2s5_fc0_2299010100_rstas.nc.gz . TRUE
+ cp -f /gpfs/projects/bsc32/repository/ic/coupler/T511-ORCA025/a2s5/a2s5_fc0_2299010100_rstas.nc.gz .
+ gunzip -c a2s5_fc0_2299010100_rstas.nc.gz
+ rm -f a2s5_fc0_2299010100_rstas.nc.gz
+ for f in '${oasis_restart_files}'
+ [[ FALSE == \F\A\L\S\E ]]
+ rm -f oasis/T511-ORCA025/rst/rstos.nc
+ copy_intermediate_storage /gpfs/projects/bsc32/repository/ic/coupler/T511-ORCA025/a2s5/a2s5_fc0_2299010100_rstos.nc.gz . TRUE
+ cp -f /gpfs/projects/bsc32/repository/ic/coupler/T511-ORCA025/a2s5/a2s5_fc0_2299010100_rstos.nc.gz .
+ gunzip -c a2s5_fc0_2299010100_rstos.nc.gz
+ rm -f a2s5_fc0_2299010100_rstos.nc.gz
+ [[ FALSE == \T\R\U\E ]]
+ [[ ecearth3 != \l\s\m ]]
+ [[ FALSE == \T\R\U\E ]]
+ [[ ecearth3 == \e\c\e\a\r\t\h\3 ]]
+ [[ FALSE == \T\R\U\E ]]
+ [[ ecearth3 == \l\s\m ]]
+ [[ FALSE == \T\R\U\E ]]
+ cd /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/inidata
+ [[ -e inidata/rcf ]]
ls -A ${SCRATCH_TMP_DIR}
++ ls -A /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/tmp/t0ni_19931101_fc0_INI
+ '[' '!' -z 'ifs
oasis
restart_ice.nc
restart_oce.nc
xios' ']'
+ cp -rf --remove-destination /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/tmp/t0ni_19931101_fc0_INI/ifs /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/tmp/t0ni_19931101_fc0_INI/oasis /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/tmp/t0ni_19931101_fc0_INI/restart_ice.nc /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/tmp/t0ni_19931101_fc0_INI/restart_oce.nc /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/tmp/t0ni_19931101_fc0_INI/xios .
+ cd /gpfs/scratch/bsc32/bsc32627/t0ni
+ rm -rf /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/tmp/t0ni_19931101_fc0_INI
+ ls -l
+ echo 'Generating checksum file to check if the inidata is modified during the experiment'
+ cd /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0
+ inidata_checksum_generate ini
+ filename=ini
+ [[ ini == '' ]]
+ echo 'Generating list of files (and md5sum of each) in the directory, time taken:'
time (find inidata/ -type f -exec md5sum {} \; | sort -k 2 &> ${filename}_tmpl_files.txt ) 2>&1 1>/dev/null 
+ files_time='++ sort -k 2
++ find inidata/ -type f -exec md5sum '\''{}'\'' '\'';'\''

real	1m14.518s
user	1m1.929s
sys	0m4.175s'
+ echo '++ sort -k 2
++ find inidata/ -type f -exec md5sum '\''{}'\'' '\'';'\''

real	1m14.518s
user	1m1.929s
sys	0m4.175s'
+ echo 'Generating list of links in the directory, time taken:'
time (find -L inidata/ -type f -exec readlink -f {} \; | sort &> ${filename}_tmpl_links.txt ) 2>&1 1>/dev/null 
+ links_time='++ sort
++ find -L inidata/ -type f -exec readlink -f '\''{}'\'' '\'';'\''

real	2m15.528s
user	0m0.968s
sys	0m29.998s'
+ echo '++ sort
++ find -L inidata/ -type f -exec readlink -f '\''{}'\'' '\'';'\''

real	2m15.528s
user	0m0.968s
sys	0m29.998s'
+ echo 'common.ini Done'
+ [[ false == \t\r\u\e ]]

###################
# Autosubmit tailer
###################
set -xuve
+ set -xuve
echo $(date +%s) >> ${job_name_ptrn}_STAT
date +%s
++ date +%s
+ echo 1639809873
touch ${job_name_ptrn}_COMPLETED
+ touch /gpfs/scratch/bsc32/bsc32627/t0ni/LOG_t0ni/t0ni_19931101_fc0_INI_COMPLETED
exit 0
+ exit 0
