[INFO] JOBID=54049

job_name_ptrn='/gpfs/scratch/bsc32/bsc32627/t0ni/LOG_t0ni/t0ni_19931101_fc0_TRANSFER_MEMBER'
+ job_name_ptrn=/gpfs/scratch/bsc32/bsc32627/t0ni/LOG_t0ni/t0ni_19931101_fc0_TRANSFER_MEMBER
echo $(date +%s) > ${job_name_ptrn}_STAT
++ date +%s
+ echo 1639863695

###################
# Autosubmit job
###################

#####################################################################################################################
# script for transferring files at member level such as runtime folder and logs
# HPC to local storage
# Maintainers: J.R.Berlin
#####################################################################################################################

set -v
+ set -v
#
# Architecture
#
CHUNK=1
+ CHUNK=1
Chunk_end_date=19931130
+ Chunk_end_date=19931130
Chunk_start_date=19931101
+ Chunk_start_date=19931101
EXPID=t0ni
+ EXPID=t0ni
HPCARCH=marenostrum4
+ HPCARCH=marenostrum4
HPCUSER=bsc32627
+ HPCUSER=bsc32627
HPCHOST=mn1.bsc.es
+ HPCHOST=mn1.bsc.es
CURRENT_ARCH=transfer_node
+ CURRENT_ARCH=transfer_node
HPCPROJ=bsc32
+ HPCPROJ=bsc32
JOBNAME=t0ni_19931101_fc0_TRANSFER_MEMBER
+ JOBNAME=t0ni_19931101_fc0_TRANSFER_MEMBER
#members=fc0
MEMBER=fc0
+ MEMBER=fc0
MODEL=ecearth
+ MODEL=ecearth
PROJDIR=/esarchive/autosubmit/t0ni/proj/auto-ecearth3
+ PROJDIR=/esarchive/autosubmit/t0ni/proj/auto-ecearth3
proj=bsc32
+ proj=bsc32
SCRATCH_DIR=/gpfs/scratch
+ SCRATCH_DIR=/gpfs/scratch
START_date=19931101
+ START_date=19931101
stamp=%Y_%m_%d_%H_%M
+ stamp=%Y_%m_%d_%H_%M
ROOTDIR=/gpfs/scratch/bsc32/bsc32627/t0ni
+ ROOTDIR=/gpfs/scratch/bsc32/bsc32627/t0ni
START_date=19931101
+ START_date=19931101
PROJDEST=auto-ecearth3
+ PROJDEST=auto-ecearth3
USE_DT_COMMANDS=TRUE
+ USE_DT_COMMANDS=TRUE
if [[ -z ${USE_DT_COMMANDS} ]]; then USE_DT_COMMANDS=TRUE; fi
+ [[ -z TRUE ]]
[[ "FALSE" == TRUE ]] && USE_INTERMEDIATE_STORAGE=TRUE || USE_INTERMEDIATE_STORAGE=FALSE
+ [[ FALSE == TRUE ]]
+ USE_INTERMEDIATE_STORAGE=FALSE

if [[ ! -d /gpfs/archive/bsc32 ]]; then
  . ${PROJDIR}/platforms/${CURRENT_ARCH}/filesystem.sh
  # copy ( depending of the executing machine ) the platform folder to the host
  copy_platform_environment_to_host
fi
+ [[ ! -d /gpfs/archive/bsc32 ]]

export HPCPROJ
+ export HPCPROJ
PROJDIR=${ROOTDIR}/${PROJDEST}
+ PROJDIR=/gpfs/scratch/bsc32/bsc32627/t0ni/auto-ecearth3

#
# Dealing with Model Output Transferring Back to Home (plugin)
#
. ${PROJDIR}/platforms/${CURRENT_ARCH}/configure.sh
+ . /gpfs/scratch/bsc32/bsc32627/t0ni/auto-ecearth3/platforms/transfer_node/configure.sh
#####################################################################################################################
# Globals:
#   PROJDIR
# Arguments:
#   None
# Returns:
#   None
# Purpose: main function to prepare execution environment for the underlying platform, that basically consist on
#   setting up the modules to be used, common environment variables and the common paths
#####################################################################################################################
function load_platform_environment() {
  # load utils/filesystem libraries for common env functions
  . ${PROJDIR}/platforms/transfer_node/filesystem.sh
  . ${PROJDIR}/platforms/transfer_node/utils.sh
  #configure all modules needed for run the job
  setup_environment_vars
  # define all paths needed
  setup_paths
}

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: Prepare the modules needed to execute the OASIS restart generation script for Marenostrum4 platform
#####################################################################################################################
function load_modules_oas_rsts_gen() {
  echo "Function generate_oasis_restarts not available in this platform (transfer_node)"
}


# prepare running environment
load_platform_environment
+ load_platform_environment
+ . /gpfs/scratch/bsc32/bsc32627/t0ni/auto-ecearth3/platforms/transfer_node/filesystem.sh
#####################################################################################################################
# Functions for handling filesystem operations for DT platform
# Author: J.R.Berlin
#####################################################################################################################

# load base library to be used in DT machines, for DT machines it needs to be the Architecture choosen for the experiment
. ${PROJDIR}/platforms/${HPCARCH}/filesystem.sh
++ . /gpfs/scratch/bsc32/bsc32627/t0ni/auto-ecearth3/platforms/marenostrum4/filesystem.sh
#####################################################################################################################
# Functions for handling filesystem operations for marenostrum4 platform
# Author: J.R.Berlin
#####################################################################################################################

. ${PROJDIR}/platforms/common/common.filesystem.sh
+++ . /gpfs/scratch/bsc32/bsc32627/t0ni/auto-ecearth3/platforms/common/common.filesystem.sh
#####################################################################################################################
# Common Functions for handling file system operations on the different host types
# of the supported machines
# Author/Maintainer: J.R.Berlin
#####################################################################################################################

#####################################################################################################################
# Executes mkdir + chmod to handle proper rights in the target folder in esarchive partition
# Globals:
#   None
# Arguments:
#   target_dir
# Returns:
#   None
#####################################################################################################################
function mkdir_esarchive() {
  target_dir=$1
  mkdir -p ${target_dir}
  chmod_esarchive ${target_dir}
}

#####################################################################################################################
# Executes chmod with g+s so users of a given group doesnt have access problems to certain outputs and files
# Globals:
#   None
# Arguments:
#   target_dir
# Returns:
#   None
#####################################################################################################################
function chmod_esarchive() {
  target_dir=$1
  chmod =0770,g+s ${target_dir}
}


#####################################################################################################################
# Globals:
# RUN_dir,  SCRATCH_TMP_DIR, MEMBER, out_member
# Arguments:
#   None
# Returns:
#   None
# Purpose: creates the folder passed as parameter and assign proper group rights
#
#####################################################################################################################
function mkdir_intermediate_storage() {
  #default group
  target_group=bsc32
  target_dir=$1
  if [[ "$#" -eq 2 ]]; then
    target_group=$2
  fi

  if [[ ! -d "${target_dir}" ]]; then
    sg ${target_group} "mkdir -p ${target_dir}"
    chmod g+w ${target_dir}
  fi
}

#####################################################################################################################
# Globals:
#   None,
# Arguments:
#   $1: the folder to be removed
# Returns:
#   None
# Purpose: removes the intermediate storage folder passed as parameter
#
#####################################################################################################################
function delete_intermediate_storage() {
  rm -rf $1 2>/dev/null
}

#####################################################################################################################
# Globals:
#   None,
# Arguments:
#   $1: the source folder/file to be copied
#   $2: the target directory
# Returns:
#   None
# Purpose: copies the intermediate storage folder passed as parameter
#
#####################################################################################################################
function copy_intermediate_storage() {
  cp -f $1 $2
}

#####################################################################################################################
# Globals:
#   None,
# Arguments:
#   $1: the ecm folder whose content we want to enumerate
# Returns:
#   None
# Purpose: enumerate the content of the intermediate storage folder passed as parameter
#
#####################################################################################################################
function list_intermediate_storage() {
  ls $1
}

#####################################################################################################################
# Globals:
#   None,
# Arguments:
#   $1: the folder whose permissions we want to change
# Returns:
#   None
# Purpose: change the permissions of the intermediate storage folder passed as parameter
#
#####################################################################################################################
function chmod_intermediate_storage() {
  chmod 744 $1
}

#####################################################################################################################
# Globals:
#   None,
# Arguments:
#   $1: the folder we want to test
# Returns:
#   None
# Purpose: test the intermediate storage folder passed as parameter
#
#####################################################################################################################
function test_intermediate_storage() {
  test $1
}

# check if directory is empty and doesnt contain any file
#####################################################################################################################
# Globals:
# RUN_dir,  SCRATCH_TMP_DIR, MEMBER, out_member
# Arguments:
#   None
# Returns:
#   None
# Purpose: Main function for initialization - ifs3
#
#####################################################################################################################
function is_empty_directory() {
  local target_path=$1
  local result=false
  if [[ -z "$(ls -A ${target_path})" ]]; then
    result=true
  fi
  return result
}

#####################################################################################################################
# Globals:
# RUN_dir,
# Arguments:
#   source= source directory where are the files to be synchronized
#   destination=path to the directory that will be synchronized with the source dir
#   flags=options for the rsync
# Returns:
#   None
# Purpose: synchronizes two directories from different file systems
#          depending of the context, dt commands will be used by using blocking_rsync,
#          and rsync otherwise, <how_many_retry_copy_file> attempts will allowed to try to transfer the files in case
#          of failure
#####################################################################################################################
function rsync_with_retry() {

  source=$2
  destination=$3
  flags=$1
  group=$4
  retry=0
  how_many_retry_copy_file=5
  use_dt_rsync=FALSE

  exist_dtrsync=$(echo $(command -v dtrsync))
  if [[ ! -z "${exist_dtrsync-}" ]]; then
    use_dt_rsync=TRUE
  fi

  while [[ $retry -le $how_many_retry_copy_file ]]; do

    t1=$(date +%s)
    if [[ "${use_dt_rsync-}" == "FALSE" ]]; then
      USE_DT_COMMANDS=FALSE
      echo "Calling blocking_rsync with DT_COMMANDS=FALSE since they are not available on this machine"
    fi
    blocking_rsync "${flags}" ${source} ${destination} ${group}

    if [[ $? ]]; then
      t2=$(date +%s)
      tr=$(date -d "0 -$t1 sec + $t2 sec" +%T)
      echo "done."
      break
    else
      ((retry++))
    fi
  done
  if [[ $retry -eq how_many_retry_copy_file ]]; then
    echo "Transfer error in rsync"
    exit 1
  fi
}

#####################################################################################################################
# Globals:
# PROJDIR, target_destination, REMOTE_PATH
# Arguments:
#   None
# Returns:
#   None
# Purpose: synchronizes the auto-ecearth source folder with changes done in autosubmit auto-ecearth folder of the
# experiment
#
#####################################################################################################################
function transfer_model() {
  local folders_that_will_not_be_transferred=""
  . ${PROJDIR}/plugins/utils.sh
  source=${PROJDIR}
  destination=${target_destination}
  exclusions="--exclude={'templates','utils','test_cases','*.o','*.mod','*.exe','/BLD','ifs-36r4/bin','ifs-36r4/lib/*.a','COMPILE/arch*','COMPILE/cpp*','lpjg/build'}"
  options="${exclusions} --no-group --delete --force --recursive --links --perms --times --specials --verbose"
  if [[ "$#" -eq 1 ]]; then
    folders_that_will_not_be_transferred="$1"
    for folder in ${folders_that_will_not_be_transferred-}; do
      options+=" --exclude='${source}/${folder}' "
    done
  fi
  group=$(id -gn)
  rsync_with_retry "${options}" ${source} "${REMOTE_PATH}${destination}" ${group}
}

#####################################################################################################################
# Globals:
# RUN_dir,
# Arguments:
#   filenames: the file or list of files to be moved to the target location,
#   target_dir: the target directory where the files will be moved
#   is_transfer: defines if the target filesystem is the intermediate storage or esarchive ( permanent storage)
# Returns:
#   None
# Purpose: move files across different file systems, used mainly in CLEAN and TRANSFER jobs, this is the base function,
# each platform can have its own custom implementation if needed ( for instance ecmwf-sc40 has its own version of this
# function, the moved files are removed from the source location
#
#####################################################################################################################
function move_files() {
  local filenames=$1
  local target_dir=$2
  local is_transfer=$3

  if [[ "${is_transfer-}" == "TRUE" ]]; then
    # we are transferring from Scratch to Esarchive directly
    blocking_rsync '--no-perms --links --times --recursive --remove-source-files --no-group  --omit-dir-times --chmod=g+rwxs,o-rwx' "${filenames}" ${target_dir} Earth
  else
    files_to_send=$(echo "${filenames}")
    rsync --perms --links --times --recursive --remove-source-files --no-group  --omit-dir-times --chmod=g+rwxs,o-rwx ${files_to_send} ${target_dir}
  fi
  # check if there were errors
  if [[ $? != 0 ]]; then
    echo "Error occurred on move_files....exiting"
    exit 1
  fi
}

#####################################################################################################################
# Globals:
# RUN_dir, ini_data_dir
# Arguments:
#   None
# Returns:
#   None
# Purpose: copy the initialization files to inidata directory in the scratch directory of the experiment, the command
#   creates links to the files instead of copying these physically for optimizing storage
#
#####################################################################################################################
function copy_inidata() {
  cp --symbolic-link --recursive --force --update ${ini_data_dir} ${RUN_dir}
}

#####################################################################################################################
# Globals:
#
# Arguments:
#   src: source folder for the needed restarts
#   dst: target location where these will be copied
# Returns:
#   None
# Purpose: prepare restarts to be linked in to the inidata folder
#####################################################################################################################
function copy_default_restarts() {
  local src=$1
  local dst=$2
  cp --symbolic-link --recursive --force --update ${src} ${dst}
}

#####################################################################################################################
# Globals:
# RUN_dir, SCRATCH_TMP_DIR, MEMBER, ROOTDIR
# Arguments:
#   None
# Returns:
#   None
# Purpose: clean up the temporary directories used and set run dir, used in almost all platforms
#
#####################################################################################################################
function setup_paths_transfer() {
  rm -rf ${SCRATCH_TMP_DIR}
  mkdir -p ${SCRATCH_TMP_DIR}
  RUN_dir=${ROOTDIR}/${START_date}/${MEMBER}/runtime
}

#####################################################################################################################
# Globals:
# SCRATCH_DIR, HPCPROJ, HPCUSER, EXPID,PROJDEST
# Arguments:
#   None
# Returns:
#   None
# Purpose:  check the existence of the auto-ecearth folder in the experiment folder in the cluster
#
#####################################################################################################################
function check_model_existence() {
  SCRATCH_MODEL_PATH=${SCRATCH_DIR}/${HPCPROJ}/${HPCUSER}/${EXPID}/${PROJDEST}
  MODEL_EXISTS="FALSE"
  if [[ -d ${SCRATCH_MODEL_PATH} ]]; then
    MODEL_EXISTS="TRUE"
    export MODEL_EXISTS
  fi
}

#####################################################################################################################
# Globals:
# PROJDIR, SCRATCH_DIR, HPCPROJ, HPCUSER, EXPID, HPCARCH
# Arguments:
#   None
# Returns:
#   None
# Purpose: prepare/load needed libraries in order to execute the synchronize process
#
#####################################################################################################################
function setup_synchronize() {
  export target_destination=$SCRATCH_DIR/$HPCPROJ/$HPCUSER/$EXPID/
  export REMOTE_PATH=' '
  # import custom architecture
  . ${PROJDIR}/platforms/${HPCARCH}/filesystem.sh
}

#####################################################################################################################
# Globals:
# RUN_dir, USE_REMOTE_HOST, HPCUSER,  HPCHOST
# Arguments:
#    source_cmor_dir= source location for the cmorized files , usually the folder that corresponds to the chunk folder
#    target_cmor_dir= target destination where the files will be moved
#    is_transfer= indicates if the temporary storage will be used or not
# Returns:
#   None
# Purpose: upper level function to transfer the cmorized outputs either in CLEAN ( when USE_INTERMEDIATE_STORAGE=FALSE )
#    or the standard TRANSFER, this is the base function and other platforms may have different implementations such as
#    ecmwf-xc40
#####################################################################################################################
function move_cmor_files() {
  local source_cmor_dir=$1
  local target_cmor_dir=$2
  local is_transfer=$3
  local remote_host=' '

  # used in bscearth000 machine, maybe wise to create a function in platforms for this particular case
  if [[ "${USE_REMOTE_HOST-}" == "TRUE" ]]; then
    cmor_folders=$(ssh ${HPCUSER}@${HPCHOST} "ls -d ${source_cmor_dir}/cmor_*_${CHUNK}/" 2>/dev/null) && returncode=$? || returncode=$?
    if [[ ! -z "${cmor_folders-}" && ${returncode} -eq 0 ]]; then
      for cmor_remote_dir in ${cmor_folders}; do
        list_of_cmor_files=$(ssh ${HPCUSER}@${HPCHOST} "ls ${cmor_remote_dir}" 2>/dev/null) && returncode=$? || returncode=$?
        if [[ ! -z "${list_of_cmor_files-}" && ${returncode} -eq 0 ]]; then
          echo "moving cmor files ....."
          move_files ${HPCUSER}@${HPCHOST}:${cmor_remote_dir} ${target_cmor_dir} ${is_transfer}
          # remove cmor folder after moving the files
          # if there where some files left behind, this method will throw an exception making the job to crash
          ssh ${HPCUSER}@${HPCHOST} "find ${cmor_remote_dir} -type d -delete"
        fi
      done
    fi
  else
    if [[ -d ${source_cmor_dir} ]]; then
      echo "moving cmor files ....."
      cmor_folders=$(ls -d ${source_cmor_dir}/cmor_*_${CHUNK}/ 2>/dev/null) && returncode=$? || returncode=$?
      if [[ ! -z "${cmor_folders-}" && ${returncode} -eq 0 ]]; then
        base_storage_cmor_target_dir=${target_cmor_dir}
        for cmor_remote_dir in ${cmor_folders}; do
          #defaults when we use intermediate storage structure must be the same in the intermediate storage
          if [[ "${is_transfer-}" != "TRUE" ]]; then
            target_cmor_dir=${base_storage_cmor_target_dir}/$(basename ${cmor_remote_dir})
          fi
          move_files ${cmor_remote_dir} ${target_cmor_dir} ${is_transfer}
          # delete the folder, if there where some files left behind, this method will throw an exception making the job to crash
          find ${cmor_remote_dir} -type d -delete
        done
      fi
    fi
  fi
}

#####################################################################################################################
# Globals:
# RUN_dir, ROOTDIR, EXPID
# Arguments:
#   source_pattern: is the destination path for the output
#   destination_path: is the name of the final folder of the destination path /
#   list_of_files_to_copy: list of files to copy
# Returns:
#   None
# Purpose: Local transfer of model output, the function first retrieve the list of files to be transferred and then
#     if such file is not empty proceed to transfer them
#####################################################################################################################
function do_transfer() {

  #  to leave the dt command output in log folder
  cd ${ROOTDIR}/LOG_${EXPID}

  source_pattern=$1
  destination_path=$2
  source_path=$(dirname ${source_pattern[0]})
  local remote_host=' '

  get_list_of_files_to_copy && returncode=$? || returncode=$?
  #apply mkdir and let the rsync apply the chmod
  if [[ ! -z "${list_of_files_to_copy-}" ]]; then
    mkdir -p ${destination_path}
    for source_file_name in $list_of_files_to_copy; do
      echo "transferring file ... "${source_file_name} ${destination_path}
      blocking_rsync '--no-perms --links --times --recursive --remove-source-files --no-group  --omit-dir-times --chmod=g+rwxs,o-rwx' ${remote_host}${source_file_name} ${destination_path} Earth
    done
  fi
}

#####################################################################################################################
# Globals:
# USE_REMOTE_HOST, HPCUSER, HPCHOST, source_pattern
# Arguments:
#   None
# Returns:
#   None
# Purpose: get the list of files to be copied to the cluster, used by transfer/clean templates
#
#####################################################################################################################
function get_list_of_files_to_copy() {
  if [[ "${USE_REMOTE_HOST-}" == "TRUE" ]]; then
    list_of_files_to_copy=$(ssh ${HPCUSER}@${HPCHOST} "ls ${source_pattern}" 2>/dev/null)
    remote_host=${HPCUSER}@${HPCHOST}:
  else
    list_of_files_to_copy=$(ls ${source_pattern} 2>/dev/null)
  fi
}

#####################################################################################################################
# Globals:
# RUN_dir,
# Arguments:
#   None
# Returns:
#   None
# Purpose: performs the synchronization by transferring the model files to the cluster for preparation before running
# the experiment, is the common interface for the synchronization template, implementation may be different depending
# on the underlying architecture where the sync needs to be executed
#
#####################################################################################################################
function do_synchronize() {
  folders_that_will_not_be_transferred=""
  if [[ "$#" -eq 1 ]] ; then
    folders_that_will_not_be_transferred="$1"
  fi
  echo "Calling function transfer_model"
  transfer_model ${folders_that_will_not_be_transferred}
  echo "common.synchronize Done"
}

#####################################################################################################################
# Globals:
# RUN_dir, INTERMEDIATE_EXP_DIR, MODEL, EXPID
# Arguments:
#   None
# Returns:
#   None
# Purpose: Prepare source and target paths for CLEAN/TRANSFER
#####################################################################################################################
function setup_paths_localtrans() {
  src=${INTERMEDIATE_EXP_DIR}
  local_INTERMEDIATE_EXP_DIR=/gpfs/archive/bsc32/esarchive/exp/${MODEL}
  destination_folder=${local_INTERMEDIATE_EXP_DIR}/${EXPID}/original_files/
}

# copy to scratch the platforms files
#####################################################################################################################
# Globals:
# ROOTDIR, PROJDEST, PROJDIR
# Arguments:
#   None
# Returns:
#   None
# Purpose: send the initial set of libraries used by the synchronize job in order to be able to run
#
#####################################################################################################################
function copy_platform_environment_to_host() {
  mkdir -p ${ROOTDIR}/${PROJDEST}/platforms
  rsync --recursive --links --perms --times --owner --devices --specials --verbose --omit-dir-times ${PROJDIR}/platforms/ ${ROOTDIR}/${PROJDEST}/platforms
  rsync --recursive --links --perms --times --owner --devices --specials --verbose --omit-dir-times ${PROJDIR}/plugins/ ${ROOTDIR}/${PROJDEST}/plugins
}

#####################################################################################################################
# Globals:
# RUN_dir,  LOGDIR, CHUNK, START_date, MEMBER, STAMP, PATHOUT
# Arguments:
#   None
# Returns:
#   None
# Purpose: copy logs to the proper storage location before wiping out the experiment folder ( used by CLEAN_EXPERIMENT )
#####################################################################################################################
function save_logs() {
  echo "We are now storing log files"
  cd ${LOGDIR}
  logs_tarfile="logfiles_${EXPID}_${STAMP}.tar"
  IS_TRANSFER=TRUE
  prepare_logs && returncode=$? || returncode=$?
  if [[ ${returncode} -eq 0 ]] || [[ ${returncode} -eq 123 ]]; then
    [[ ! -f ../${logs_tarfile}.gz ]] && gzip -9 ../${logs_tarfile}
    local pathout_logfiles="${PATHOUT}/logfiles/"
    move_files ../${logs_tarfile}.gz ${pathout_logfiles} ${IS_TRANSFER}
    rm -f ../${logs_tarfile}.gz
  fi
}

#####################################################################################################################
# Globals:
# INTERMEDIATE_EXP_DIR, EXPID, ROOTDIR, RM ( alias for deleting files )
# Arguments:
#   None
# Returns:
#   None
# Purpose: wipe out the experiment folders (used intermediate storage, scratch)
#####################################################################################################################
function remove_experiment_folders() {
  #remove the folder in intermediate storage (if it was used)
  delete_intermediate_storage ${INTERMEDIATE_EXP_DIR}/${EXPID}
  #delete scratch folder
  rm -rf ${ROOTDIR}/*
  # needed for indicating that the job finished successfully by autosubmit
  mkdir -p ${ROOTDIR}/LOG_${EXPID}/
}

#####################################################################################################################
# any override from the common libraries defined in filesystem.sh or new function goes here
#####################################################################################################################

#####################################################################################################################
# Globals:
#   EXPID, MODEL, INTERMEDIATE_EXP_DIR, EXPID, USE_INTERMEDIATE_STORAGE
# Arguments:
#   None
# Returns:
#   None
# Purpose: template function for Marenostrum4 for setting up common paths ( used as setup_paths_transfer_${HPCARCH} )
#
#####################################################################################################################
function setup_paths_transfer_marenostrum4() {
  #base function
  setup_paths_transfer
  #custom settings
  if [[ "${USE_INTERMEDIATE_STORAGE-}" == "TRUE" ]]; then
    PATHOUT="${INTERMEDIATE_EXP_DIR}/${EXPID}/"
  else
    local_INTERMEDIATE_EXP_DIR=/gpfs/archive/bsc32/esarchive/exp/${MODEL}
    PATHOUT="${local_INTERMEDIATE_EXP_DIR}/${EXPID}/original_files/"
  fi
}

#####################################################################################################################
# any override from the common libraries defined in filesystem.sh or new function goes here
#####################################################################################################################

#####################################################################################################################
# Globals:
# ROOTDIR, PROJDEST, PROJDIR
# Arguments:
#   None
# Returns:
#   None
# Purpose: send the initial set of libraries used by the synchronize job in order to be able to run
#
#####################################################################################################################
function copy_platform_environment_to_host() {
  echo "DT host doesn´t need to transfer the platforms files"
}

#####################################################################################################################
# Globals:
# HPCARCH
# Arguments:
#   None
# Returns:
#   None
# Purpose: Prepare common paths for transfer for the dt machine
#
#####################################################################################################################
function setup_paths_transfer_transfer_node() {
  setup_paths_transfer_${HPCARCH}
}
+ . /gpfs/scratch/bsc32/bsc32627/t0ni/auto-ecearth3/platforms/transfer_node/utils.sh
#####################################################################################################################
# Functions for handling filesystem operations for data transfer machines part of marenostrum4 platform
# Author: J.R.Berlin
#####################################################################################################################

. ${PROJDIR}/platforms/marenostrum4/utils.sh
++ . /gpfs/scratch/bsc32/bsc32627/t0ni/auto-ecearth3/platforms/marenostrum4/utils.sh
#####################################################################################################################
# Functions for handling filesystem operations for marenostrum4 platform
# Author: J.R.Berlin
#####################################################################################################################

#. ${PROJDIR}/platforms/common/common.utils.sh

#####################################################################################################################
# any override from the common libraries defined in filesystem.sh or new function goes here
#####################################################################################################################

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: load the common modules required by standard jobs in Marenostrum4
#####################################################################################################################
function setup_modules() {
  set +xuve
  module purge
  module load intel/2018.3
  module load mkl/2018.3
  module load impi/2018.3
  module load netcdf/4.2
  module load hdf5/1.8.19
  module load perl/5.26
  module load fftw/3.3.6
  module load python/2.7.13
  module load udunits/2.2.25
  module load gsl/2.4
  module load nco/4.2.3_netcdf-4.2
  module load CDO/1.7.2
  module list
  set -xuve
}

#####################################################################################################################
# Globals:
#   USER
# Arguments:
#   None
# Returns:
#   None
# Purpose: Setup common paths as global variables, required by the different jobs running un the underlying platform
#           used in nord3, transfer_node and bscearth000 as well
####################################################################################################################
function setup_common_paths() {
  # common globals
  MODELS_DIR=/gpfs/projects/bsc32/models
  export SCRATCH=/gpfs/scratch/$(id -gn)/${USER}
  GROUP=bsc32
  # check if the machine has the environment tmp var defined
  SCRATCH_TMP_DIR=${SCRATCH}/${EXPID}/${START_date}/${MEMBER}/tmp/${JOBNAME}
  #initial conditions dirr for the platform, in order to have this working properly we need
  #all IC/rst files in place synchronized from bsc32/projects
  INTERMEDIATE_EXP_DIR=/gpfs/projects/${GROUP}/repository/exp
  EXTRA_DIR=/gpfs/projects/${GROUP}/repository
  IC_DIR=/gpfs/projects/${GROUP}/repository/ic
}

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: set the global variables and functions to be used by the jobs running in the underlying architecture
#          used in nord3, transfer_node and bscearth000 as well
#####################################################################################################################
function setup_common_environment_vars() {
  # common globals
  MODELS_DIR=/gpfs/projects/bsc32/models
  GROUP=bsc32
}

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: set the common set of global variables to be used for the jobs running on the underlying archictecture
#####################################################################################################################
function setup_environment_vars() {
  # common globals
  setup_common_environment_vars
  export ECE3_POSTPROC_MACHINE=mn4
}

#####################################################################################################################
# Globals:
#   CURRENT_ARCH, HPCARCH
# Arguments:
#   None
# Returns:
#   None
# Purpose: set the global variables refering to common paths to be used by the jobs running in the
#         underlying architecture, used in nord3, transfer_node and bscearth000 as well
#####################################################################################################################
function setup_paths() {
  setup_common_paths
  if [[ "${CURRENT_ARCH}" == "${HPCARCH}" ]]; then
    setup_paths_libs
  fi
}

#####################################################################################################################
# Globals:
#   USER, PATH
# Arguments:
#   None
# Returns:
#   None
# Purpose: auxiliary function to load paths to the different modules that will be load when the job
#         executes in the cluster
#####################################################################################################################
function setup_paths_libs() {
  grib_api_path=/apps/GRIB/1.14.0/INTEL
  PATH=${grib_api_path}/bin:${PATH}

  export SCRATCH=/gpfs/scratch/$(id -gn)/${USER}
  export PATH=/gpfs/projects/bsc32/repository/apps/rebuild_nemo_marenostrum4:${PATH}
  export PYTHONPATH=${grib_api_path}/lib/python2.7/site-packages/grib_api:${PYTHONPATH-}
  export GRIB_BIN_PATH=${grib_api_path}/bin
  export GRIB_DEFINITION_PATH=${grib_api_path}/share/grib_api/definitions
  export GRIB_SAMPLES_PATH=${grib_api_path}/share/grib_api/ifs_samples/grib1
}

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: auxiliary function to load paths to the different modules that will be load when the job
#          executes in the cluster, used in transfer machines!
#####################################################################################################################
function setup_environment_vars_local() {
  setup_common_environment_vars
  export ECE3_POSTPROC_MACHINE=mn4
}

#used in transfer machines !

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: auxiliary function to set the intermediate storage root path, used in transfer machines, this function
# overrides the base function in common.filesystem.sh
#####################################################################################################################
function setup_paths_local() {
  echo "MN4 Paths"
  GROUP=bsc32
  export INTERMEDIATE_EXP_DIR=/gpfs/projects/${GROUP}/repository/exp
}

#####################################################################################################################
# any override from the common libraries defined in filesystem.sh or new function goes below here
#####################################################################################################################

#####################################################################################################################
# Globals:
#   GROUP, USER, EXPID, START_date, MEMBER, GROUP, PATH
# Arguments:
#   None
# Returns:
#   None
# Purpose: set the global variables refering to common paths to be used by the jobs running in the
#         underlying architecture ( initial conditions, grib modules, intermediate storage,etc )
#####################################################################################################################
function setup_paths() {
  setup_common_paths
}

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: auxiliary function to common environment vars that will be used when the job
#          executes in the transfer node
#####################################################################################################################
function setup_environment_vars() {
  setup_common_environment_vars
  #Machine for the ece3postproc
  export ECE3_POSTPROC_MACHINE=mn4
  export SCRATCH=/gpfs/scratch/$(id -gn)/${USER}
}
+ setup_environment_vars
+ setup_common_environment_vars
+ MODELS_DIR=/gpfs/projects/bsc32/models
+ GROUP=bsc32
+ export ECE3_POSTPROC_MACHINE=mn4
+ ECE3_POSTPROC_MACHINE=mn4
++ id -gn
+ export SCRATCH=/gpfs/scratch/bsc32/bsc32627
+ SCRATCH=/gpfs/scratch/bsc32/bsc32627
+ setup_paths
+ setup_common_paths
+ MODELS_DIR=/gpfs/projects/bsc32/models
++ id -gn
+ export SCRATCH=/gpfs/scratch/bsc32/bsc32627
+ SCRATCH=/gpfs/scratch/bsc32/bsc32627
+ GROUP=bsc32
+ SCRATCH_TMP_DIR=/gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/tmp/t0ni_19931101_fc0_TRANSFER_MEMBER
+ INTERMEDIATE_EXP_DIR=/gpfs/projects/bsc32/repository/exp
+ EXTRA_DIR=/gpfs/projects/bsc32/repository
+ IC_DIR=/gpfs/projects/bsc32/repository/ic

set -vx
+ set -vx

. ${PROJDIR}/plugins/utils.sh
+ . /gpfs/scratch/bsc32/bsc32627/t0ni/auto-ecearth3/plugins/utils.sh


#####################################################################################################################
# blocking_rsync using dt_commands
# Globals:
#   None
# Arguments:
#   options, source, destination, target_group USE_DT_COMMANDS
# Returns:
#   None
#####################################################################################################################
function blocking_rsync() {
  options=$1
  source=$2
  destination=$3
  target_group=$4
  if [[ -z ${USE_DT_COMMANDS-} ]]; then USE_DT_COMMANDS=TRUE; fi

  if [[ ${USE_DT_COMMANDS} == "FALSE" ]]; then
    sg ${target_group} "rsync ${options} ${source} ${destination}"
  else
    t1=$(date +%s)
    jid=$(dtsgrsync ${target_group} $options $source $destination | awk '{print $4}')
    if [[ -z ${jid} ]]; then
      USE_DT_COMMANDS=FALSE
      blocking_rsync "${options}" ${source} ${destination} ${target_group}
    else
      jsc=$(dtq -j $jid | wc -l)
      jst=$(dtq -j $jid | awk '{print $4}' | tail -1)
      echo "The job $jid is $jst"

      while [[ $jsc -gt 1 && $jst != "STATE" ]]; do
        sleep ${sleep_time:-60}
        jsc=$(dtq -j $jid | wc -l)
        jst=$(dtq -j $jid | awk '{print $4}' | tail -1)
        echo "The job $jid is $jst"
      done

      t2=$(date +%s)
      tr=$(date -d "0 -$t1 sec + $t2 sec" +%T)

      # exit ok
      # .err empty and .err file exists
      if [[ -f dtsgrsync_${jid}.err ]] && [[ ! -s dtsgrsync_${jid}.err ]]; then
        rm -f dtsgrsync_${jid}*
        return
      else
        echo "the "dtsgrsync_${jid}.err " is not empty"
        cat dtsgrsync_${jid}.err
        exit 1
      fi
    fi
  fi

}

# Functions to get the grid size to be used for CDO and S2DV
#
# Written by Pablo Echevarria
#
# Barcelona Supercomputing Center - Earth Science

#####################################################################################################################
# Get grid dims
# Globals:  NEMO_GRID
# Arguments:
#   TEMPLATE NEMGRID IFSGRID
# Returns:
#   GRID_S2DV GRID_CDO
#####################################################################################################################
function get_grid_dims() {

  local MODEL=${TEMPLATE/3/}

  NEMO_GRID=$(echo $NEMGRID | awk -F\L '{print $1}')

  if [[ $MODEL == 'ecearth' ]] || [[ $MODEL == 'ifs' ]] || [[ $MODEL == 'lsm' ]]; then
    GRID_POST=$IFSGRID
    case ${IFSGRID} in
    'T159L62')
      GRID_CDO='t106grid'
      GRID_PLOT='320x160'
      ;; # For cdo, the T159 is the t106
      # Correspondance betweeen reduced and regular gaussian grid
    'T255L91' | 'T255L62') GRID_CDO='t170grid' GRID_PLOT='512x256' ;;
    'T511L91') GRID_CDO='t340grid' GRID_PLOT='1024x512' ;;
    'T799L62' | 'T799L91')
      echo "Is the mask ready for this configuration?"
      stop
      ;;
    *)
      echo "IFS grid is not defined"
      exit 1
      ;;
    esac

  elif [[ $MODEL == 'nemo' ]]; then
    case $NEMO_GRID in
    'ORCA2')
      GRID_POST='T159L62'
      GRID_CDO='t106grid'
      GRID_PLOT='320x160'
      ;;
    'ORCA1')
      GRID_POST='T255L91'
      GRID_CDO='t170grid'
      GRID_PLOT='512x256'
      ;;
    'ORCA025')
      GRID_POST='T511L91'
      GRID_CDO='t340grid'
      GRID_PLOT='1024x512'
      ;;
    *)
      echo "Nemo grid is not defined"
      exit 1
      ;;
    esac
  else
    echo "Model is not defined"
    exit 1
  fi
  export GRID_POST
  export GRID_CDO
  export GRID_PLOT

}

#####################################################################################################################
# generate a checksum and store it in a file for the inidata folder
# Globals:
#   None
# Arguments:
#   Checksum_filename
# Returns:
#   None
#####################################################################################################################
function inidata_checksum_generate() {

  filename=$1

  if [[ "$filename" == "" ]]; then
    echo "Filename argument is required to store the checksum results, exiting now"
    exit 1
  fi

  # this generates the md5sum for every file present in the experiment inidata folder and a global md5sum of the complete list
  echo "Generating list of files (and md5sum of each) in the directory, time taken:"
  files_time="$(time (find inidata/ -type f -exec md5sum {} \; | sort -k 2 &> ${filename}_tmpl_files.txt ) 2>&1 1>/dev/null )"
  echo "${files_time}"

  # this creates a list of the paths of all the links and files and generates the md5sum for that list
  echo "Generating list of links in the directory, time taken:"
  links_time="$(time (find -L inidata/ -type f -exec readlink -f {} \; | sort &> ${filename}_tmpl_links.txt ) 2>&1 1>/dev/null )"
  echo "${links_time}"
}
. ${PROJDIR}/plugins/transfer.sh
+ . /gpfs/scratch/bsc32/bsc32627/t0ni/auto-ecearth3/plugins/transfer.sh
#####################################################################################################################
# Common Functions for handling file system operations to deal with transferring the model output from the
# HPC to local storage.
# machines for job handling/management
# Maintainers: J.R.Berlin
#####################################################################################################################

. ${PROJDIR}/platforms/${HPCARCH}/utils.sh
++ . /gpfs/scratch/bsc32/bsc32627/t0ni/auto-ecearth3/platforms/marenostrum4/utils.sh
#####################################################################################################################
# Functions for handling filesystem operations for marenostrum4 platform
# Author: J.R.Berlin
#####################################################################################################################

#. ${PROJDIR}/platforms/common/common.utils.sh

#####################################################################################################################
# any override from the common libraries defined in filesystem.sh or new function goes here
#####################################################################################################################

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: load the common modules required by standard jobs in Marenostrum4
#####################################################################################################################
function setup_modules() {
  set +xuve
  module purge
  module load intel/2018.3
  module load mkl/2018.3
  module load impi/2018.3
  module load netcdf/4.2
  module load hdf5/1.8.19
  module load perl/5.26
  module load fftw/3.3.6
  module load python/2.7.13
  module load udunits/2.2.25
  module load gsl/2.4
  module load nco/4.2.3_netcdf-4.2
  module load CDO/1.7.2
  module list
  set -xuve
}

#####################################################################################################################
# Globals:
#   USER
# Arguments:
#   None
# Returns:
#   None
# Purpose: Setup common paths as global variables, required by the different jobs running un the underlying platform
#           used in nord3, transfer_node and bscearth000 as well
####################################################################################################################
function setup_common_paths() {
  # common globals
  MODELS_DIR=/gpfs/projects/bsc32/models
  export SCRATCH=/gpfs/scratch/$(id -gn)/${USER}
  GROUP=bsc32
  # check if the machine has the environment tmp var defined
  SCRATCH_TMP_DIR=${SCRATCH}/${EXPID}/${START_date}/${MEMBER}/tmp/${JOBNAME}
  #initial conditions dirr for the platform, in order to have this working properly we need
  #all IC/rst files in place synchronized from bsc32/projects
  INTERMEDIATE_EXP_DIR=/gpfs/projects/${GROUP}/repository/exp
  EXTRA_DIR=/gpfs/projects/${GROUP}/repository
  IC_DIR=/gpfs/projects/${GROUP}/repository/ic
}

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: set the global variables and functions to be used by the jobs running in the underlying architecture
#          used in nord3, transfer_node and bscearth000 as well
#####################################################################################################################
function setup_common_environment_vars() {
  # common globals
  MODELS_DIR=/gpfs/projects/bsc32/models
  GROUP=bsc32
}

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: set the common set of global variables to be used for the jobs running on the underlying archictecture
#####################################################################################################################
function setup_environment_vars() {
  # common globals
  setup_common_environment_vars
  export ECE3_POSTPROC_MACHINE=mn4
}

#####################################################################################################################
# Globals:
#   CURRENT_ARCH, HPCARCH
# Arguments:
#   None
# Returns:
#   None
# Purpose: set the global variables refering to common paths to be used by the jobs running in the
#         underlying architecture, used in nord3, transfer_node and bscearth000 as well
#####################################################################################################################
function setup_paths() {
  setup_common_paths
  if [[ "${CURRENT_ARCH}" == "${HPCARCH}" ]]; then
    setup_paths_libs
  fi
}

#####################################################################################################################
# Globals:
#   USER, PATH
# Arguments:
#   None
# Returns:
#   None
# Purpose: auxiliary function to load paths to the different modules that will be load when the job
#         executes in the cluster
#####################################################################################################################
function setup_paths_libs() {
  grib_api_path=/apps/GRIB/1.14.0/INTEL
  PATH=${grib_api_path}/bin:${PATH}

  export SCRATCH=/gpfs/scratch/$(id -gn)/${USER}
  export PATH=/gpfs/projects/bsc32/repository/apps/rebuild_nemo_marenostrum4:${PATH}
  export PYTHONPATH=${grib_api_path}/lib/python2.7/site-packages/grib_api:${PYTHONPATH-}
  export GRIB_BIN_PATH=${grib_api_path}/bin
  export GRIB_DEFINITION_PATH=${grib_api_path}/share/grib_api/definitions
  export GRIB_SAMPLES_PATH=${grib_api_path}/share/grib_api/ifs_samples/grib1
}

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: auxiliary function to load paths to the different modules that will be load when the job
#          executes in the cluster, used in transfer machines!
#####################################################################################################################
function setup_environment_vars_local() {
  setup_common_environment_vars
  export ECE3_POSTPROC_MACHINE=mn4
}

#used in transfer machines !

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: auxiliary function to set the intermediate storage root path, used in transfer machines, this function
# overrides the base function in common.filesystem.sh
#####################################################################################################################
function setup_paths_local() {
  echo "MN4 Paths"
  GROUP=bsc32
  export INTERMEDIATE_EXP_DIR=/gpfs/projects/${GROUP}/repository/exp
}

#####################################################################################################################
# Globals:
# EXPID, RUN_dir,  CHUNK, START_date, MEMBER, LPJG, OSM, PATHOUT, CURRENT_ARCH, destination_folder,
# USE_INTERMEDIATE_STORAGE, MODEL
# Arguments:
#   Chunk_START_DATE, Chunk_END_DATE
#   local_INTERMEDIATE_EXP_DIR, START_date, MEMBER, HPCARCH, INTERMEDIATE_EXP_DIR, EXPID, Chunk_start_date
#   Chunk_end_date
# Arguments :
#   None
# Returns:
#   None
# Purpose: Perform the transfer of the model output and several other type of files generated by the different Auto-EC-
#   Earth jobs, this function is used in TRANSFER and TRANSFER_MEMBER
#####################################################################################################################
function transfer_output() {

  Chunk_start_date=$1 #%Chunk_START_DATE%
  Chunk_end_date=$2   #%Chunk_END_DATE%

  echo "platform is: ${HPCARCH}"
  echo "EXPID is: ${EXPID}"
  echo "start dates: ${START_date}"
  echo "members per start date: ${MEMBER}"

  setup_paths_localtrans
  #apply mkdir + chmod will be invoked in each transfer to handle the folder/file rights properly
  if [[ ! -d ${destination_folder} ]]; then
    mkdir_esarchive ${destination_folder}
  fi
  #needed for TRANSFER_MEMBER
  if [[ ${USE_INTERMEDIATE_STORAGE-} == "TRUE" ]]; then
    remote_folder=${src}/${EXPID}/${START_date}/${MEMBER}
  else
    remote_folder=${src}/${MODEL}/${EXPID}/original_files/${START_date}/${MEMBER}
  fi
  # set the mask for padding the chunk number to three digits
  chunk_zero_padded=$(printf "%03d\n" ${CHUNK})

  for type in $output_types; do
    case $type in
    logfiles)
      # transfer logs corresponding to the JOBS that have ran
      do_transfer "${src}/${EXPID}/logfiles/${type}_${EXPID}_${START_date}_${MEMBER}_*" ${destination_folder}/logfiles
      ;;
    LOG)
      # transfer logs corresponding to the model logs that have ran for the given chunk
      do_transfer "${src}/${EXPID}/logfiles/LOG_${EXPID}_${START_date}_${MEMBER}_${chunk_zero_padded}_${Chunk_start_date}-${Chunk_end_date}.tar*" ${destination_folder}/logfiles
      ;;
    MM* | diags)
      do_transfer "${remote_folder}/outputs/${type}*_*${Chunk_start_date}-${Chunk_end_date}*" ${destination_folder}/${START_date}/${MEMBER}/outputs
      ;;
    REST* | restart)
      do_transfer "${remote_folder}/restarts/${type}_${EXPID}_${START_date}_${MEMBER}_${chunk_zero_padded}_${Chunk_start_date}-${Chunk_end_date}.tar*" ${destination_folder}/${START_date}/${MEMBER}/restarts
      ;;
    IC)
      do_transfer "${remote_folder}/ic/${type}_*${Chunk_start_date}-${Chunk_end_date}*" ${destination_folder}/${START_date}/${MEMBER}/ic
      ;;
    ORCA | $(eval echo ${EXPID}_))
      do_transfer "${remote_folder}/outputs/${type}*${Chunk_start_date}-${Chunk_end_date}*" ${destination_folder}/${START_date}/${MEMBER}/outputs
      ;;
    ICM)
      do_transfer "${remote_folder}/outputs/DDA_${EXPID}_${START_date}_${MEMBER}_${chunk_zero_padded}_${Chunk_start_date}-${Chunk_end_date}.tar*" ${destination_folder}/${START_date}/${MEMBER}/outputs
      ;;
    CMOR)
      move_cmor_files ${src}/${EXPID}/cmorfiles/${START_date}/${MEMBER} ${destination_folder}/cmorfiles "TRUE"
      # TODO ET check this!
      #-      move_cmor_files ${src}/${EXPID}/cmorfiles/cmor_${CHUNK}/ ${destination_folder}/cmorfiles "TRUE"
      #+      move_cmor_files ${src}/${EXPID}/cmorfiles/ ${destination_folder}/cmorfiles "TRUE"
      ;;
    ECE3POST)
      do_transfer "${remote_folder}/ece3post/*_${Chunk_start_date}-${Chunk_end_date}.tar*" ${destination_folder}/${START_date}/${MEMBER}/ece3post
      ;;
    runtime)
      do_transfer ${src}/${EXPID}/runtime/runtime_${EXPID}_${START_date}_${MEMBER}* ${destination_folder}/runtime
      # TODO ET check this!
      #-      do_transfer ${src}/${EXPID}/${START_date}/${MEMBER}/runtime* ${destination_folder}/runtime
      #+      do_transfer "${src}/${EXPID}/${START_date}/${MEMBER}/runtime*" ${destination_folder}/runtime
      ;;
    land_param | lpjg_forcing | icmcl)
      # TODO check correct paths in other cases - should not be remote_folder in first arg!
      do_transfer "${src}/${EXPID}/${START_date}/${MEMBER}/outputs/osm/${type}/*" ${destination_folder}/${START_date}/${MEMBER}/outputs/${type}
      ;;
    *)
      echo "$type is not a valid model output type"
      exit 1
      ;;
    esac

    if [ "$?" -ne "0" ]; then
      exit 1
    fi

  done

}

# transfer logfiles and runtime
# For IFS+LPJG or OSM, we end up with a bunch of small files in icmcl, land_param and lpjg_forcing
# for now we copy them without the dt commands since do_transfer copies them one at a time
# and this takes much longer than doing rsync directly
# This is done at the end of the experiment to avoid transfering the files in several TRANSFER jobs at the same time.
# An improvement would be to do this in the transfer.tmpl.sh, selecting the files which belong to the chunk.
output_types="logfiles runtime icmcl land_param lpjg_forcing"
+ output_types='logfiles runtime icmcl land_param lpjg_forcing'
transfer_output ${Chunk_start_date} ${Chunk_end_date}
+ transfer_output 19931101 19931130
+ Chunk_start_date=19931101
+ Chunk_end_date=19931130
+ echo 'platform is: marenostrum4'
+ echo 'EXPID is: t0ni'
+ echo 'start dates: 19931101'
+ echo 'members per start date: fc0'
+ setup_paths_localtrans
+ src=/gpfs/projects/bsc32/repository/exp
+ local_INTERMEDIATE_EXP_DIR=/gpfs/archive/bsc32/esarchive/exp/ecearth
+ destination_folder=/gpfs/archive/bsc32/esarchive/exp/ecearth/t0ni/original_files/
+ [[ ! -d /gpfs/archive/bsc32/esarchive/exp/ecearth/t0ni/original_files/ ]]
+ [[ FALSE == \T\R\U\E ]]
+ remote_folder=/gpfs/projects/bsc32/repository/exp/ecearth/t0ni/original_files/19931101/fc0
++ printf '%03d\n' 1
+ chunk_zero_padded=001
+ for type in '$output_types'
+ case $type in
+ do_transfer '/gpfs/projects/bsc32/repository/exp/t0ni/logfiles/logfiles_t0ni_19931101_fc0_*' /gpfs/archive/bsc32/esarchive/exp/ecearth/t0ni/original_files//logfiles
+ cd /gpfs/scratch/bsc32/bsc32627/t0ni/LOG_t0ni
+ source_pattern='/gpfs/projects/bsc32/repository/exp/t0ni/logfiles/logfiles_t0ni_19931101_fc0_*'
+ destination_path=/gpfs/archive/bsc32/esarchive/exp/ecearth/t0ni/original_files//logfiles
++ dirname '/gpfs/projects/bsc32/repository/exp/t0ni/logfiles/logfiles_t0ni_19931101_fc0_*'
+ source_path=/gpfs/projects/bsc32/repository/exp/t0ni/logfiles
+ local 'remote_host= '
+ get_list_of_files_to_copy
+ [[ '' == \T\R\U\E ]]
++ ls '/gpfs/projects/bsc32/repository/exp/t0ni/logfiles/logfiles_t0ni_19931101_fc0_*'
+ list_of_files_to_copy=
+ returncode=2
+ [[ ! -z '' ]]
+ '[' 0 -ne 0 ']'
+ for type in '$output_types'
+ case $type in
++ eval echo t0ni_
+++ echo t0ni_
+ do_transfer '/gpfs/projects/bsc32/repository/exp/t0ni/runtime/runtime_t0ni_19931101_fc0*' /gpfs/archive/bsc32/esarchive/exp/ecearth/t0ni/original_files//runtime
+ cd /gpfs/scratch/bsc32/bsc32627/t0ni/LOG_t0ni
+ source_pattern='/gpfs/projects/bsc32/repository/exp/t0ni/runtime/runtime_t0ni_19931101_fc0*'
+ destination_path=/gpfs/archive/bsc32/esarchive/exp/ecearth/t0ni/original_files//runtime
++ dirname '/gpfs/projects/bsc32/repository/exp/t0ni/runtime/runtime_t0ni_19931101_fc0*'
+ source_path=/gpfs/projects/bsc32/repository/exp/t0ni/runtime
+ local 'remote_host= '
+ get_list_of_files_to_copy
+ [[ '' == \T\R\U\E ]]
++ ls '/gpfs/projects/bsc32/repository/exp/t0ni/runtime/runtime_t0ni_19931101_fc0*'
+ list_of_files_to_copy=
+ returncode=2
+ [[ ! -z '' ]]
+ '[' 0 -ne 0 ']'
+ for type in '$output_types'
+ case $type in
++ eval echo t0ni_
+++ echo t0ni_
+ do_transfer '/gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0/outputs/osm/icmcl/*' /gpfs/archive/bsc32/esarchive/exp/ecearth/t0ni/original_files//19931101/fc0/outputs/icmcl
+ cd /gpfs/scratch/bsc32/bsc32627/t0ni/LOG_t0ni
+ source_pattern='/gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0/outputs/osm/icmcl/*'
+ destination_path=/gpfs/archive/bsc32/esarchive/exp/ecearth/t0ni/original_files//19931101/fc0/outputs/icmcl
++ dirname '/gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0/outputs/osm/icmcl/*'
+ source_path=/gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0/outputs/osm/icmcl
+ local 'remote_host= '
+ get_list_of_files_to_copy
+ [[ '' == \T\R\U\E ]]
++ ls '/gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0/outputs/osm/icmcl/*'
+ list_of_files_to_copy=
+ returncode=2
+ [[ ! -z '' ]]
+ '[' 0 -ne 0 ']'
+ for type in '$output_types'
+ case $type in
++ eval echo t0ni_
+++ echo t0ni_
+ do_transfer '/gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0/outputs/osm/land_param/*' /gpfs/archive/bsc32/esarchive/exp/ecearth/t0ni/original_files//19931101/fc0/outputs/land_param
+ cd /gpfs/scratch/bsc32/bsc32627/t0ni/LOG_t0ni
+ source_pattern='/gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0/outputs/osm/land_param/*'
+ destination_path=/gpfs/archive/bsc32/esarchive/exp/ecearth/t0ni/original_files//19931101/fc0/outputs/land_param
++ dirname '/gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0/outputs/osm/land_param/*'
+ source_path=/gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0/outputs/osm/land_param
+ local 'remote_host= '
+ get_list_of_files_to_copy
+ [[ '' == \T\R\U\E ]]
++ ls '/gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0/outputs/osm/land_param/*'
+ list_of_files_to_copy=
+ returncode=2
+ [[ ! -z '' ]]
+ '[' 0 -ne 0 ']'
+ for type in '$output_types'
+ case $type in
++ eval echo t0ni_
+++ echo t0ni_
+ do_transfer '/gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0/outputs/osm/lpjg_forcing/*' /gpfs/archive/bsc32/esarchive/exp/ecearth/t0ni/original_files//19931101/fc0/outputs/lpjg_forcing
+ cd /gpfs/scratch/bsc32/bsc32627/t0ni/LOG_t0ni
+ source_pattern='/gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0/outputs/osm/lpjg_forcing/*'
+ destination_path=/gpfs/archive/bsc32/esarchive/exp/ecearth/t0ni/original_files//19931101/fc0/outputs/lpjg_forcing
++ dirname '/gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0/outputs/osm/lpjg_forcing/*'
+ source_path=/gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0/outputs/osm/lpjg_forcing
+ local 'remote_host= '
+ get_list_of_files_to_copy
+ [[ '' == \T\R\U\E ]]
++ ls '/gpfs/projects/bsc32/repository/exp/t0ni/19931101/fc0/outputs/osm/lpjg_forcing/*'
+ list_of_files_to_copy=
+ returncode=2
+ [[ ! -z '' ]]
+ '[' 0 -ne 0 ']'

###################
# Autosubmit tailer
###################
set -xuve
+ set -xuve
echo $(date +%s) >> ${job_name_ptrn}_STAT
++ date +%s
+ echo 1639863695
touch ${job_name_ptrn}_COMPLETED
+ touch /gpfs/scratch/bsc32/bsc32627/t0ni/LOG_t0ni/t0ni_19931101_fc0_TRANSFER_MEMBER_COMPLETED
exit 0
+ exit 0
