[INFO] JOBID=19279317

job_name_ptrn='/gpfs/scratch/bsc32/bsc32627/t0ni/LOG_t0ni/t0ni_19931101_fc0_1_SAVEIC'
+ job_name_ptrn=/gpfs/scratch/bsc32/bsc32627/t0ni/LOG_t0ni/t0ni_19931101_fc0_1_SAVEIC
echo $(date +%s) > ${job_name_ptrn}_STAT
date +%s
++ date +%s
+ echo 1639812679

###################
# Autosubmit job
###################

set -xuve
+ set -xuve
#
# Architecture
#
STAMP=$(date +%Y_%m_%d_%H_%M)
date +%Y_%m_%d_%H_%M
++ date +%Y_%m_%d_%H_%M
+ STAMP=2021_12_18_08_31
HPCARCH=marenostrum4
+ HPCARCH=marenostrum4
CURRENT_ARCH=marenostrum4
+ CURRENT_ARCH=marenostrum4
HPCPROJ=bsc32
+ HPCPROJ=bsc32
HPCUSER=bsc32627
+ HPCUSER=bsc32627
EXPID=t0ni
+ EXPID=t0ni
JOBNAME=t0ni_19931101_fc0_1_SAVEIC
+ JOBNAME=t0ni_19931101_fc0_1_SAVEIC
ROOTDIR=/gpfs/scratch/bsc32/bsc32627/t0ni
+ ROOTDIR=/gpfs/scratch/bsc32/bsc32627/t0ni
LOGDIR=$ROOTDIR/LOG_$EXPID
+ LOGDIR=/gpfs/scratch/bsc32/bsc32627/t0ni/LOG_t0ni
PROJDEST=auto-ecearth3
+ PROJDEST=auto-ecearth3
PROJDIR=$ROOTDIR/$PROJDEST
+ PROJDIR=/gpfs/scratch/bsc32/bsc32627/t0ni/auto-ecearth3
CURRENT_ARCH=marenostrum4
+ CURRENT_ARCH=marenostrum4
SCRATCH_DIR=/gpfs/scratch
+ SCRATCH_DIR=/gpfs/scratch
NUMTHREADS=48
+ NUMTHREADS=48

#
# General Paths and Conf.
#
MODEL=ecearth
+ MODEL=ecearth
VERSION=trunk
+ VERSION=trunk
CHUNK=1
+ CHUNK=1
# set the mask for padding the chunk number to three digits
chunk_zero_padded=$(printf "%03d\n" ${CHUNK})
printf "%03d\n" ${CHUNK}
++ printf '%03d\n' 1
+ chunk_zero_padded=001
Chunk_start_date=19931101
+ Chunk_start_date=19931101
Chunk_end_date=19931130
+ Chunk_end_date=19931130
START_date=19931101
+ START_date=19931101
START_date_1=19931031
+ START_date_1=19931031
MEMBER=fc0
+ MEMBER=fc0

# always use intermediate storage so we save to IC repos when SAVE_IC_REPOS=TRUE
# in any case, .tar files are kept in runtime/save_ic/ic, CLEAN will move it to intermediate_storage
USE_INTERMEDIATE_STORAGE=TRUE
+ USE_INTERMEDIATE_STORAGE=TRUE
IS_TRANSFER=FALSE
+ IS_TRANSFER=FALSE

export HPCPROJ
+ export HPCPROJ

. ${PROJDIR}/platforms/${HPCARCH}/configure.sh
+ . /gpfs/scratch/bsc32/bsc32627/t0ni/auto-ecearth3/platforms/marenostrum4/configure.sh
#####################################################################################################################
# Globals:
#   PROJDIR
# Arguments:
#   None
# Returns:
#   None
# Purpose: main function to prepare execution environment for the underlying platform, that basically consist on
#   setting up the modules to be used, common environment variables and the common paths
#####################################################################################################################
function load_platform_environment() {
  # load utils/filesystem libraries for common env functions
  . ${PROJDIR}/platforms/marenostrum4/filesystem.sh
  . ${PROJDIR}/platforms/marenostrum4/utils.sh
  #load global environment variables
  setup_modules
  #configure all modules needed for run the job
  setup_environment_vars
  # define all paths needed
  setup_paths
}

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: Prepare the modules needed to execute ece2cmor component for Marenostrum4 platform
#####################################################################################################################
function load_modules_ece2cmor() {
  set +xuve
  module purge
  module load impi/2017.4 intel/2017.4 mkl/2017.4 python/2.7.16 CMOR/3.5.0 CDO/1.8.2 gsl/2.4 nco/4.6.7 git eccodes/2.8.0
  set -xuve
}

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: Prepare the modules needed to execute the OASIS restart generation script for Marenostrum4 platform
#####################################################################################################################
function load_modules_oas_rsts_gen() {
  set +xuve
  module load hdf5/1.8.19_serial
  module load netcdf/4.4.1.1_serial
  module load udunits/2.2.25
  module load gsl/2.4
  module load nco/4.6.7
  module load CDO
  set -xuve
}

#####################################################################################################################
# Globals:
#  PATH, PROJDIR
# Arguments:
#   None
# Returns:
#   None
# Purpose: define the global vars with the path to the grib modules needed to run the ece2cmor module
#####################################################################################################################
function define_paths_ece2cmor() {
  grib_api_path=/apps/ECCODES/2.8.0/INTEL/
  PATH=${grib_api_path}/bin:${PATH}
  export GRIB_BIN_PATH=${grib_api_path}/bin
  export GRIB_DEFINITION_PATH=${PROJDIR}/sources/sources/util/grib_table_126:${grib_api_path}/share/eccodes/definitions
  export GRIB_SAMPLES_PATH=${grib_api_path}/share/eccodes/ifs_samples/grib1
}
load_platform_environment
+ load_platform_environment
+ . /gpfs/scratch/bsc32/bsc32627/t0ni/auto-ecearth3/platforms/marenostrum4/filesystem.sh
#####################################################################################################################
# Functions for handling filesystem operations for marenostrum4 platform
# Author: J.R.Berlin
#####################################################################################################################

. ${PROJDIR}/platforms/common/common.filesystem.sh
++ . /gpfs/scratch/bsc32/bsc32627/t0ni/auto-ecearth3/platforms/common/common.filesystem.sh
#####################################################################################################################
# Common Functions for handling file system operations on the different host types
# of the supported machines
# Author/Maintainer: J.R.Berlin
#####################################################################################################################

#####################################################################################################################
# Executes mkdir + chmod to handle proper rights in the target folder in esarchive partition
# Globals:
#   None
# Arguments:
#   target_dir
# Returns:
#   None
#####################################################################################################################
function mkdir_esarchive() {
  target_dir=$1
  mkdir -p ${target_dir}
  chmod_esarchive ${target_dir}
}

#####################################################################################################################
# Executes chmod with g+s so users of a given group doesnt have access problems to certain outputs and files
# Globals:
#   None
# Arguments:
#   target_dir
# Returns:
#   None
#####################################################################################################################
function chmod_esarchive() {
  target_dir=$1
  chmod =0770,g+s ${target_dir}
}


#####################################################################################################################
# Globals:
# RUN_dir,  SCRATCH_TMP_DIR, MEMBER, out_member
# Arguments:
#   None
# Returns:
#   None
# Purpose: creates the folder passed as parameter and assign proper group rights
#
#####################################################################################################################
function mkdir_intermediate_storage() {
  #default group
  target_group=bsc32
  target_dir=$1
  if [[ "$#" -eq 2 ]]; then
    target_group=$2
  fi

  if [[ ! -d "${target_dir}" ]]; then
    sg ${target_group} "mkdir -p ${target_dir}"
    chmod g+w ${target_dir}
  fi
}

#####################################################################################################################
# Globals:
#   None,
# Arguments:
#   $1: the folder to be removed
# Returns:
#   None
# Purpose: removes the intermediate storage folder passed as parameter
#
#####################################################################################################################
function delete_intermediate_storage() {
  rm -rf $1 2>/dev/null
}

#####################################################################################################################
# Globals:
#   None,
# Arguments:
#   $1: the source folder/file to be copied
#   $2: the target directory
# Returns:
#   None
# Purpose: copies the intermediate storage folder passed as parameter
#
#####################################################################################################################
function copy_intermediate_storage() {
  cp -f $1 $2
}

#####################################################################################################################
# Globals:
#   None,
# Arguments:
#   $1: the ecm folder whose content we want to enumerate
# Returns:
#   None
# Purpose: enumerate the content of the intermediate storage folder passed as parameter
#
#####################################################################################################################
function list_intermediate_storage() {
  ls $1
}

#####################################################################################################################
# Globals:
#   None,
# Arguments:
#   $1: the folder whose permissions we want to change
# Returns:
#   None
# Purpose: change the permissions of the intermediate storage folder passed as parameter
#
#####################################################################################################################
function chmod_intermediate_storage() {
  chmod 744 $1
}

#####################################################################################################################
# Globals:
#   None,
# Arguments:
#   $1: the folder we want to test
# Returns:
#   None
# Purpose: test the intermediate storage folder passed as parameter
#
#####################################################################################################################
function test_intermediate_storage() {
  test $1
}

# check if directory is empty and doesnt contain any file
#####################################################################################################################
# Globals:
# RUN_dir,  SCRATCH_TMP_DIR, MEMBER, out_member
# Arguments:
#   None
# Returns:
#   None
# Purpose: Main function for initialization - ifs3
#
#####################################################################################################################
function is_empty_directory() {
  local target_path=$1
  local result=false
  if [[ -z "$(ls -A ${target_path})" ]]; then
    result=true
  fi
  return result
}

#####################################################################################################################
# Globals:
# RUN_dir,
# Arguments:
#   source= source directory where are the files to be synchronized
#   destination=path to the directory that will be synchronized with the source dir
#   flags=options for the rsync
# Returns:
#   None
# Purpose: synchronizes two directories from different file systems
#          depending of the context, dt commands will be used by using blocking_rsync,
#          and rsync otherwise, <how_many_retry_copy_file> attempts will allowed to try to transfer the files in case
#          of failure
#####################################################################################################################
function rsync_with_retry() {

  source=$2
  destination=$3
  flags=$1
  group=$4
  retry=0
  how_many_retry_copy_file=5
  use_dt_rsync=FALSE

  exist_dtrsync=$(echo $(command -v dtrsync))
  if [[ ! -z "${exist_dtrsync-}" ]]; then
    use_dt_rsync=TRUE
  fi

  while [[ $retry -le $how_many_retry_copy_file ]]; do

    t1=$(date +%s)
    if [[ "${use_dt_rsync-}" == "FALSE" ]]; then
      USE_DT_COMMANDS=FALSE
      echo "Calling blocking_rsync with DT_COMMANDS=FALSE since they are not available on this machine"
    fi
    blocking_rsync "${flags}" ${source} ${destination} ${group}

    if [[ $? ]]; then
      t2=$(date +%s)
      tr=$(date -d "0 -$t1 sec + $t2 sec" +%T)
      echo "done."
      break
    else
      ((retry++))
    fi
  done
  if [[ $retry -eq how_many_retry_copy_file ]]; then
    echo "Transfer error in rsync"
    exit 1
  fi
}

#####################################################################################################################
# Globals:
# PROJDIR, target_destination, REMOTE_PATH
# Arguments:
#   None
# Returns:
#   None
# Purpose: synchronizes the auto-ecearth source folder with changes done in autosubmit auto-ecearth folder of the
# experiment
#
#####################################################################################################################
function transfer_model() {
  local folders_that_will_not_be_transferred=""
  . ${PROJDIR}/plugins/utils.sh
  source=${PROJDIR}
  destination=${target_destination}
  exclusions="--exclude={'templates','utils','test_cases','*.o','*.mod','*.exe','/BLD','ifs-36r4/bin','ifs-36r4/lib/*.a','COMPILE/arch*','COMPILE/cpp*','lpjg/build'}"
  options="${exclusions} --no-group --delete --force --recursive --links --perms --times --specials --verbose"
  if [[ "$#" -eq 1 ]]; then
    folders_that_will_not_be_transferred="$1"
    for folder in ${folders_that_will_not_be_transferred-}; do
      options+=" --exclude='${source}/${folder}' "
    done
  fi
  group=$(id -gn)
  rsync_with_retry "${options}" ${source} "${REMOTE_PATH}${destination}" ${group}
}

#####################################################################################################################
# Globals:
# RUN_dir,
# Arguments:
#   filenames: the file or list of files to be moved to the target location,
#   target_dir: the target directory where the files will be moved
#   is_transfer: defines if the target filesystem is the intermediate storage or esarchive ( permanent storage)
# Returns:
#   None
# Purpose: move files across different file systems, used mainly in CLEAN and TRANSFER jobs, this is the base function,
# each platform can have its own custom implementation if needed ( for instance ecmwf-sc40 has its own version of this
# function, the moved files are removed from the source location
#
#####################################################################################################################
function move_files() {
  local filenames=$1
  local target_dir=$2
  local is_transfer=$3

  if [[ "${is_transfer-}" == "TRUE" ]]; then
    # we are transferring from Scratch to Esarchive directly
    blocking_rsync '--no-perms --links --times --recursive --remove-source-files --no-group  --omit-dir-times --chmod=g+rwxs,o-rwx' "${filenames}" ${target_dir} Earth
  else
    files_to_send=$(echo "${filenames}")
    rsync --perms --links --times --recursive --remove-source-files --no-group  --omit-dir-times --chmod=g+rwxs,o-rwx ${files_to_send} ${target_dir}
  fi
  # check if there were errors
  if [[ $? != 0 ]]; then
    echo "Error occurred on move_files....exiting"
    exit 1
  fi
}

#####################################################################################################################
# Globals:
# RUN_dir, ini_data_dir
# Arguments:
#   None
# Returns:
#   None
# Purpose: copy the initialization files to inidata directory in the scratch directory of the experiment, the command
#   creates links to the files instead of copying these physically for optimizing storage
#
#####################################################################################################################
function copy_inidata() {
  cp --symbolic-link --recursive --force --update ${ini_data_dir} ${RUN_dir}
}

#####################################################################################################################
# Globals:
#
# Arguments:
#   src: source folder for the needed restarts
#   dst: target location where these will be copied
# Returns:
#   None
# Purpose: prepare restarts to be linked in to the inidata folder
#####################################################################################################################
function copy_default_restarts() {
  local src=$1
  local dst=$2
  cp --symbolic-link --recursive --force --update ${src} ${dst}
}

#####################################################################################################################
# Globals:
# RUN_dir, SCRATCH_TMP_DIR, MEMBER, ROOTDIR
# Arguments:
#   None
# Returns:
#   None
# Purpose: clean up the temporary directories used and set run dir, used in almost all platforms
#
#####################################################################################################################
function setup_paths_transfer() {
  rm -rf ${SCRATCH_TMP_DIR}
  mkdir -p ${SCRATCH_TMP_DIR}
  RUN_dir=${ROOTDIR}/${START_date}/${MEMBER}/runtime
}

#####################################################################################################################
# Globals:
# SCRATCH_DIR, HPCPROJ, HPCUSER, EXPID,PROJDEST
# Arguments:
#   None
# Returns:
#   None
# Purpose:  check the existence of the auto-ecearth folder in the experiment folder in the cluster
#
#####################################################################################################################
function check_model_existence() {
  SCRATCH_MODEL_PATH=${SCRATCH_DIR}/${HPCPROJ}/${HPCUSER}/${EXPID}/${PROJDEST}
  MODEL_EXISTS="FALSE"
  if [[ -d ${SCRATCH_MODEL_PATH} ]]; then
    MODEL_EXISTS="TRUE"
    export MODEL_EXISTS
  fi
}

#####################################################################################################################
# Globals:
# PROJDIR, SCRATCH_DIR, HPCPROJ, HPCUSER, EXPID, HPCARCH
# Arguments:
#   None
# Returns:
#   None
# Purpose: prepare/load needed libraries in order to execute the synchronize process
#
#####################################################################################################################
function setup_synchronize() {
  export target_destination=$SCRATCH_DIR/$HPCPROJ/$HPCUSER/$EXPID/
  export REMOTE_PATH=' '
  # import custom architecture
  . ${PROJDIR}/platforms/${HPCARCH}/filesystem.sh
}

#####################################################################################################################
# Globals:
# RUN_dir, USE_REMOTE_HOST, HPCUSER,  HPCHOST
# Arguments:
#    source_cmor_dir= source location for the cmorized files , usually the folder that corresponds to the chunk folder
#    target_cmor_dir= target destination where the files will be moved
#    is_transfer= indicates if the temporary storage will be used or not
# Returns:
#   None
# Purpose: upper level function to transfer the cmorized outputs either in CLEAN ( when USE_INTERMEDIATE_STORAGE=FALSE )
#    or the standard TRANSFER, this is the base function and other platforms may have different implementations such as
#    ecmwf-xc40
#####################################################################################################################
function move_cmor_files() {
  local source_cmor_dir=$1
  local target_cmor_dir=$2
  local is_transfer=$3
  local remote_host=' '

  # used in bscearth000 machine, maybe wise to create a function in platforms for this particular case
  if [[ "${USE_REMOTE_HOST-}" == "TRUE" ]]; then
    cmor_folders=$(ssh ${HPCUSER}@${HPCHOST} "ls -d ${source_cmor_dir}/cmor_*_${CHUNK}/" 2>/dev/null) && returncode=$? || returncode=$?
    if [[ ! -z "${cmor_folders-}" && ${returncode} -eq 0 ]]; then
      for cmor_remote_dir in ${cmor_folders}; do
        list_of_cmor_files=$(ssh ${HPCUSER}@${HPCHOST} "ls ${cmor_remote_dir}" 2>/dev/null) && returncode=$? || returncode=$?
        if [[ ! -z "${list_of_cmor_files-}" && ${returncode} -eq 0 ]]; then
          echo "moving cmor files ....."
          move_files ${HPCUSER}@${HPCHOST}:${cmor_remote_dir} ${target_cmor_dir} ${is_transfer}
          # remove cmor folder after moving the files
          # if there where some files left behind, this method will throw an exception making the job to crash
          ssh ${HPCUSER}@${HPCHOST} "find ${cmor_remote_dir} -type d -delete"
        fi
      done
    fi
  else
    if [[ -d ${source_cmor_dir} ]]; then
      echo "moving cmor files ....."
      cmor_folders=$(ls -d ${source_cmor_dir}/cmor_*_${CHUNK}/ 2>/dev/null) && returncode=$? || returncode=$?
      if [[ ! -z "${cmor_folders-}" && ${returncode} -eq 0 ]]; then
        base_storage_cmor_target_dir=${target_cmor_dir}
        for cmor_remote_dir in ${cmor_folders}; do
          #defaults when we use intermediate storage structure must be the same in the intermediate storage
          if [[ "${is_transfer-}" != "TRUE" ]]; then
            target_cmor_dir=${base_storage_cmor_target_dir}/$(basename ${cmor_remote_dir})
          fi
          move_files ${cmor_remote_dir} ${target_cmor_dir} ${is_transfer}
          # delete the folder, if there where some files left behind, this method will throw an exception making the job to crash
          find ${cmor_remote_dir} -type d -delete
        done
      fi
    fi
  fi
}

#####################################################################################################################
# Globals:
# RUN_dir, ROOTDIR, EXPID
# Arguments:
#   source_pattern: is the destination path for the output
#   destination_path: is the name of the final folder of the destination path /
#   list_of_files_to_copy: list of files to copy
# Returns:
#   None
# Purpose: Local transfer of model output, the function first retrieve the list of files to be transferred and then
#     if such file is not empty proceed to transfer them
#####################################################################################################################
function do_transfer() {

  #  to leave the dt command output in log folder
  cd ${ROOTDIR}/LOG_${EXPID}

  source_pattern=$1
  destination_path=$2
  source_path=$(dirname ${source_pattern[0]})
  local remote_host=' '

  get_list_of_files_to_copy && returncode=$? || returncode=$?
  #apply mkdir and let the rsync apply the chmod
  if [[ ! -z "${list_of_files_to_copy-}" ]]; then
    mkdir -p ${destination_path}
    for source_file_name in $list_of_files_to_copy; do
      echo "transferring file ... "${source_file_name} ${destination_path}
      blocking_rsync '--no-perms --links --times --recursive --remove-source-files --no-group  --omit-dir-times --chmod=g+rwxs,o-rwx' ${remote_host}${source_file_name} ${destination_path} Earth
    done
  fi
}

#####################################################################################################################
# Globals:
# USE_REMOTE_HOST, HPCUSER, HPCHOST, source_pattern
# Arguments:
#   None
# Returns:
#   None
# Purpose: get the list of files to be copied to the cluster, used by transfer/clean templates
#
#####################################################################################################################
function get_list_of_files_to_copy() {
  if [[ "${USE_REMOTE_HOST-}" == "TRUE" ]]; then
    list_of_files_to_copy=$(ssh ${HPCUSER}@${HPCHOST} "ls ${source_pattern}" 2>/dev/null)
    remote_host=${HPCUSER}@${HPCHOST}:
  else
    list_of_files_to_copy=$(ls ${source_pattern} 2>/dev/null)
  fi
}

#####################################################################################################################
# Globals:
# RUN_dir,
# Arguments:
#   None
# Returns:
#   None
# Purpose: performs the synchronization by transferring the model files to the cluster for preparation before running
# the experiment, is the common interface for the synchronization template, implementation may be different depending
# on the underlying architecture where the sync needs to be executed
#
#####################################################################################################################
function do_synchronize() {
  folders_that_will_not_be_transferred=""
  if [[ "$#" -eq 1 ]] ; then
    folders_that_will_not_be_transferred="$1"
  fi
  echo "Calling function transfer_model"
  transfer_model ${folders_that_will_not_be_transferred}
  echo "common.synchronize Done"
}

#####################################################################################################################
# Globals:
# RUN_dir, INTERMEDIATE_EXP_DIR, MODEL, EXPID
# Arguments:
#   None
# Returns:
#   None
# Purpose: Prepare source and target paths for CLEAN/TRANSFER
#####################################################################################################################
function setup_paths_localtrans() {
  src=${INTERMEDIATE_EXP_DIR}
  local_INTERMEDIATE_EXP_DIR=/gpfs/archive/bsc32/esarchive/exp/${MODEL}
  destination_folder=${local_INTERMEDIATE_EXP_DIR}/${EXPID}/original_files/
}

# copy to scratch the platforms files
#####################################################################################################################
# Globals:
# ROOTDIR, PROJDEST, PROJDIR
# Arguments:
#   None
# Returns:
#   None
# Purpose: send the initial set of libraries used by the synchronize job in order to be able to run
#
#####################################################################################################################
function copy_platform_environment_to_host() {
  mkdir -p ${ROOTDIR}/${PROJDEST}/platforms
  rsync --recursive --links --perms --times --owner --devices --specials --verbose --omit-dir-times ${PROJDIR}/platforms/ ${ROOTDIR}/${PROJDEST}/platforms
  rsync --recursive --links --perms --times --owner --devices --specials --verbose --omit-dir-times ${PROJDIR}/plugins/ ${ROOTDIR}/${PROJDEST}/plugins
}

#####################################################################################################################
# Globals:
# RUN_dir,  LOGDIR, CHUNK, START_date, MEMBER, STAMP, PATHOUT
# Arguments:
#   None
# Returns:
#   None
# Purpose: copy logs to the proper storage location before wiping out the experiment folder ( used by CLEAN_EXPERIMENT )
#####################################################################################################################
function save_logs() {
  echo "We are now storing log files"
  cd ${LOGDIR}
  logs_tarfile="logfiles_${EXPID}_${STAMP}.tar"
  IS_TRANSFER=TRUE
  prepare_logs && returncode=$? || returncode=$?
  if [[ ${returncode} -eq 0 ]] || [[ ${returncode} -eq 123 ]]; then
    [[ ! -f ../${logs_tarfile}.gz ]] && gzip -9 ../${logs_tarfile}
    local pathout_logfiles="${PATHOUT}/logfiles/"
    move_files ../${logs_tarfile}.gz ${pathout_logfiles} ${IS_TRANSFER}
    rm -f ../${logs_tarfile}.gz
  fi
}

#####################################################################################################################
# Globals:
# INTERMEDIATE_EXP_DIR, EXPID, ROOTDIR, RM ( alias for deleting files )
# Arguments:
#   None
# Returns:
#   None
# Purpose: wipe out the experiment folders (used intermediate storage, scratch)
#####################################################################################################################
function remove_experiment_folders() {
  #remove the folder in intermediate storage (if it was used)
  delete_intermediate_storage ${INTERMEDIATE_EXP_DIR}/${EXPID}
  #delete scratch folder
  rm -rf ${ROOTDIR}/*
  # needed for indicating that the job finished successfully by autosubmit
  mkdir -p ${ROOTDIR}/LOG_${EXPID}/
}

#####################################################################################################################
# any override from the common libraries defined in filesystem.sh or new function goes here
#####################################################################################################################

#####################################################################################################################
# Globals:
#   EXPID, MODEL, INTERMEDIATE_EXP_DIR, EXPID, USE_INTERMEDIATE_STORAGE
# Arguments:
#   None
# Returns:
#   None
# Purpose: template function for Marenostrum4 for setting up common paths ( used as setup_paths_transfer_${HPCARCH} )
#
#####################################################################################################################
function setup_paths_transfer_marenostrum4() {
  #base function
  setup_paths_transfer
  #custom settings
  if [[ "${USE_INTERMEDIATE_STORAGE-}" == "TRUE" ]]; then
    PATHOUT="${INTERMEDIATE_EXP_DIR}/${EXPID}/"
  else
    local_INTERMEDIATE_EXP_DIR=/gpfs/archive/bsc32/esarchive/exp/${MODEL}
    PATHOUT="${local_INTERMEDIATE_EXP_DIR}/${EXPID}/original_files/"
  fi
}
+ . /gpfs/scratch/bsc32/bsc32627/t0ni/auto-ecearth3/platforms/marenostrum4/utils.sh
#####################################################################################################################
# Functions for handling filesystem operations for marenostrum4 platform
# Author: J.R.Berlin
#####################################################################################################################

#. ${PROJDIR}/platforms/common/common.utils.sh

#####################################################################################################################
# any override from the common libraries defined in filesystem.sh or new function goes here
#####################################################################################################################

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: load the common modules required by standard jobs in Marenostrum4
#####################################################################################################################
function setup_modules() {
  set +xuve
  module purge
  module load intel/2018.3
  module load mkl/2018.3
  module load impi/2018.3
  module load netcdf/4.2
  module load hdf5/1.8.19
  module load perl/5.26
  module load fftw/3.3.6
  module load python/2.7.13
  module load udunits/2.2.25
  module load gsl/2.4
  module load nco/4.2.3_netcdf-4.2
  module load CDO/1.7.2
  module list
  set -xuve
}

#####################################################################################################################
# Globals:
#   USER
# Arguments:
#   None
# Returns:
#   None
# Purpose: Setup common paths as global variables, required by the different jobs running un the underlying platform
#           used in nord3, transfer_node and bscearth000 as well
####################################################################################################################
function setup_common_paths() {
  # common globals
  MODELS_DIR=/gpfs/projects/bsc32/models
  export SCRATCH=/gpfs/scratch/$(id -gn)/${USER}
  GROUP=bsc32
  # check if the machine has the environment tmp var defined
  SCRATCH_TMP_DIR=${SCRATCH}/${EXPID}/${START_date}/${MEMBER}/tmp/${JOBNAME}
  #initial conditions dirr for the platform, in order to have this working properly we need
  #all IC/rst files in place synchronized from bsc32/projects
  INTERMEDIATE_EXP_DIR=/gpfs/projects/${GROUP}/repository/exp
  EXTRA_DIR=/gpfs/projects/${GROUP}/repository
  IC_DIR=/gpfs/projects/${GROUP}/repository/ic
}

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: set the global variables and functions to be used by the jobs running in the underlying architecture
#          used in nord3, transfer_node and bscearth000 as well
#####################################################################################################################
function setup_common_environment_vars() {
  # common globals
  MODELS_DIR=/gpfs/projects/bsc32/models
  GROUP=bsc32
}

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: set the common set of global variables to be used for the jobs running on the underlying archictecture
#####################################################################################################################
function setup_environment_vars() {
  # common globals
  setup_common_environment_vars
  export ECE3_POSTPROC_MACHINE=mn4
}

#####################################################################################################################
# Globals:
#   CURRENT_ARCH, HPCARCH
# Arguments:
#   None
# Returns:
#   None
# Purpose: set the global variables refering to common paths to be used by the jobs running in the
#         underlying architecture, used in nord3, transfer_node and bscearth000 as well
#####################################################################################################################
function setup_paths() {
  setup_common_paths
  if [[ "${CURRENT_ARCH}" == "${HPCARCH}" ]]; then
    setup_paths_libs
  fi
}

#####################################################################################################################
# Globals:
#   USER, PATH
# Arguments:
#   None
# Returns:
#   None
# Purpose: auxiliary function to load paths to the different modules that will be load when the job
#         executes in the cluster
#####################################################################################################################
function setup_paths_libs() {
  grib_api_path=/apps/GRIB/1.14.0/INTEL
  PATH=${grib_api_path}/bin:${PATH}

  export SCRATCH=/gpfs/scratch/$(id -gn)/${USER}
  export PATH=/gpfs/projects/bsc32/repository/apps/rebuild_nemo_marenostrum4:${PATH}
  export PYTHONPATH=${grib_api_path}/lib/python2.7/site-packages/grib_api:${PYTHONPATH-}
  export GRIB_BIN_PATH=${grib_api_path}/bin
  export GRIB_DEFINITION_PATH=${grib_api_path}/share/grib_api/definitions
  export GRIB_SAMPLES_PATH=${grib_api_path}/share/grib_api/ifs_samples/grib1
}

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: auxiliary function to load paths to the different modules that will be load when the job
#          executes in the cluster, used in transfer machines!
#####################################################################################################################
function setup_environment_vars_local() {
  setup_common_environment_vars
  export ECE3_POSTPROC_MACHINE=mn4
}

#used in transfer machines !

#####################################################################################################################
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
# Purpose: auxiliary function to set the intermediate storage root path, used in transfer machines, this function
# overrides the base function in common.filesystem.sh
#####################################################################################################################
function setup_paths_local() {
  echo "MN4 Paths"
  GROUP=bsc32
  export INTERMEDIATE_EXP_DIR=/gpfs/projects/${GROUP}/repository/exp
}
+ setup_modules
+ set +xuve
remove mkl/2017.4 (LD_LIBRARY_PATH) 
remove impi/2017.4 (PATH, MANPATH, LD_LIBRARY_PATH) 
Set INTEL compilers as MPI wrappers backend 
load mkl/2018.3 (LD_LIBRARY_PATH) 
load impi/2018.3 (PATH, MANPATH, LD_LIBRARY_PATH) 
load netcdf/4.2 (PATH, LD_LIBRARY_PATH, NETCDF, NETCDF_PREFIX, NETCDF_HOME,
NETCDF_INC, NETCDF_LIB, NETCDF_ROOT) 
load hdf5/1.8.19 (PATH, LD_LIBRARY_PATH, LIBRARY_PATH, C_INCLUDE_PATH,
CPLUS_INCLUDE_PATH) 
load perl/5.26 (PATH, LD_LIBRARY_PATH, MANPATH, PERL5LIB) 
load fftw/3.3.6 (FFTW_CFLAGS, FFTW_FFLAGS, FFTW_LDFLAGS, FFTW_INCL, FFTW_LIBS,
LIBRARY_PATH, LD_LIBRARY_PATH, C_INCLUDE_PATH) 
load PYTHON/2.7.13 (PATH, MANPATH, LD_LIBRARY_PATH, LIBRARY_PATH,
PKG_CONFIG_PATH, C_INCLUDE_PATH, CPLUS_INCLUDE_PATH, PYTHONHOME) 
load lzo/2.10 (LD_LIBRARY_PATH,PKG_CONFIG_PATH,CFLAGS,CXXFLAGS,LDFLAGS) 
load udunits/2.2.25 (PATH, LD_LIBRARY_PATH, LIBRARY_PATH, INFOPATH,
C_INCLUDE_PATH, CPLUS_INCLUDE_PATH, UDUNITS2_PATH, UDUNITS2_XML_PATH) 
load gsl/2.4 (PATH, LD_LIBRARY_PATH, LIBRARY_PATH, MANPATH, INFOPATH,
C_INCLUDE_PATH, CPLUS_INCLUDE_PATH, PKG_CONFIG_PATH, GSL_DIR) 
load nco/4.2.3_netcdf-4.2 (PATH, LD_LIBRARY_PATH, LIBRARY_PATH, MANPATH,
INFOPATH, C_INCLUDE_PATH, CPLUS_INCLUDE_PATH) 
load CDO/1.7.2 (PATH) 

Currently Loaded Modules:
  1) intel/2018.3   5) hdf5/1.8.19   9) python/2.7.13         13) CDO/1.7.2
  2) mkl/2018.3     6) perl/5.26    10) udunits/2.2.25
  3) impi/2018.3    7) fftw/3.3.6   11) gsl/2.4
  4) netcdf/4.2     8) lzo/2.10     12) nco/4.2.3_netcdf-4.2

 

+ setup_environment_vars
+ setup_common_environment_vars
+ MODELS_DIR=/gpfs/projects/bsc32/models
+ GROUP=bsc32
+ export ECE3_POSTPROC_MACHINE=mn4
+ ECE3_POSTPROC_MACHINE=mn4
+ setup_paths
+ setup_common_paths
+ MODELS_DIR=/gpfs/projects/bsc32/models
id -gn
++ id -gn
+ export SCRATCH=/gpfs/scratch/bsc32/bsc32627
+ SCRATCH=/gpfs/scratch/bsc32/bsc32627
+ GROUP=bsc32
+ SCRATCH_TMP_DIR=/gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/tmp/t0ni_19931101_fc0_1_SAVEIC
+ INTERMEDIATE_EXP_DIR=/gpfs/projects/bsc32/repository/exp
+ EXTRA_DIR=/gpfs/projects/bsc32/repository
+ IC_DIR=/gpfs/projects/bsc32/repository/ic
+ [[ marenostrum4 == \m\a\r\e\n\o\s\t\r\u\m\4 ]]
+ setup_paths_libs
+ grib_api_path=/apps/GRIB/1.14.0/INTEL
+ PATH=/apps/GRIB/1.14.0/INTEL/bin:/apps/CDO/1.7.2/INTEL/IMPI/bin:/apps/NCO/4.2.3_netcdf-4.2/INTEL/bin:/apps/GSL/2.4/INTEL/bin:/apps/UDUNITS/2.2.25/INTEL/bin:/apps/PYTHON/2.7.13/INTEL/bin:/apps/FFTW/3.3.6/INTEL/IMPI/bin:/apps/PERL/5.26/bin:/apps/HDF5/1.8.19/INTEL/IMPI/bin:/apps/NETCDF/4.2/INTEL/IMPI/bin:/apps/INTEL/2018.3.051/impi/2018.3.222/bin64:/apps/INTEL/2018.3.051/bin:/usr/local/bin:/usr/bin:/bin:/usr/games:/usr/lpp/mmfs/bin
id -gn
++ id -gn
+ export SCRATCH=/gpfs/scratch/bsc32/bsc32627
+ SCRATCH=/gpfs/scratch/bsc32/bsc32627
+ export PATH=/gpfs/projects/bsc32/repository/apps/rebuild_nemo_marenostrum4:/apps/GRIB/1.14.0/INTEL/bin:/apps/CDO/1.7.2/INTEL/IMPI/bin:/apps/NCO/4.2.3_netcdf-4.2/INTEL/bin:/apps/GSL/2.4/INTEL/bin:/apps/UDUNITS/2.2.25/INTEL/bin:/apps/PYTHON/2.7.13/INTEL/bin:/apps/FFTW/3.3.6/INTEL/IMPI/bin:/apps/PERL/5.26/bin:/apps/HDF5/1.8.19/INTEL/IMPI/bin:/apps/NETCDF/4.2/INTEL/IMPI/bin:/apps/INTEL/2018.3.051/impi/2018.3.222/bin64:/apps/INTEL/2018.3.051/bin:/usr/local/bin:/usr/bin:/bin:/usr/games:/usr/lpp/mmfs/bin
+ PATH=/gpfs/projects/bsc32/repository/apps/rebuild_nemo_marenostrum4:/apps/GRIB/1.14.0/INTEL/bin:/apps/CDO/1.7.2/INTEL/IMPI/bin:/apps/NCO/4.2.3_netcdf-4.2/INTEL/bin:/apps/GSL/2.4/INTEL/bin:/apps/UDUNITS/2.2.25/INTEL/bin:/apps/PYTHON/2.7.13/INTEL/bin:/apps/FFTW/3.3.6/INTEL/IMPI/bin:/apps/PERL/5.26/bin:/apps/HDF5/1.8.19/INTEL/IMPI/bin:/apps/NETCDF/4.2/INTEL/IMPI/bin:/apps/INTEL/2018.3.051/impi/2018.3.222/bin64:/apps/INTEL/2018.3.051/bin:/usr/local/bin:/usr/bin:/bin:/usr/games:/usr/lpp/mmfs/bin
+ export PYTHONPATH=/apps/GRIB/1.14.0/INTEL/lib/python2.7/site-packages/grib_api:/apps/PYTHON/2.7.13/INTEL/lib/python2.7/site-packages
+ PYTHONPATH=/apps/GRIB/1.14.0/INTEL/lib/python2.7/site-packages/grib_api:/apps/PYTHON/2.7.13/INTEL/lib/python2.7/site-packages
+ export GRIB_BIN_PATH=/apps/GRIB/1.14.0/INTEL/bin
+ GRIB_BIN_PATH=/apps/GRIB/1.14.0/INTEL/bin
+ export GRIB_DEFINITION_PATH=/apps/GRIB/1.14.0/INTEL/share/grib_api/definitions
+ GRIB_DEFINITION_PATH=/apps/GRIB/1.14.0/INTEL/share/grib_api/definitions
+ export GRIB_SAMPLES_PATH=/apps/GRIB/1.14.0/INTEL/share/grib_api/ifs_samples/grib1
+ GRIB_SAMPLES_PATH=/apps/GRIB/1.14.0/INTEL/share/grib_api/ifs_samples/grib1

MODEL_DIR=$MODELS_DIR/$MODEL/$VERSION
+ MODEL_DIR=/gpfs/projects/bsc32/models/ecearth/trunk
BSC_OUTCLASS=reduced
+ BSC_OUTCLASS=reduced
CMIP6_OUTCLASS=
+ CMIP6_OUTCLASS=
if [[ -n "${BSC_OUTCLASS-}" ]]; then
  outclass=/auto-ecearth3/outclass/${BSC_OUTCLASS-}
elif [[ -n "${CMIP6_OUTCLASS-}" ]]; then
  outclass=ctrl/output-control-files/cmip6/${CMIP6_OUTCLASS}
elif [[ -z "${BSC_OUTCLASS-}" ]] && [[ -z "${CMIP6_OUTCLASS-}" ]]; then
  outclass=ctrl
fi
+ [[ -n reduced ]]
+ outclass=/auto-ecearth3/outclass/reduced

RUN_dir=$ROOTDIR/$START_date/$MEMBER
+ RUN_dir=/gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0
SETUP_dir=$PROJDIR/setup/outclass/${outclass}
+ SETUP_dir=/gpfs/scratch/bsc32/bsc32627/t0ni/auto-ecearth3/setup/outclass//auto-ecearth3/outclass/reduced
INIPATH=$RUN_dir/inidata
+ INIPATH=/gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/inidata

PATHOUT_IC=${RUN_dir}/runtime/save_ic/ic
+ PATHOUT_IC=/gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/ic

TMP_DIR=${RUN_dir}/runtime/save_ic/tmp
+ TMP_DIR=/gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/tmp
mkdir -p $TMP_DIR
+ mkdir -p /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/tmp
cd $TMP_DIR
+ cd /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/tmp

#
# Model Paths and Conf.
#
ini_data_dir=$MODEL_DIR/inidata
+ ini_data_dir=/gpfs/projects/bsc32/models/ecearth/trunk/inidata

#
# get model config from AS variables
#
#ifs
TEMPLATE_NAME=ecearth3
+ TEMPLATE_NAME=ecearth3
ifs=""
+ ifs=
if [[ "$TEMPLATE_NAME" == ecearth3* ]] || [[ "$TEMPLATE_NAME" == ifs* ]]; then
  ifs="ifs"
fi
+ [[ ecearth3 == ecearth3* ]]
+ ifs=ifs
[[ "$TEMPLATE_NAME" == ifs3* ]] && amip="amip" || amip=""
+ [[ ecearth3 == ifs3* ]]
+ amip=
[ "FALSE" = TRUE ] && atmnudg="atmnudg" || atmnudg=""
+ '[' FALSE = TRUE ']'
+ atmnudg=
[ "FALSE" = TRUE ] && sppt="sppt" || sppt=""
+ '[' FALSE = TRUE ']'
+ sppt=

#nemo
nemo=""
+ nemo=
pisces=""
+ pisces=
lim3=""
+ lim3=
rnfmapper=""
+ rnfmapper=
xios=""
+ xios=
if [[ "$TEMPLATE_NAME" == ecearth3* ]] || [[ "$TEMPLATE_NAME" == nemo3* ]]; then
  [[ "a2s5" == *[!\ ]* ]] && start_from_restart=":start_from_restart" || start_from_restart=""
  [ "%OCE_NUDG" = TRUE ] && ocenudg=":ocenudg" || ocenudg=""
  [ "FALSE" = TRUE ] && surfresto=":surfresto" || surfresto=""
  [ "TRUE" = TRUE ] && elpin=":elpin" || elpin=""
  nemo="nemo"${start_from_restart}${ocenudg}${surfresto}${elpin}
  [ "FALSE" = TRUE ] && pisces="pisces" || pisces=""
  lim3="lim3"
  xios="xios:detached"
  rnfmapper="rnfmapper"
fi
+ [[ ecearth3 == ecearth3* ]]
+ [[ a2s5 == *[!\ ]* ]]
+ start_from_restart=:start_from_restart
+ '[' %OCE_NUDG = TRUE ']'
+ ocenudg=
+ '[' FALSE = TRUE ']'
+ surfresto=
+ '[' TRUE = TRUE ']'
+ elpin=:elpin
+ nemo=nemo:start_from_restart:elpin
+ '[' FALSE = TRUE ']'
+ pisces=
+ lim3=lim3
+ xios=xios:detached
+ rnfmapper=rnfmapper

#others
[[ "FALSE" == TRUE ]] && lpjg=lpjg:fdbck || lpjg=""
+ [[ FALSE == TRUE ]]
+ lpjg=
[[ "FALSE" == TRUE ]] && tm5=tm5:chem,o3fb,ch4fb,aerfb || tm5=""
+ [[ FALSE == TRUE ]]
+ tm5=
[ "ifs" == "" ] && ifs_veg_source="era20c" || ifs_veg_source="ifs"
+ '[' ifs == '' ']'
+ ifs_veg_source=ifs
if [[ "end_leg" == FALSE ]] || [[ "end_leg" == "" ]]; then save_ic=""; else save_ic="save_ic:end_leg"; fi
+ [[ end_leg == FALSE ]]
+ [[ end_leg == '' ]]
+ save_ic=save_ic:end_leg

config="${ifs} ${amip} ${atmnudg} ${sppt} ${nemo} ${pisces} ${lim3} ${rnfmapper} ${xios} oasis ${lpjg} ${tm5} ${save_ic}"
+ config='ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg'

#
# Dealing with initial condtions (plugin)
#
#. $PROJDIR/plugins/initialization.sh

#
# IFS
#
ifs_lastout=false
+ ifs_lastout=false
ifs_grid=T511L91
+ ifs_grid=T511L91
ifs_grid_wol=$(echo ${ifs_grid} | cut -d 'L' -f 1) # without level (wol)
echo ${ifs_grid} | cut -d 'L' -f 1
++ echo T511L91
++ cut -d L -f 1
+ ifs_grid_wol=T511
ATM_ini=t0ni
+ ATM_ini=t0ni
ATM_ini_member=fc0
+ ATM_ini_member=fc0

#
# NEMO
#
nem_grid=ORCA025L75
+ nem_grid=ORCA025L75
nem_grid_wol=$(echo ${nem_grid} | cut -d 'L' -f 1) # without level (wol)
echo ${nem_grid} | cut -d 'L' -f 1
++ cut -d L -f 1
++ echo ORCA025L75
+ nem_grid_wol=ORCA025
OCEAN_ini=t0ni
+ OCEAN_ini=t0ni
OCEAN_ini_member=fc0
+ OCEAN_ini_member=fc0

#
# ICE
#
ICE_ini=t0ni
+ ICE_ini=t0ni
ICE_ini_member=fc0
+ ICE_ini_member=fc0

#
# PISCES
#
PISCES=FALSE
+ PISCES=FALSE
PISCES_ini=t0ni
+ PISCES_ini=t0ni
PISCES_ini_member=fc0
+ PISCES_ini_member=fc0
PISCES_OFF=FALSE
+ PISCES_OFF=FALSE
PISCES_OFF_DYN=
+ PISCES_OFF_DYN=

#
# LPJG
#
LPJG_ini=t0ni
+ LPJG_ini=t0ni
LPJG_ini_member=fc0
+ LPJG_ini_member=fc0

#
# TM5
#
TM5_ini=t0ni
+ TM5_ini=t0ni
TM5_ini_member=fc0
+ TM5_ini_member=fc0
TM5_NLEVS=34
+ TM5_NLEVS=34

#
# COMMON
#

#
# Prepare stuff for experiment
#

function count_files() {
  local ifiles=$1
  local nifiles=$(ls $ifiles 2>/dev/null) || nifiles=""
  nifiles=($nifiles)
  nifiles=${#nifiles[@]}
  echo $nifiles
}

# Function to create IC files for IFS from special output based on a script from K. Wyser
# copied from EC-Earth libsave_ic.sh with some simplifications
function save_ic_ifs_out2init_as() {
  icdate=${save_ic_date1[$1]}
  #srcdir=$2
  filter_output=true
  grib_filter=${GRIB_BIN_PATH}${GRIB_BIN_PATH:+/}grib_filter

  # temporary directory
  tmpdir=tmp_save_ic
  [ -d $tmpdir ] && rm -rf ${tmpdir}
  mkdir -p $tmpdir

  # first IFS timestep of a month is saved in last output file
  if [[ "$(date -d "${icdate}" +%d)" == "01" ]]; then
    yyyymm=$(date -d "${icdate} - 6 hours" +%Y%m)
  else
    yyyymm=$(date -d "${icdate}" +%Y%m)
  fi

  # find the ICMSH/GG files containing the last timestep, which are in $srcdir/output/ifs/???
  #local ifile_sh=`find $srcdir -name ICMSH${exp_name}+$yyyymm -print -quit`
  #local ifile_gg=`find $srcdir -name ICMGG${exp_name}+$yyyymm -print -quit`
  pwd
  ls
  #ifile_sh=ICMSH${exp_name}+$yyyymm
  #ifile_gg=ICMGG${exp_name}+$yyyymm
  ifile_sh=ICMSH${exp_name}+$yyyymm-out+init
  ifile_gg=ICMGG${exp_name}+$yyyymm-out+init
  if [[ ! -f "$ifile_sh" ]]; then
    info "cannot find ICMSH file $ifile_sh to create initial conditions at $icdate!"
    return 0
  fi
  if [[ ! -f "$ifile_gg" ]]; then
    info "cannot find ICMGG file $ifile_gg to create initial conditions at $icdate!"
    return 0
  fi

  if $filter_output; then
    # define grib filters to separate data for IC from normal output
    # filter_time_init is to define which timestep contains the ICs, currently only one per month is supported
    filter_time_init="dataDate == ${icdate} && dataTime == 0"

    # SH file
    ofile_sh_init=${tmpdir}/sh_init.grb
    filter_sh=${tmpdir}/filter_sh
    write_sh_init="if ( $filter_time_init ) { write \"${ofile_sh_init}\"; }"
    cat >$filter_sh <<EOT
if ( typeOfLevel is "hybrid" ) { ${write_sh_init}; }
EOT

    #GG file
    ofile_gg_init=${tmpdir}/gg_init.grb
    filter_gg=${tmpdir}/filter_gg
    write_gg_init="if ( $filter_time_init ) { write \"${ofile_gg_init}\"; }"
    cat >$filter_gg <<EOT
if ( levelType is "ml" ) { ${write_gg_init}; }
else {
  if ( ! ( levelType is "pl" ) ) { ${write_gg_init}; }
}
EOT

    #run grib_filter on SH and GG files
    $grib_filter $filter_sh $ifile_sh
    $grib_filter $filter_gg $ifile_gg
    if [[ ! -f "$ofile_sh_init" ]]; then
      info "ICMSH file $ifile_sh does not contain data to create initial conditions at $icdate"
      return 0
    fi
    if [[ ! -f "$ofile_gg_init" ]]; then
      info "ICMGG file $ifile_gg does not contain data to create initial conditions at $icdate"
      return 0
    fi

    #rename files
    ifile_sh=${ofile_sh_init}
    ifile_gg=${ofile_gg_init}
  fi # $filter_output

  if [[ ! -f "$ifile_sh" ]]; then
    info "cannot find ICMSH file $ifile_sh to create initial conditions at $icdate!"
    return 0
  fi
  if [[ ! -f "$ifile_gg" ]]; then
    info "cannot find ICMGG file $ifile_gg to create initial conditions at $icdate!"
    return 0
  fi

  #tgtdir=save_ic/$icdate
  tgtdir=.
  ofile_sh=${tgtdir}/ICMSH${exp_name}INIT
  ofile_gg_init=${tgtdir}/ICMGG${exp_name}INIT
  ofile_gg_iniua=${tgtdir}/ICMGG${exp_name}INIUA

  # make sure the output folder is created and delete any existing files
  #[ -d $tgtdir ] && rm -rf ${tgtdir}/* || mkdir -p $tgtdir
  mkdir -p $tgtdir
  rm -f $ofile_sh $ofile_gg_init $ofile_gg_iniua

  cat >${tmpdir}/gf1 <<EOT
if ( typeOfLevel is "hybrid" ) { write "${tmpdir}/shinit.[shortName].[level]"; }
EOT
  $grib_filter ${tmpdir}/gf1 $ifile_sh

  cp -f ${tmpdir}/shinit.lnsp.1 $ofile_sh
  for lev in {1..91}; do
    for var in vo d t; do
      cat ${tmpdir}/shinit.$var.$lev >>$ofile_sh
    done
  done
  cat ${tmpdir}/shinit.z.1 >>$ofile_sh

  cat >${tmpdir}/gf2 <<EOT
write "${tmpdir}/gginit.[shortName]";
EOT
  $grib_filter ${tmpdir}/gf2 $ifile_gg

  for var in stl1 stl2 stl3 stl4 swvl1 swvl2 swvl3 swvl4 sd src skt ci tsn asn \
    rsn sst istl1 istl2 istl3 istl4 chnk lsm sr al aluvp alnip aluvd alnid \
    lai_lv lai_hv sdfor slt sdor isor anor slor lsrh cvh cvl tvh tvl; do
    cat ${tmpdir}/gginit.$var >>$ofile_gg_init
  done

  cat >${tmpdir}/gf3 <<EOT
write "${tmpdir}/gginiua.[shortName].[level]";
EOT
  $grib_filter ${tmpdir}/gf3 $ifile_gg

  #   ${tmpdir}/gginiua.o3 \ check this!
  for lev in {1..91}; do
    for var in q; do
      cat ${tmpdir}/gginiua.$var.$lev >>$ofile_gg_iniua
    done
  done

  for lev in {1..91}; do
    for var in crwc cswc clwc ciwc cc; do
      cat ${tmpdir}/gginiua.$var.$lev >>$ofile_gg_iniua
    done
  done

  # delete tmp folder
  #rm -rf $tmpdir

  echo "save_ic_ifs_out2init ended successfully, results are in $tgtdir"
}

# most of this code copied from runtime script, preserving indentation for easy comparison

# define variables used in the runscript
source $ROOTDIR/librunscript.sh
+ source /gpfs/scratch/bsc32/bsc32627/t0ni/librunscript.sh
# librunscript.sh is a library of shell script functions that can be used in
# EC-Earth run scripts.
#
# Usage: source ./librunscript.sh

# Function info writes information to standard out.
#
# Usage: info MESSAGE ...
#
function info()
{
    echo "*II* $@"
}

# Function error writes information to standard out and exits the script with
# error code 1.
#
# Usage: error MESSAGE ...
#
function error()
{
    echo "*EE* $@"
    exit 1
}

# Function has_config checks it's arguments for matches in the $config variable
# and returns true (0) or false (1) accordingly. Optionally, the first argument
# can be either "all" (all arguments must match) or "any" (at least one
# argument must match). If the first argument is neither "all" nor "any", the
# function behaves like "all" was given as the first argument.
#
# Usage: has_config [all|any] ARGS ...
#
# Syntax rules:
#
# The $config variable takes a list of names (typically software components),
# separated by white spaces:
#
# config="foo bar baz"  # Specifies three components: 'foo', 'bar', and 'baz'
#
# It is possible to add comma-separated lists of options to components. The
# list is separated from the component by a colon:
#
# config="foo bar:qux,fred baz:plugh"  # Adds the options 'qux' and 'fred' to
#                                      # component 'bar' as well as option
#                                      # 'plugh' to  component 'baz'
#
# When using the has_config function to check the $config variable, it is
# important to list every component-option pair separately. To check for both
# the 'qux' and 'fred' options of component 'bar' in the above example, use:
#
# has_config bar:qux bar:fred && echo "Got it!"
#
function has_config()
{
    # If called without arguments, return false
    (( $# )) || return 1

    # If $config unset or empty, return false
    [[ -z "${config:-}" ]] && return 1

    local __c
    local __m

    # If first argument is "any" then only one of the arguments needs to match
    # to return true. Return false otherwise
    if [ "$1" == "any" ]
    then
        shift
        for __c in "$@"
        do
            for __m in $config
            do
                [[ "$__m" =~ "${__c%:*}" ]] && [[ "$__m" =~ "${__c#*:}" ]] && return 0
            done
        done
        return 1
    fi

    # If first argument is "all", or neither "any" nor "all", all arguments
    # must match to return true. Return false otherwise.
    [[ "$1" == "all" ]] && shift

    local __f
    for __c in "$@"
    do
        __f=0
        for __m in $config
        do
            [[ "$__m" =~ "${__c%:*}" ]] && [[ "$__m" =~ "${__c#*:}" ]] && __f=1
        done
        (( __f )) || return 1
    done
    return 0
}

# Function leap days calculates the number of leap days (29th of Februrary) in
# a time intervall between two dates.
#
# Usage leap_days START_DATE END_DATE
function leap_days()
{
    local ld=0
    local frstYYYY=$(date -ud "$1" +%Y)
    local lastYYYY=$(date -ud "$2" +%Y)

    set +e

    # Check first year for leap day between start and end date
    $(date -ud "${frstYYYY}-02-29" > /dev/null 2>&1) \
    && (( $(date -ud "$1" +%s) < $(date -ud "${frstYYYY}-03-01" +%s) )) \
    && (( $(date -ud "$2" +%s) > $(date -ud "${frstYYYY}-02-28" +%s) )) \
    && (( ld++ ))

    # Check intermediate years for leap day
    for (( y=(( ${frstYYYY}+1 )); y<=(( ${lastYYYY}-1 )); y++ ))
    do
        $(date -ud "$y-02-29" > /dev/null 2>&1) && (( ld++ ))
    done

    # Check last year (if different from first year) for leap day between start
    # and end date
    (( $lastYYYY > $frstYYYY )) \
    && $(date -ud "${lastYYYY}-02-29" > /dev/null 2>&1) \
    && (( $(date -ud "$1" +%s) < $(date -ud "${lastYYYY}-03-01" +%s) )) \
    && (( $(date -ud "$2" +%s) > $(date -ud "${lastYYYY}-02-28" +%s) )) \
    && (( ld++ ))

    set -e

    echo "$ld"
}

# Helper function for absolute_date_noleap().
#
# Recursively 'fixes' a relative date into an absolute date referenced to
# the "noleap" calendar instead of Julian/Gregorian.
#
# Usage: __noleap_fixer base_date modifier
#

function __noleap_fixer ()
{
    local base_date="$1"
    local modifier="$2"
    # Evaluate the base and the modified date.
    local t0=$(date -u -d "${base_date}" +'%Y%m%d %H')
    local t1=$(date -u -d "${base_date} ${modifier}" +'%Y%m%d %H')
    local nleap
    local op
    # Compute the number of leap days between the base date and the modified
    # date.
    if [[ $(date -u -d "${t0}" +%Y%j) -lt $(date -u -d "${t1}" +%Y%j) ]]; then
        nleap=$(leap_days "${t0}" "${t1}")
    else
        nleap=$(leap_days "${t1}" "${t0}")
    fi
    if [[ ${nleap} -eq 0 ]]; then
        # Return the modified date if there are no leap days to account for.
        echo "${t1}"
    else
        # Obtain the modifier symbol (either + or -), this is used to determine
        # the direction to apply corrections in.
        if echo "${modifier}" | grep '+' 2>&1 > /dev/null; then
            op='+'
        else
            op='-'
        fi
        if [[ $(date -ud "${t1}" +%m%d) == 0229 ]]; then
            # If the modified day is the leap day we shift it by one day in the
            # appropriate direction and subtract 1 from the number of leap days
            # we need to account for.
            t1=$(date -ud "${t1} ${op} 1 day" +'%Y%m%d %H')
            nleap=$(( nleap - 1 ))
        fi
        if [[ ${nleap}  -eq 0 ]]; then
            # After correction there maybe no leap days left to account for, if
            # so we just return the corrected modified date.
            echo "${t1}"
        else
            # Otherwise we recurse, to apply our modification safely accounting
            # for leap days.
            __noleap_fixer "${t1}" "${op} ${nleap} days"
        fi
    fi
}

# Convert a relative date to an absolute date assuming the "noleap" calendar.
#
# A relative date is a base date with some modifier, such as
# "2012-01-01 + 2 months".
#
# Usage: absolute_date_noleap date_specifier.
function absolute_date_noleap ()
{
    local input_date="$1"
    # Split the input into a date part and a modifier part.
    local regex='([a-z|A-Z|0-9|-|+|,|:|\s]*)\s*([+|-]\s*[0-9]+\s*(second|minute|hour|day)s{0,1}).*'
    local base_date=$(echo "$1" | sed -r "s/${regex}/\1^\2/" | cut -d "^" -f 1)
    local modifier=$(echo "$1" | sed -r "s/${regex}/\1^\2/" | cut -d "^" -f 2)
    if [ -z "${modifier}" ] || [ "${modifier}" = "${input_date}" ] || \
            ! echo "${modifier}" | egrep -i -e 'second|minute|hour|day' 2>&1 >/dev/null; then
        # Either there was no modifier or it performs calendar independent
        # modifications, and thus we can safely use the normal `date` tool.
        date -uRd "${input_date}"
    else
        # Special care is needed to make sure the modifier is applied within
        # the noleap calendar.
        date -uRd "$(__noleap_fixer "${base_date}" "${modifier}")"
    fi
}

# Functions to compute computationa performance according to CPMIP metrics
#
# cpmip_sypd: Computes Simulated Years Per Day (SYPD)
#             Needs two arguments:
#             $1 - length of leg in seconds (use $leg_length_sec)
#             $2 - run time of leg in seconds (use $(( t2 - t1 )))
function cpmip_sypd ()
{
    bc <<< "scale=2 ; $1 / ( $2 * 365 )"
}

# cpmip_chpsy: Computes Core Hours Per Simulated Years (CHPSY)
#              Needs three arguments:
#              $1 - length of leg in seconds (use $leg_length_sec)
#              $2 - run time of leg in seconds (use $(( t2 - t1 )))
#              $3 - overall number of cores
function cpmip_chpsy ()
{
    bc <<< "scale=0 ; 365 * 24 * $2 * $3 / $1"
}
exp_name=t0ni
+ exp_name=t0ni
nem_time_step_sec=2700 # default, set below to real value if has_config nemo
+ nem_time_step_sec=2700
run_dir=$RUN_dir/runtime
+ run_dir=/gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime
leg_number=${CHUNK}
+ leg_number=1

# Time step settings
if has_config nemo; then
  case "${nem_grid}" in
  ORCA1L*)
    nem_time_step_sec=2700
    lim_time_step_sec=2700
    ;;
  ORCA025L*)
    nem_time_step_sec=900
    lim_time_step_sec=900
    ;;
  *)
    error "Can't set time steps for unknown horizontal grid: ${nem_grid}"
    ;;
  esac
fi
+ has_config nemo
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' nemo == any ']'
+ [[ nemo == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ nemo ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ nemo ]]
+ [[ nemo:start_from_restart:elpin =~ nemo ]]
+ __f=1
+ for __m in '$config'
+ [[ lim3 =~ nemo ]]
+ for __m in '$config'
+ [[ rnfmapper =~ nemo ]]
+ for __m in '$config'
+ [[ xios:detached =~ nemo ]]
+ for __m in '$config'
+ [[ oasis =~ nemo ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ nemo ]]
+ ((  __f  ))
+ return 0
+ case "${nem_grid}" in
+ nem_time_step_sec=900
+ lim_time_step_sec=900

# -----------------------------------------------------------------------------
# *** Extra initial conditions saved during the run
# -----------------------------------------------------------------------------
save_ic_lpjg_days=""
+ save_ic_lpjg_days=
if has_config save_ic; then
  source $ROOTDIR/libsave_ic.sh
  declare -a save_ic_date save_ic_date1 save_ic_sec save_ic_day save_ic_ppt_file save_ic_nemo_ts
  #else
  #     echo "doing nothing"
  #fi

  run_start_date="19931101"
  run_end_date="19931101 + 1 month"
  leg_start_date="19931101"
  leg_end_date="19931130 + 1 day"

  # Regularise the format of the start and end date of the simulation
  run_start_date=$(date -uR -d "${run_start_date}")
  run_end_date=$(date -uR -d "${run_end_date}")
  leg_start_date=$(date -uR -d "${leg_start_date}")
  leg_end_date=$(date -uR -d "${leg_end_date}")

  # Some time variables needed later
  leg_length_sec=$(($(date -d "${leg_end_date}" +%s) - $(date -d "${leg_start_date}" +%s)))
  leg_start_sec=$(($(date -d "${leg_start_date}" +%s) - $(date -d "${run_start_date}" +%s)))
  leg_end_sec=$(($(date -d "${leg_end_date}" +%s) - $(date -d "${run_start_date}" +%s)))
  leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d)
  leg_start_date_yyyy=$(date -u -d "${leg_start_date}" +%Y)
  leg_end_date_yyyy=$(date -u -d "${leg_end_date}" +%Y)
  leg_end_date_yyyymmdd=$(date -u -d "${leg_end_date}" +%Y%m%d)

  # Initial conditions saved during the run
  do_save_ic=false
  save_ic_custom=false
  has_config save_ic && save_ic_get_config
  # if you do not use an option with save_ic, you must define 'do_save_ic' and
  # 'save_ic_date_offset' here or in ../libsave_ic.sh/save_ic_get_config()
  # with AS runtime, no need to edit the script, set SAVE_IC_OFFSET (and optionally SAVE_IC_COND)
  if $save_ic_custom; then
    [[ "true" == "" ]] && save_ic_cond=true || save_ic_cond='true'
    if eval $save_ic_cond; then do_save_ic=true; else do_save_ic=false; fi
    save_ic_date_offset=()
  fi
  ${do_save_ic} && save_ic_define_vars

  #do_save_ic=end_leg ; [ -z "${do_save_ic}" ] && do_save_ic=false
  SAVE_IC_REPOS=FALSE
  [ -z "${SAVE_IC_REPOS}" ] && SAVE_IC_REPOS=FALSE

  if ${do_save_ic}; then
    # process initial conditions
    for ((i = 0; i < ${#save_ic_date[@]}; i++)); do
      echo =====================================
      outdir=${run_dir}/save_ic/${save_ic_date1[$i]}

      # continue if ICs have already been processed by previous job, and outdir is empty
      # outdir is removed at the end of the loop, so it can only not be empty if SIM was re-run in the meantime
      tarfile="IC_${EXPID}_${START_date}_${MEMBER}_${chunk_zero_padded}_${i}_${Chunk_start_date}-${Chunk_end_date}.tar"
      if [ -f ${PATHOUT_IC}/${tarfile} ] && [ ! -d ${outdir} ]; then
        echo "skipping save_ic for date "${save_ic_date1[$i]}" since it has apparently been done in a previous job"
        continue
      fi

      # make sure we have some output - this might not be a problem if running the lsm template (with only LPJG ICs)
      if [ ! -d $outdir ] ; then
        echo "SAVEIC problem - there are no ICs for date ${save_ic_date1[$i]}"
        [[ "$TEMPLATE_NAME" != "lsm" ]] && exit 1 || mkdir -p ${outdir}
      fi

      ifs_timestamp=${save_ic_date1[$i]}00
      nemo_timestamp=$(date -u -d "${save_ic_date[$i]} -1 day" +%Y%m%d)
      cd ${outdir}
      mkdir -p delete

      echo files for ${save_ic_date1[$i]} are in ${outdir}
      ls $outdir

      echo =====================================

      # IFS files
      if has_config ifs; then
        [ ! -d ifs ] && echo "SAVEIC problem - there are no ICs for IFS for date ${save_ic_date1[$i]}" && exit 1
        cd ifs
        ifiles="ICMGG${exp_name}INIT ICMGG${exp_name}INIUA ICMSH${exp_name}INIT"
        odir=${outdir}/ic/atmos/${ifs_grid}/${ATM_ini}
        ofile=${ATM_ini}_${ATM_ini_member}_${ifs_timestamp}.tar.gz
        # tar and move to temporary ic repository
        # TODO should we remove the full output file (e.g. ICMGGa0fh+195001)
        nifiles=$(count_files "$ifiles")
        # if nothing is found, try and rebuild the ICs with what is left from a previous run
        if [[ $nifiles -ne 3 ]] && ( ! ([[ -f ${odir}/${ofile} ]] && [[ -f ${outdir}/ifs_ok ]])); then
          if [[ $(count_files "ICMGG${exp_name}+??????-out+init ICMSH${exp_name}+??????-out+init") -ge 2 ]]; then
            save_ic_ifs_out2init_as $i
            nifiles=$(count_files "$ifiles")
          fi
        fi

        if [[ $nifiles -eq 3 ]]; then
          echo "IFS ifiles $ifiles found!"
          rm -rf $odir ${outdir}/ifs_ok ${outdir}/tmp_save_ic
          mkdir -p $odir
          tar -czvf ${odir}/${ofile} $ifiles
          touch ${outdir}/ifs_ok
          #echo copied `basename $ofile` to $odir
          mv ${ifiles} ICMGG${exp_name}+??????* ICMSH${exp_name}+??????* ../delete
        elif [[ -f ${odir}/${ofile} ]] && [[ -f ${outdir}/ifs_ok ]]; then
          echo "IFS ofile ${odir}/${ofile} found!"
        else
          echo "SAVEIC problem - IFS ifiles $ifiles not found!"
          exit 1
        fi
        cd ..
      fi

      echo =====================================

      # NEMO files
      if has_config nemo; then
        if [[ ${save_ic_nemo_ts[$i]} -ne -1 ]]; then
          [ ! -d nemo ] && echo "SAVEIC problem - there are no ICs for NEMO for date ${save_ic_date1[$i]}" && exit 1
          cd nemo
          ns=$(printf %08d $((save_ic_nemo_ts[$i])))
          #ftype_=( oce ice )
          #filebase_=( ${exp_name}_${ns}_restart_oce ${exp_name}_${ns}_restart_ice )
          #ofile_=( ${OCEAN_ini}_${OCEAN_ini_member}_${nemo_timestamp}_restart.nc.gz ${ICE_ini}_${ICE_ini_member}_${nemo_timestamp}_restart_ice.nc.gz )
          #odir_=( ${outdir}/ic/ocean/${nem_grid}/${OCEAN_ini} ${outdir}/ic/ice/${nem_grid_wol}_${ICE}/${ICE_ini})
          ftype_=(oce)
          filebase_=(${exp_name}_${ns}_restart_oce)
          ofile_=(${OCEAN_ini}_${OCEAN_ini_member}_${nemo_timestamp}_restart.nc.gz)
          odir_=(${outdir}/ic/ocean/${nem_grid}/${OCEAN_ini})
          if has_config lim3; then
            ftype_+=(ice)
            filebase_+=(${exp_name}_${ns}_restart_ice)
            ofile_+=(${ICE_ini}_${ICE_ini_member}_${nemo_timestamp}_restart_ice.nc.gz)
            odir_+=(${outdir}/ic/ice/${nem_grid_wol}_LIM3/${ICE_ini})
          fi
          if has_config pisces; then
            ftype_+=(trc)
            filebase_+=(${exp_name}_${ns}_restart_trc)
            ofile_+=(${PISCES_ini}_${PISCES_ini_member}_${nemo_timestamp}_restart_trc.nc.gz)
            odir_+=(${outdir}/ic/oceantrc/${nem_grid}/${PISCES_ini})
          fi
          echo filebase_oce: ${filebase_[0]} filebase_ice: ${filebase_[1]}
          for ((j = 0; j < ${#filebase_[@]}; j++)); do
            #ndomain=$3
            filebase=${filebase_[j]}
            ofile=${ofile_[j]}
            odir=${odir_[j]}

            ifiles=(${filebase}_*)
            ndomain=${#ifiles[@]}
            if [ $ndomain -gt 1 ]; then
              rm -f ${outdir}/nemo/${filebase}_ok
              # rebuild restarts
              echo found $ndomain files matching pattern $filebase
              rm -f ${filebase}.nc ${filebase}.nc.gz ${outdir}/nemo/${filebase}_ok
              # TODO rebuild_nemo in scratch dir
              rebuild_nemo -t${NUMTHREADS:-1} $filebase $ndomain
              rm -f nam_rebuild
              gzip ${filebase}.nc
              # move rebuilds to temporary ic repository
              rm -rf $odir
              mkdir -p $odir
              mv ${filebase}.nc.gz ${odir}/${ofile}
              touch ${outdir}/nemo/${filebase}_ok
              #echo copied $ifile to ${odir}/${ofile}
              mv ${filebase}*.nc ../delete
            elif [[ -f ${outdir}/nemo/${filebase}_ok ]] && [[ -f ${odir}/${ofile} ]]; then
              echo ofile ${odir}/${ofile} found!
            else
              echo "SAVEIC problem - did not find NEMO restart files matching pattern $filebase"
              echo "you can put the rebuilt restarts in $odir or the raw restarts in ${outdir}/nemo and re-run SAVEIC"
              exit 1
            fi
          done
          cd ..
        fi
      fi

      echo =====================================

      # oasis files - don't crash if not found since they are not produced in the middle of a CHUNK
      if has_config oasis && [ -d oasis ]; then
        cd oasis
        #ifiles=`ls $oas_rst_files 2>/dev/null || true`
        oas_rst_dir=${outdir}/oasis

        # Restart files for the coupling fields (note 8 character limit in OASIS)
        #   rstas.nc : atmosphere single-category fields
        #   rstam.nc : atmosphere multi-category fields
        #   rstos.nc : ocean single-category fields
        #   rstom.nc : ocean multi-category fields
        oas_rst_ifs_nemo="rstas.nc rstos.nc"

        if has_config ifs nemo; then
          ifiles=$(ls ${oas_rst_ifs_nemo} 2>/dev/null || true)
          odir=${outdir}/ic/coupler/${ifs_grid_wol}-${nem_grid_wol}/${ATM_ini}
          # gzip and move to temporary ic repository
          if [[ $ifiles != "" ]]; then
            echo oasis ifs nemo restart files $ifiles found!
            rm -f ${outdir}/oasis_ok
            rm -rf $odir
            mkdir -p $odir
            for ifile in $(ls $ifiles 2>/dev/null); do
              ofile=${ATM_ini}_${ATM_ini_member}_${ifs_timestamp}_$(basename ${ifile}).gz
              gzip -c $ifile >${odir}/${ofile}
              #echo moved $ifile to ${odir}/${ofile}
              mv ${ifile} ../delete
            done
            touch ${outdir}/oasis_ok
          elif [[ -f ${outdir}/oasis_ok ]]; then
            echo "oasis files already present"
          else
            echo "SAVEIC problem - no oasis ifs nemo restart files found in ${oas_rst_dir} !"
          fi
        fi

        oas_rst_ifs_lpjg="vegin.nc lpjgv.nc"

        if has_config ifs lpjg; then
          ifiles=$(ls ${oas_rst_ifs_lpjg} 2>/dev/null || true)
          odir=${outdir}/ic/coupler/${ifs_grid_wol}/${ATM_ini}
          # gzip and move to temporary ic repository
          if [[ $ifiles != "" ]]; then
            echo oasis restart files $ifiles found!
            rm -f ${outdir}/oasis_ok
            rm -rf $odir
            mkdir -p $odir
            for ifile in $(ls $ifiles 2>/dev/null); do
              ofile=${ATM_ini}_${ATM_ini_member}_${ifs_timestamp}_$(basename ${ifile}).gz
              gzip -c $ifile >${odir}/${ofile}
              #echo moved $ifile to ${odir}/${ofile}
              mv ${ifile} ../delete
            done
            touch ${outdir}/oasis_ok
          else
            echo "SAVEIC problem - no oasis ifs lpjg restart files found in ${oas_rst_dir} !"
          fi
        fi

        if has_config ifs tm5; then
          # get all files in oasis dir except the ones for IFS+NEMO and IFS+LPJG
          ifiles=$(ls 2>/dev/null | grep -v -e ${oas_rst_ifs_nemo// / -e } -e ${oas_rst_ifs_lpjg// / -e })
          odir=${outdir}/ic/coupler/${ifs_grid_wol}-TM5-${TM5_NLEVS}-levels/${ATM_ini}
          # gzip and move to temporary ic repository
          if [[ $ifiles != "" ]]; then
            echo "oasis ifs tm5 restart files $ifiles found!"
            rm -f ${outdir}/oasis_ok
            rm -rf $odir
            mkdir -p $odir
            for ifile in $(ls $ifiles 2>/dev/null); do
              ofile=${ATM_ini}_${ATM_ini_member}_${ifs_timestamp}_$(basename ${ifile}).gz
              gzip -c $ifile >${odir}/${ofile}
              #echo moved $ifile to ${odir}/${ofile}
              mv ${ifile} ../delete
            done
            touch ${outdir}/oasis_ok
          else
            echo "SAVEIC problem - no oasis ifs tm5 restart files found in ${oas_rst_dir} !"
          fi
        fi

        cd ..
      fi

      echo =====================================

      # LPJG
      if has_config lpjg; then
        # end-of-leg restarts are in restart/lpjg/???, intermediate restarts are in save_ic/ folder
        if [[ $leg_end_date_yyyymmdd == ${save_ic_date1[$i]} ]]; then
          full_lpjg_restart=false
          rstdir=${run_dir}/restart/lpjg/$( printf "%03d\n" ${CHUNK} )
        else
          full_lpjg_restart=true
          rstdir=${outdir}/lpjg
        fi
        ofile=${LPJG_ini}_${LPJG_ini_member}_${ifs_timestamp}_lpjg.tgz
        odir=${outdir}/ic/lpjg/${ifs_grid_wol}/${LPJG_ini}
        if [[ -f ${outdir}/lpjg_ok && -f ${odir}/${ofile} ]] ; then
          echo "lpjg files already present"
        elif [[ -d $rstdir ]] ; then
          rm -f ${outdir}/lpjg_ok
          mkdir -p $odir
          cd $rstdir
          #tar -czvf ${odir}/${ofile} *
          # use pigz (parallel gzip) instead of gzip since these files are quite big
          tar -cvf - * | pigz -p ${NUMTHREADS} > ${odir}/${ofile}
          cd -
          ${full_lpjg_restart} && rm -rf ${rstdir}
          touch ${outdir}/lpjg_ok
        else
          echo "SAVEIC problem - no lpjg restart files found in ${rstdir}"
          exit 1
        fi
      fi #LPJG

      echo =====================================

      # TM5 - only get ICs if this is the last timestep of the leg
      if has_config tm5 && [[ $leg_end_date_yyyymmdd == ${save_ic_date1[$i]} ]]; then
        [ ! -d tm5 ] && echo "SAVEIC problem - there are no ICs for TM5 for date ${save_ic_date1[$i]}" && exit 1
        cd tm5
        ifiles=$(ls TM5_restart_${leg_end_date_yyyymmdd}_0000_glb300x200.nc save_${leg_end_date_yyyymmdd}00_glb300x200.hdf 2>/dev/null || true)
        odir=${outdir}/ic/tm5/${TM5_ini}
        # gzip and move to temporary ic repository
        if [[ $ifiles != "" ]]; then
          echo tm5 restart files $ifiles found!
          rm -f ${outdir}/tm5_ok
          rm -rf $odir
          mkdir -p $odir
          for ifile in $(ls $ifiles 2>/dev/null); do
            ofile=${TM5_ini}_${TM5_ini_member}_${ifs_timestamp}_$(basename ${ifile}).gz
            gzip -c $ifile >${odir}/${ofile}
            #echo moved $ifile to ${odir}/${ofile}
            mv ${ifile} ../delete
          done
          touch ${outdir}/tm5_ok
        elif [[ -f ${outdir}/tm5_ok ]]; then
          echo "tm5 files already present"
        else
          echo "SAVEIC problem - no tm5 restart files found"
          exit 1
        fi
        cd -
      fi #TM5

      echo =====================================

      # exit if no output is found
      if [ ! -d ${outdir}/ic ]; then
        echo "SAVEIC problem! no files in ${outdir}/ic"
        exit 1
      fi

      #Apply proper permissions
      chmod 775 $(find ${outdir}/ic/ -type d)
      chmod 654 $(find ${outdir}/ic/ -type f)

      # optionally copy files to IC repository
      if [[ $SAVE_IC_REPOS == "TRUE" ]]; then
        # TODO test this on CCA
        if [[ "${HPCARCH}" = "ecmwf-xc40" ]]; then
          echo "SAVE_IC_REPOS = TRUE not supported for platform ${HPCARCH}"
          exit 1
        fi
        # copy files to IC_DIR
        pathout_ic=${IC_DIR}
        # move each file to IC repository in the proper directory
        for ifile in $( (cd ${outdir}/ic && find * -type f)); do
          fdir=$(dirname ${ifile})
          # Create the folder && apply permissions in case we are overwritting files in the target file
          # todo a platform-dependent function/option for this
          target_group=bsc32
          sg ${target_group} "mkdir -p -m775 ${pathout_ic}/${fdir}"
          cp -f ${outdir}/ic/${ifile} ${pathout_ic}/${fdir}
          chmod 744 ${pathout_ic}/${ifile}
          echo moved ${ifile} to ${pathout_ic}/${fdir}
        done
      fi

      # create a tarfile which can be untarred later in the IC repository
      pushd .
      cd ${outdir}/ic
      tar -cvf ../${tarfile} *
      chmod 774 ../${tarfile}
      cd ..

      # move to temp folder, clean/transfer will transfer to /esarchive
      mkdir -p ${PATHOUT_IC}
      mv -f ${tarfile} ${PATHOUT_IC}
      echo moved ${tarfile} to ${PATHOUT_IC}

      popd

      # cleanup, this assumes that files have been copied correctly
      rm -rf ${outdir}

    done #for

  else
    echo "doing nothing"
  fi

else
  echo "doing nothing"
fi
+ has_config save_ic
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' save_ic == any ']'
+ [[ save_ic == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ save_ic ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ save_ic ]]
+ for __m in '$config'
+ [[ lim3 =~ save_ic ]]
+ for __m in '$config'
+ [[ rnfmapper =~ save_ic ]]
+ for __m in '$config'
+ [[ xios:detached =~ save_ic ]]
+ for __m in '$config'
+ [[ oasis =~ save_ic ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ save_ic ]]
+ [[ save_ic:end_leg =~ save_ic ]]
+ __f=1
+ ((  __f  ))
+ return 0
+ source /gpfs/scratch/bsc32/bsc32627/t0ni/libsave_ic.sh
#================================================================================
# save_ic functions - these are used to save EC-Earth states during a run,
# which can be used as Initial Conditions (ICs) for other runs.
#
# save_ic_get_config : define the config to use based on the save_ic config parameter
#
# save_ic has two options available:
#   save_ic:end_leg (save ICs at the end of every leg)
#   save_ic:end_run (save ICs at the end of the run)
#
# By default, without an option, ICs are saved at given offsets from the leg
# start, possibly if conditions are met. These offset(s) and condition(s) must
# be coded in the "else" clause below. See commented example.
#
function save_ic_get_config ()
{
    do_save_ic=false
    save_ic_custom=false
    if $(has_config save_ic)
    then
        if $(has_config save_ic:end_leg)
        then
            # save ICs at the end of every leg
            do_save_ic=true
            save_ic_date_offset=( "+ ${leg_length_sec} sec" )
        elif $(has_config save_ic:end_run)
        then
            # save ICs at the end of the run
            leg_end_date_yyyymmdd=$(date -u -d "${leg_end_date}" +%Y%m%d)
            run_end_date_yyyymmdd=$(date -u -d "${run_end_date}" +%Y%m%d)
            if [ ${leg_end_date_yyyymmdd} -eq ${run_end_date_yyyymmdd} ]
            then
                do_save_ic=true && save_ic_date_offset=( "+ ${leg_length_sec} sec" )
            fi
        else
            echo "WARNING: save_ic requires to define condition and offset here or in the runscript"
            save_ic_custom=true
            do_save_ic=false
            save_ic_date_offset=( )
            # edit here or in the runscript the conditional statement to decide which legs to save ICs and the offset from leg start
            # e.g. to save at the end of every 10 legs:
            #EXAMPLE if (( leg_number%10 == 0 ))
            #EXAMPLE then
            #EXAMPLE     do_save_ic=true
            #EXAMPLE     save_ic_date_offset=( "+ ${leg_length_sec} sec" )
            #EXAMPLE fi
        fi
    fi
}

# save_ic_define_vars : define variables used in the save_ic functions
#
# The following variables must be defined in the runscript before calling this function
#
# 1) do_save_ic : set to true/false to enable save_ic feature
#
# 2) save_ic_date_offset : array of dates (offset from start of chunk, at 00:00h) to save ICs.
#    e.g. save_ic_date_offset=( "+1 month" "+2 month" )
#    The following restrictions apply, it is up to the user to ensure compliance
#     - first timestep of run is not supported
#     - if requesting first timestep of leg start, NEMO restarts from previous legs will
#       be used if present
#     - only one IC per month is supported (to simplify the IFS filter)
#     - maximum 9 ICs per leg (because of current restrictions in NEMO namelist)
#     - if requesting model-level output for IFS (e.g. for PRIMAVERA),
#       there will be no model-level variables in output on the timestep which
#       ICs are requested, unless requesting the last timestep of the leg
#
function save_ic_define_vars ()
{
    if true
    then
        # save original value of ifs_lastout
        has_config ifs && [[ -z "${save_ic_ifs_lastout_orig:-}" ]] && save_ic_ifs_lastout_orig=${ifs_lastout}
        save_ic_ifs_lastout=false

        # sanity checks
        [ ${#save_ic_date_offset[@]} -gt 9 ] && error "Cannot create more than 9 ICs in a given leg"

        # fill vars
        for (( i=0; i< ${#save_ic_date_offset[@]}; i++ ))
        do
            #save_ic_date1[$i]=${save_ic_date[$i]}
            save_ic_date1[$i]=$(date -u -d "${leg_start_date} ${save_ic_date_offset[$i]}" +%Y%m%d)
            save_ic_date[$i]=$(date -uR -d "${save_ic_date1[$i]}")
            save_ic_sec[$i]=$(( $(date -d "${save_ic_date[$i]}" +%s) - $(date -d "${run_start_date}" +%s) ))
            # Correct for leap days because NEMO standalone uses no-leap calendar
            has_config nemo && ! has_config ifs && save_ic_sec[$i]=$(( save_ic_sec[$i] - $(leap_days "${run_start_date}" "${save_ic_date[$i]}")*24*3600 ))
            save_ic_day[$i]=$(( ${save_ic_sec[$i]} / 86400 ))
            has_config ifs && save_ic_ppt_files[$i]="ppt"$(printf %06d ${save_ic_day[$i]})"0000"
            has_config nemo && save_ic_nemo_ts[$i]=$(( save_ic_sec[$i] / nem_time_step_sec ))
            # sanity checks
            if [ ${save_ic_sec[$i]} -eq 0 ]
            then
                info "Initial conditions cannot be created for first timestep of run"
                has_config ifs && save_ic_ppt_files[$i]=""
                has_config nemo && save_ic_nemo_ts[$i]=-1
            elif [ ${save_ic_sec[$i]} -eq $leg_start_sec ]
            then
                if has_config nemo
                then
                    info "Cannot create NEMO ICs for first timestep of leg at save_ic_date1[$i], will use restarts from previous leg"
                    save_ic_nemo_ts[$i]=-1
                fi
            fi
            # if saving ICs at the end of the leg, set ifs_lastout=true
            if [ ${save_ic_sec[$i]} -eq $leg_end_sec ]
            then
                if $(has_config ifs) && ! ${save_ic_ifs_lastout_orig}
                then
                    ifs_lastout=true
                    save_ic_ifs_lastout=true
                fi
            fi
        done
    fi
}

# prepare output configuration (NEMO namelist, IFS ppt file)
function save_ic_prepare_output()
{
    # For NEMO this is handled in the namelist variables ln_rst_list and nn_stocklist in namelist.nemo.ref.sh
    if has_config nemo
    then
        nemo_stocklist=""
        #for (( i=0; i< ${#save_ic_date[@]}; i++ )) ; do nemo_stocklist+=$(( save_ic_sec[$i] / nem_time_step_sec )), ; done
        for (( i=0; i< ${#save_ic_date[@]}; i++ ))
        do
            [ ${save_ic_nemo_ts[$i]} -ne -1 ] && nemo_stocklist+=${save_ic_nemo_ts[$i]},
        done
        nemo_stocklist+=$(( leg_end_sec / nem_time_step_sec )),
        for (( i=${#save_ic_date[@]}; i<9 ; i++ )) ; do nemo_stocklist+=0, ; done
        nemo_stocklist+=$'\n'
    fi

    # For IFS we need to create links to special ppt file
    if has_config ifs
    then
        # create the special ppt file
        save_ic_ppt_filename=pptdddddd0000_save_ic
        save_ic_ppt_ifile=pptdddddd0000 # change this for e.g. primavera
        # these are the codes which must be added to the special ppt file used for generating
        # required IFS model-level output at each requested timestep
        save_ic_MFP2DF_param_ids="129,152"
        save_ic_MFPPHY_param_ids="139,170,183,236,39,40,41,42,141,198,235,31,238,32,33,34,35,36,37,38,148,172,173,174,15,17,16,18,66,67,74,43,160,161,162,163,234,28,27,30,29" #all variables needed
        #save_ic_MFPPHY_param_ids="198,36,37,38,148,172,173,174,15,17,16,18,66,67,74,43,160,161,162,163,234,28,27,30,29" #all variables (old)
        #save_ic_MFPPHY_param_ids="198,36,37,38,148,172,173,174,15,17,16,18,74,43,160,161,162,163,234" #trunk
        #save_ic_MFPPHY_param_ids="173,174,15,17,16,18,66,67,74,43,160,161,162,163,234,28,27,30,29" #primavera
        save_ic_filter_param_ids=""
        # get requested output codes as a single line without space and without the last comma
        NFPPHY=`grep NFPPHY postins/$save_ic_ppt_ifile | sed -e 's/^\s*NFPPHY\s*=\s*//' -e 's/[,]$//'`
        MFPPHY=$(sed -e '1h;2,$H;$!d;g' -re 's/.*MFPPHY *=([0-9,\. \n]+).*/\1/' -e 's/[ \n]//g' -e 's/,$//' <postins/$save_ic_ppt_ifile)
        [[ $MFPPHY == "" ]] && error "IFS ppt file postins/$save_ic_ppt_ifile does not contain any entries in MFPPHY."

        NFP2DF=`grep NFP2DF postins/$save_ic_ppt_ifile | sed -e 's/^\s*NFP2DF\s*=\s*//' -e 's/[,]$//'`
        MFP2DF=$(sed -e '1h;2,$H;$!d;g' -re 's/.*MFP2DF *=([0-9,\. \n]+).*/\1/' -e 's/[ \n]//g' -e 's/,$//' <postins/$save_ic_ppt_ifile)
        [[ $MFP2DF == "" ]] && NFP2DF=0 && MFP2DF=","

        IFS=","
        old_MFPPHY_params=( ${MFPPHY} )
        old_MFP2DF_params=( ${MFP2DF} )
        new_MFPPHY_params=( ${save_ic_MFPPHY_param_ids} )
        new_MFP2DF_params=( ${save_ic_MFP2DF_param_ids} )
        unset IFS
        # update requested output codes by adding necessary variables
        for new_param_id in ${new_MFPPHY_params[@]}
        do
            new_param_id=`printf "%0*d" 3 $new_param_id`
            found=false
            for old_param_id in ${old_MFPPHY_params[@]}
            do
                old_param_id=`printf "%03g" $old_param_id`
                [ ${old_param_id} -eq ${new_param_id} ] && found=true && break
            done
            if ! $found
            then
                NFPPHY=$(( $NFPPHY + 1 ))
                MFPPHY="${MFPPHY},$new_param_id"
                save_ic_filter_param_ids="${save_ic_filter_param_ids}$new_param_id,"
            fi
        done
        for new_param_id in ${new_MFP2DF_params[@]}
        do
            new_param_id=`printf "%0*d" 3 $new_param_id`
            found=false
            for old_param_id in ${old_MFP2DF_params[@]}
            do
                old_param_id=`printf "%03g" $old_param_id`
                [ ${old_param_id} -eq ${new_param_id} ] && found=true && break
            done
            if ! $found
            then
                NFP2DF=$(( $NFP2DF + 1 ))
                MFP2DF="${MFP2DF},$new_param_id"
            fi
        done

        # create special ppt file with modified codes from original ppt file
        sed -r -e '1h;2,$H;$!d;g' \
            -e 's|(NFPPHY\s*=)[0-9 ]+|\1'${NFPPHY}'|' \
            -e 's|(MFPPHY\s*=)[0-9, \n]+\n|\1'${MFPPHY}',\n|' \
            -e 's|(NFP2DF\s*=)\s*[0-9]+|\1'${NFP2DF}'|' \
            -e 's|(MFP2DF\s*=)[0-9, \n]+\n|\1'${MFP2DF}',\n|' \
            -e 's|NFP3DFS\s*=\s*[0-9]+,||' \
            -e 's|MFP3DFS\s*=[0-9, \n]+\n|\n|' \
            -e 's|NRFP3S\s*=\s*\-?[0-9]+,||' \
            -e 's/&NAMFPC/\&NAMFPC\n    NFP3DFS = 9,\n    MFP3DFS = 133,75,76,246,247,248,138,155,130,\n    NRFP3S  = -99\n    NFP2DF = 2,\n    MFP2DF = 152,129,/' \
            <postins/$save_ic_ppt_ifile >postins/${save_ic_ppt_filename}

        # cleanup existing links from previous legs (required when save_ic_ifs_lastout=true)
        ls -ltr postins
        find -L postins/ -xtype l -samefile postins/${save_ic_ppt_filename} -exec rm {} \;
        # create links to ppt file for all requested timesteps
        for (( i=0; i< ${#save_ic_date[@]}; i++ ))
        do
            if [ ! -f postins/$save_ic_ppt_filename ]
            then
                error "IFS IC file $save_ic_ppt_filename does not exist."
            elif [ "${save_ic_ppt_files[$i]}" != "" ]  ; then
                ln -sf $save_ic_ppt_filename postins/${save_ic_ppt_files[$i]}
            fi
        done
        /bin/ls -1 postins/* > dirlist
        ls -ltr postins
    fi
}

# post-process save_ic results
function save_ic_postproc()
{
    # cleanup existing links for subsequent legs (see issue #345)
    if has_config ifs
    then
        ls -ltr postins
        find -L  postins/ -xtype l -samefile postins/${save_ic_ppt_filename} -exec rm {} \;
        /bin/ls -1 postins/* > dirlist
        ls -ltr postins
    fi
    for (( i=0; i< ${#save_ic_date[@]}; i++ ))
    do
        outdir="save_ic/"${save_ic_date1[$i]}

        # save IFS ICs
        if has_config ifs
        then
            [ "${save_ic_ppt_files[$i]}" != "" ] && save_ic_ifs_out2init $i
        fi
        # copy oasis restarts
        if has_config oasis
        then
            # copy oasis restarts if this is the last or first timestep of the leg
            # oasis files depend on the components and configuration
            # so they go in their own sub-directory, easier to separate from ifs and nemo files
            if [ ${save_ic_sec[$i]} -eq $leg_end_sec ]
            then
                mkdir -p ${outdir}/oasis
                for f in ${oas_rst_files} ; do test -f ${f} && cp ${f} ${outdir}/oasis ; done
            elif [ ${save_ic_sec[$i]} -eq $leg_start_sec ]
            then
                mkdir -p ${outdir}/oasis
                oas_rst_dir="restart/oasis/$(printf %03d $((leg_number)))"
                for f in ${oas_rst_files} ; do test -f ${oas_rst_dir}/${f} && cp ${oas_rst_dir}/${f} ${outdir}/oasis ; done
            fi
        fi
        # copy NEMO restarts
        if has_config nemo
        then
            nemo_ts=$(( save_ic_sec[$i] / nem_time_step_sec ))
            ns=$(printf %08d $(( nemo_ts - nem_restart_offset )))
	    # copy each restart type seperately, depending on config
	    extensions="oce"
	    has_config lim3 && extensions+=" ice"
	    has_config pisces && extensions+=" trc"
	    for ext in $extensions
	    do
		if ls ${exp_name}_${ns}_restart_${ext}_????.nc > /dev/null 2>&1
		then
                    mkdir -p ${outdir}/nemo
                    if [[ ${save_ic_sec[$i]} -eq $leg_start_sec ]] || [[ ${save_ic_sec[$i]} -eq $leg_end_sec ]]
                    then
			cp -f ${exp_name}_${ns}_restart_${ext}_????.nc ${outdir}/nemo
                    else
			mv -f ${exp_name}_${ns}_restart_${ext}_????.nc ${outdir}/nemo
                    fi
		else
                    info "Cannot find NEMO ($ext) initial conditions for date ${save_ic_date1[$i]}"
		fi
	    done
        fi
        # copy TM5 restarts
        leg_end_date_yyyymmdd=$(date -u -d "${leg_end_date}" +%Y%m%d)
        if has_config tm5 && [[ $leg_end_date_yyyymmdd == ${save_ic_date1[$i]} ]]
        then
	    found=false
	    for f in TM5_restart_${leg_end_date_yyyymmdd}_0000_glb300x200.nc save_${leg_end_date_yyyymmdd}00_glb300x200.hdf
	    do
		if [[ -f "$f" ]] ; then
		    mkdir -p ${outdir}/tm5
		    cp -f $f ${outdir}/tm5
		    found=true
		fi
	    done
            ! $found && info "Cannot find TM5 restarts for date ${save_ic_date1[$i]}" || true
	fi
    done
}

# Function to create IC files for IFS from special output based on a script from K. Wyser
function save_ic_ifs_out2init()
{
    icdate=${save_ic_date1[$1]}
    #srcdir=$2
    filter_output=true
    grib_filter=${GRIB_BIN_PATH}${GRIB_BIN_PATH:+/}grib_filter

    # temporary directory
    tmpdir=tmp_save_ic
    [ -d $tmpdir ] && rm -rf ${tmpdir}
    mkdir -p $tmpdir

    # first IFS timestep of a month is saved in last output file
    if [[ "$(date -d "${icdate}" +%d)" == "01" ]] ; then
        yyyymm=$(date -d "${icdate} - 6 hours" +%Y%m)
    else
        yyyymm=$(date -d "${icdate}" +%Y%m)
    fi

    # find the ICMSH/GG files containing the last timestep, which are in $srcdir/output/ifs/???
    #local ifile_sh=`find $srcdir -name ICMSH${exp_name}+$yyyymm -print -quit`
    #local ifile_gg=`find $srcdir -name ICMGG${exp_name}+$yyyymm -print -quit`
    ifile_sh=ICMSH${exp_name}+$yyyymm
    ifile_gg=ICMGG${exp_name}+$yyyymm
    if [[ ! -f "$ifile_sh" ]] ; then info "cannot find ICMSH file $ifile_sh to create initial conditions at $icdate!" ; return 0 ; fi
    if [[ ! -f "$ifile_gg" ]] ; then info "cannot find ICMGG file $ifile_gg to create initial conditions at $icdate!" ; return 0 ; fi

    if $filter_output
    then
        # define grib filters to separate data for IC from normal output
        # filter_time_init is to define which timestep contains the ICs, currently only one per month is supported
        filter_time_init="dataDate == ${icdate} && dataTime == 0"
        # filter_time_out is to filter any timesteps which might contain ICs
        filter_time_out="dataTime == 0 && ( 0 "
        for (( j=0; j< ${#save_ic_date[@]}; j++ )) ; do filter_time_out+=" || ( dataDate == ${save_ic_date1[$j]} )"; done
        filter_time_out+=" )"
        $save_ic_ifs_lastout && filter_last="( dataTime == 0 ) && ( dataDate == ${save_ic_date1[$((j-1))]} )" || filter_last="0"
        # build expression to filter the paramIDs which we have to filter out, set them in ini_filter_param_ids in the runscript
        #save_ic_filter_param_ids="198,36,37,38,148,172,173,174,15,17,16,18,74,43,160,161,162,163,234"
        filter_params="1"
        IFS=","
        for p in $save_ic_filter_param_ids ; do filter_params+=" && paramId != $p" ; done
        unset IFS

        # SH file
        ofile_sh_init=${tmpdir}/sh_init.grb
        ofile_sh_out=${tmpdir}/sh_out.grb
        ofile_sh_last=${tmpdir}/sh_last.grb
        filter_sh=${tmpdir}/filter_sh
        write_sh_init="if ( $filter_time_init ) { write \"${ofile_sh_init}\"; }"
        write_sh_out="if ( $filter_last ) { write \"${ofile_sh_last}\"; } else { write \"${ofile_sh_out}\"; }"

        cat > $filter_sh << EOT
if ( ! ( $filter_time_out ) ) { write "${ofile_sh_out}"; }
else {
if ( typeOfLevel is "hybrid" ) { ${write_sh_init}; }
else { ${write_sh_out}; }
}
EOT

        #GG file
        ofile_gg_init=${tmpdir}/gg_init.grb
        ofile_gg_out=${tmpdir}/gg_out.grb
        ofile_gg_last=${tmpdir}/gg_last.grb
        filter_gg=${tmpdir}/filter_gg
        write_gg_init="if ( $filter_time_init ) { write \"${ofile_gg_init}\"; }"
        write_gg_out="if ( $filter_last ) { write \"${ofile_gg_last}\"; } else { write \"${ofile_gg_out}\"; }"

        cat > $filter_gg << EOT
if ( ! ( $filter_time_out ) ) { write "${ofile_gg_out}"; }
else {
if ( levelType is "ml" ) { ${write_gg_init}; }
else {
  if ( levelType is "pl" ) { ${write_gg_out}; }
  else {
    if ( $filter_params ) {
      ${write_gg_init};
      ${write_gg_out};
    }
    else {
      ${write_gg_init};
    }
  }
}
}
EOT

        #run grib_filter on SH and GG files
        $grib_filter $filter_sh $ifile_sh
        $grib_filter $filter_gg $ifile_gg
        if [[ ! -f "$ofile_sh_init" ]] ; then info "ICMSH file $ifile_sh does not contain data to create initial conditions at $icdate" ; return 0 ; fi
        if [[ ! -f "$ofile_gg_init" ]] ; then info "ICMGG file $ifile_gg does not contain data to create initial conditions at $icdate" ; return 0 ; fi

        #rename files
        ifile_sh=${tmpdir}/$(basename $ifile_sh)
        ifile_gg=${tmpdir}/$(basename $ifile_gg)
        mv ${ofile_sh_init} ${ifile_sh}
        mv ${ofile_gg_init} ${ifile_gg}
        #srcdir=${tmpdir}

    fi # $filter_output

    if [[ ! -f "$ifile_sh" ]] ; then info "cannot find ICMSH file $ifile_sh to create initial conditions at $icdate!" ; return 0 ; fi
    if [[ ! -f "$ifile_gg" ]] ; then info "cannot find ICMGG file $ifile_gg to create initial conditions at $icdate!" ; return 0 ; fi

    tgtdir=save_ic/$icdate/ifs
    ofile_sh=${tgtdir}/ICMSH${exp_name}INIT
    ofile_gg_init=${tgtdir}/ICMGG${exp_name}INIT
    ofile_gg_iniua=${tgtdir}/ICMGG${exp_name}INIUA

    # make sure the output folder is created and delete any existing files
    #[ -d $tgtdir ] && rm -rf ${tgtdir}/* || mkdir -p $tgtdir
    mkdir -p $tgtdir
    rm -f $ofile_sh $ofile_gg_init $ofile_gg_iniua

    cat > ${tmpdir}/gf1 << EOT
if ( typeOfLevel is "hybrid" ) { write "${tmpdir}/shinit.[shortName].[level]"; }
EOT
    $grib_filter ${tmpdir}/gf1 $ifile_sh

    cp -f ${tmpdir}/shinit.lnsp.1 $ofile_sh
    for lev in {1..91}
    do
        for var in vo d t
        do
            cat ${tmpdir}/shinit.$var.$lev >> $ofile_sh
        done
    done
    cat ${tmpdir}/shinit.z.1 >> $ofile_sh

    cat > ${tmpdir}/gf2 << EOT
write "${tmpdir}/gginit.[shortName]";
EOT
    $grib_filter ${tmpdir}/gf2 $ifile_gg

    for var in stl1 stl2 stl3 stl4 swvl1 swvl2 swvl3 swvl4 sd src skt ci tsn asn \
        rsn sst istl1 istl2 istl3 istl4 chnk lsm sr al aluvp alnip aluvd alnid \
        lai_lv lai_hv sdfor slt sdor isor anor slor lsrh cvh cvl tvh tvl
    do
        cat ${tmpdir}/gginit.$var >> $ofile_gg_init
    done

    cat > ${tmpdir}/gf3 << EOT
write "${tmpdir}/gginiua.[shortName].[level]";
EOT
    $grib_filter ${tmpdir}/gf3 $ifile_gg

    #   ${tmpdir}/gginiua.o3 \ check this!
    for lev in {1..91}
    do
        for var in q
        do
            cat ${tmpdir}/gginiua.$var.$lev >> $ofile_gg_iniua
        done
    done

    for lev in {1..91}
    do
        for var in crwc cswc clwc ciwc cc
        do
            cat ${tmpdir}/gginiua.$var.$lev >> $ofile_gg_iniua
        done
    done

    # copy filtered files to the appropriate location
    if $filter_output
    then
        # move filter files to $tgtdir, in case something went wrong
        mv -f $filter_sh $filter_gg $tgtdir
        ifile_sh=$(basename $ifile_sh)
        ifile_gg=$(basename $ifile_gg)
        # keep the full output files as a backup
        mv -f $ifile_sh $tgtdir/${ifile_sh}-out+init
        mv -f $ifile_gg $tgtdir/${ifile_gg}-out+init
        # move the filtered output file to the  runtime folder
        [ -f $ofile_sh_out ] && mv -f $ofile_sh_out $ifile_sh || error "save_ic_ifs_out2init - warning! SH file $ofile_sh_out missing!"
        [ -f $ofile_gg_out ] && mv -f $ofile_gg_out $ifile_gg || error "save_ic_ifs_out2init - warning! GG file $ofile_gg_out missing!"
        # keep a copy of the last timestep
        if $save_ic_ifs_lastout
        then
            [ -f $ofile_sh_last ] && mv -f $ofile_sh_last $tgtdir/${ifile_sh}-ifs_lastout
            [ -f $ofile_gg_last ] && mv -f $ofile_gg_last $tgtdir/${ifile_gg}-ifs_lastout
        fi
    fi #$filter_output

    # delete tmp folder
    rm -rf $tmpdir

    echo "save_ic_ifs_out2init ended successfully, results are in $tgtdir"
}
+ declare -a save_ic_date save_ic_date1 save_ic_sec save_ic_day save_ic_ppt_file save_ic_nemo_ts
+ run_start_date=19931101
+ run_end_date='19931101 + 1 month'
+ leg_start_date=19931101
+ leg_end_date='19931130 + 1 day'
date -uR -d "${run_start_date}"
++ date -uR -d 19931101
+ run_start_date='Mon, 01 Nov 1993 00:00:00 +0000'
date -uR -d "${run_end_date}"
++ date -uR -d '19931101 + 1 month'
+ run_end_date='Wed, 01 Dec 1993 00:00:00 +0000'
date -uR -d "${leg_start_date}"
++ date -uR -d 19931101
+ leg_start_date='Mon, 01 Nov 1993 00:00:00 +0000'
date -uR -d "${leg_end_date}"
++ date -uR -d '19931130 + 1 day'
+ leg_end_date='Wed, 01 Dec 1993 00:00:00 +0000'
date -d "${leg_end_date}" +%s
++ date -d 'Wed, 01 Dec 1993 00:00:00 +0000' +%s
date -d "${leg_start_date}" +%s
++ date -d 'Mon, 01 Nov 1993 00:00:00 +0000' +%s
+ leg_length_sec=2592000
date -d "${leg_start_date}" +%s
++ date -d 'Mon, 01 Nov 1993 00:00:00 +0000' +%s
date -d "${run_start_date}" +%s
++ date -d 'Mon, 01 Nov 1993 00:00:00 +0000' +%s
+ leg_start_sec=0
date -d "${leg_end_date}" +%s
++ date -d 'Wed, 01 Dec 1993 00:00:00 +0000' +%s
date -d "${run_start_date}" +%s
++ date -d 'Mon, 01 Nov 1993 00:00:00 +0000' +%s
+ leg_end_sec=2592000
date -u -d "${leg_start_date}" +%Y%m%d
++ date -u -d 'Mon, 01 Nov 1993 00:00:00 +0000' +%Y%m%d
+ leg_start_date_yyyymmdd=19931101
date -u -d "${leg_start_date}" +%Y
++ date -u -d 'Mon, 01 Nov 1993 00:00:00 +0000' +%Y
+ leg_start_date_yyyy=1993
date -u -d "${leg_end_date}" +%Y
++ date -u -d 'Wed, 01 Dec 1993 00:00:00 +0000' +%Y
+ leg_end_date_yyyy=1993
date -u -d "${leg_end_date}" +%Y%m%d
++ date -u -d 'Wed, 01 Dec 1993 00:00:00 +0000' +%Y%m%d
+ leg_end_date_yyyymmdd=19931201
+ do_save_ic=false
+ save_ic_custom=false
+ has_config save_ic
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' save_ic == any ']'
+ [[ save_ic == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ save_ic ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ save_ic ]]
+ for __m in '$config'
+ [[ lim3 =~ save_ic ]]
+ for __m in '$config'
+ [[ rnfmapper =~ save_ic ]]
+ for __m in '$config'
+ [[ xios:detached =~ save_ic ]]
+ for __m in '$config'
+ [[ oasis =~ save_ic ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ save_ic ]]
+ [[ save_ic:end_leg =~ save_ic ]]
+ __f=1
+ ((  __f  ))
+ return 0
+ save_ic_get_config
+ do_save_ic=false
+ save_ic_custom=false
has_config save_ic
++ has_config save_ic
++ ((  1  ))
++ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
++ local __c
++ local __m
++ '[' save_ic == any ']'
++ [[ save_ic == \a\l\l ]]
++ local __f
++ for __c in '"$@"'
++ __f=0
++ for __m in '$config'
++ [[ ifs =~ save_ic ]]
++ for __m in '$config'
++ [[ nemo:start_from_restart:elpin =~ save_ic ]]
++ for __m in '$config'
++ [[ lim3 =~ save_ic ]]
++ for __m in '$config'
++ [[ rnfmapper =~ save_ic ]]
++ for __m in '$config'
++ [[ xios:detached =~ save_ic ]]
++ for __m in '$config'
++ [[ oasis =~ save_ic ]]
++ for __m in '$config'
++ [[ save_ic:end_leg =~ save_ic ]]
++ [[ save_ic:end_leg =~ save_ic ]]
++ __f=1
++ ((  __f  ))
++ return 0
has_config save_ic:end_leg
++ has_config save_ic:end_leg
++ ((  1  ))
++ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
++ local __c
++ local __m
++ '[' save_ic:end_leg == any ']'
++ [[ save_ic:end_leg == \a\l\l ]]
++ local __f
++ for __c in '"$@"'
++ __f=0
++ for __m in '$config'
++ [[ ifs =~ save_ic ]]
++ for __m in '$config'
++ [[ nemo:start_from_restart:elpin =~ save_ic ]]
++ for __m in '$config'
++ [[ lim3 =~ save_ic ]]
++ for __m in '$config'
++ [[ rnfmapper =~ save_ic ]]
++ for __m in '$config'
++ [[ xios:detached =~ save_ic ]]
++ for __m in '$config'
++ [[ oasis =~ save_ic ]]
++ for __m in '$config'
++ [[ save_ic:end_leg =~ save_ic ]]
++ [[ save_ic:end_leg =~ end_leg ]]
++ __f=1
++ ((  __f  ))
++ return 0
+ do_save_ic=true
+ save_ic_date_offset=("+ ${leg_length_sec} sec")
+ false
+ true
+ save_ic_define_vars
+ true
+ has_config ifs
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' ifs == any ']'
+ [[ ifs == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ ifs ]]
+ [[ ifs =~ ifs ]]
+ __f=1
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ ifs ]]
+ for __m in '$config'
+ [[ lim3 =~ ifs ]]
+ for __m in '$config'
+ [[ rnfmapper =~ ifs ]]
+ for __m in '$config'
+ [[ xios:detached =~ ifs ]]
+ for __m in '$config'
+ [[ oasis =~ ifs ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ ifs ]]
+ ((  __f  ))
+ return 0
+ [[ -z '' ]]
+ save_ic_ifs_lastout_orig=false
+ save_ic_ifs_lastout=false
+ '[' 1 -gt 9 ']'
+ (( i=0 ))
+ (( i< 1 ))
date -u -d "${leg_start_date} ${save_ic_date_offset[$i]}" +%Y%m%d
++ date -u -d 'Mon, 01 Nov 1993 00:00:00 +0000 + 2592000 sec' +%Y%m%d
+ save_ic_date1[$i]=19931201
date -uR -d "${save_ic_date1[$i]}"
++ date -uR -d 19931201
+ save_ic_date[$i]='Wed, 01 Dec 1993 00:00:00 +0000'
date -d "${save_ic_date[$i]}" +%s
++ date -d 'Wed, 01 Dec 1993 00:00:00 +0000' +%s
date -d "${run_start_date}" +%s
++ date -d 'Mon, 01 Nov 1993 00:00:00 +0000' +%s
+ save_ic_sec[$i]=2592000
+ has_config nemo
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' nemo == any ']'
+ [[ nemo == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ nemo ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ nemo ]]
+ [[ nemo:start_from_restart:elpin =~ nemo ]]
+ __f=1
+ for __m in '$config'
+ [[ lim3 =~ nemo ]]
+ for __m in '$config'
+ [[ rnfmapper =~ nemo ]]
+ for __m in '$config'
+ [[ xios:detached =~ nemo ]]
+ for __m in '$config'
+ [[ oasis =~ nemo ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ nemo ]]
+ ((  __f  ))
+ return 0
+ has_config ifs
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' ifs == any ']'
+ [[ ifs == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ ifs ]]
+ [[ ifs =~ ifs ]]
+ __f=1
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ ifs ]]
+ for __m in '$config'
+ [[ lim3 =~ ifs ]]
+ for __m in '$config'
+ [[ rnfmapper =~ ifs ]]
+ for __m in '$config'
+ [[ xios:detached =~ ifs ]]
+ for __m in '$config'
+ [[ oasis =~ ifs ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ ifs ]]
+ ((  __f  ))
+ return 0
+ save_ic_day[$i]=30
+ has_config ifs
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' ifs == any ']'
+ [[ ifs == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ ifs ]]
+ [[ ifs =~ ifs ]]
+ __f=1
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ ifs ]]
+ for __m in '$config'
+ [[ lim3 =~ ifs ]]
+ for __m in '$config'
+ [[ rnfmapper =~ ifs ]]
+ for __m in '$config'
+ [[ xios:detached =~ ifs ]]
+ for __m in '$config'
+ [[ oasis =~ ifs ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ ifs ]]
+ ((  __f  ))
+ return 0
printf %06d ${save_ic_day[$i]}
++ printf %06d 30
+ save_ic_ppt_files[$i]=ppt0000300000
+ has_config nemo
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' nemo == any ']'
+ [[ nemo == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ nemo ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ nemo ]]
+ [[ nemo:start_from_restart:elpin =~ nemo ]]
+ __f=1
+ for __m in '$config'
+ [[ lim3 =~ nemo ]]
+ for __m in '$config'
+ [[ rnfmapper =~ nemo ]]
+ for __m in '$config'
+ [[ xios:detached =~ nemo ]]
+ for __m in '$config'
+ [[ oasis =~ nemo ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ nemo ]]
+ ((  __f  ))
+ return 0
+ save_ic_nemo_ts[$i]=2880
+ '[' 2592000 -eq 0 ']'
+ '[' 2592000 -eq 0 ']'
+ '[' 2592000 -eq 2592000 ']'
has_config ifs
++ has_config ifs
++ ((  1  ))
++ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
++ local __c
++ local __m
++ '[' ifs == any ']'
++ [[ ifs == \a\l\l ]]
++ local __f
++ for __c in '"$@"'
++ __f=0
++ for __m in '$config'
++ [[ ifs =~ ifs ]]
++ [[ ifs =~ ifs ]]
++ __f=1
++ for __m in '$config'
++ [[ nemo:start_from_restart:elpin =~ ifs ]]
++ for __m in '$config'
++ [[ lim3 =~ ifs ]]
++ for __m in '$config'
++ [[ rnfmapper =~ ifs ]]
++ for __m in '$config'
++ [[ xios:detached =~ ifs ]]
++ for __m in '$config'
++ [[ oasis =~ ifs ]]
++ for __m in '$config'
++ [[ save_ic:end_leg =~ ifs ]]
++ ((  __f  ))
++ return 0
+ false
+ ifs_lastout=true
+ save_ic_ifs_lastout=true
+ (( i++  ))
+ (( i< 1 ))
+ SAVE_IC_REPOS=FALSE
+ '[' -z FALSE ']'
+ true
+ (( i = 0 ))
+ (( i < 1 ))
+ echo =====================================
+ outdir=/gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201
+ tarfile=IC_t0ni_19931101_fc0_001_0_19931101-19931130.tar
+ '[' -f /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/ic/IC_t0ni_19931101_fc0_001_0_19931101-19931130.tar ']'
+ '[' '!' -d /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201 ']'
+ ifs_timestamp=1993120100
date -u -d "${save_ic_date[$i]} -1 day" +%Y%m%d
++ date -u -d 'Wed, 01 Dec 1993 00:00:00 +0000 -1 day' +%Y%m%d
+ nemo_timestamp=19931130
+ cd /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201
+ mkdir -p delete
+ echo files for 19931201 are in /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201
+ ls /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201
+ echo =====================================
+ has_config ifs
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' ifs == any ']'
+ [[ ifs == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ ifs ]]
+ [[ ifs =~ ifs ]]
+ __f=1
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ ifs ]]
+ for __m in '$config'
+ [[ lim3 =~ ifs ]]
+ for __m in '$config'
+ [[ rnfmapper =~ ifs ]]
+ for __m in '$config'
+ [[ xios:detached =~ ifs ]]
+ for __m in '$config'
+ [[ oasis =~ ifs ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ ifs ]]
+ ((  __f  ))
+ return 0
+ '[' '!' -d ifs ']'
+ cd ifs
+ ifiles='ICMGGt0niINIT ICMGGt0niINIUA ICMSHt0niINIT'
+ odir=/gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/atmos/T511L91/t0ni
+ ofile=t0ni_fc0_1993120100.tar.gz
count_files "$ifiles"
++ count_files 'ICMGGt0niINIT ICMGGt0niINIUA ICMSHt0niINIT'
++ local 'ifiles=ICMGGt0niINIT ICMGGt0niINIUA ICMSHt0niINIT'
ls $ifiles 2>/dev/null
+++ ls ICMGGt0niINIT ICMGGt0niINIUA ICMSHt0niINIT
++ local 'nifiles=ICMGGt0niINIT
ICMGGt0niINIUA
ICMSHt0niINIT'
++ nifiles=($nifiles)
++ nifiles=3
++ echo 3
+ nifiles=3
+ [[ 3 -ne 3 ]]
+ [[ 3 -eq 3 ]]
+ echo 'IFS ifiles ICMGGt0niINIT ICMGGt0niINIUA ICMSHt0niINIT found!'
+ rm -rf /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/atmos/T511L91/t0ni /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ifs_ok /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/tmp_save_ic
+ mkdir -p /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/atmos/T511L91/t0ni
+ tar -czvf /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/atmos/T511L91/t0ni/t0ni_fc0_1993120100.tar.gz ICMGGt0niINIT ICMGGt0niINIUA ICMSHt0niINIT
+ touch /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ifs_ok
+ mv ICMGGt0niINIT ICMGGt0niINIUA ICMSHt0niINIT ICMGGt0ni+199311-ifs_lastout ICMGGt0ni+199311-out+init ICMSHt0ni+199311-ifs_lastout ICMSHt0ni+199311-out+init ../delete
+ cd ..
+ echo =====================================
+ has_config nemo
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' nemo == any ']'
+ [[ nemo == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ nemo ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ nemo ]]
+ [[ nemo:start_from_restart:elpin =~ nemo ]]
+ __f=1
+ for __m in '$config'
+ [[ lim3 =~ nemo ]]
+ for __m in '$config'
+ [[ rnfmapper =~ nemo ]]
+ for __m in '$config'
+ [[ xios:detached =~ nemo ]]
+ for __m in '$config'
+ [[ oasis =~ nemo ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ nemo ]]
+ ((  __f  ))
+ return 0
+ [[ 2880 -ne -1 ]]
+ '[' '!' -d nemo ']'
+ cd nemo
printf %08d $((save_ic_nemo_ts[$i]))
++ printf %08d 2880
+ ns=00002880
+ ftype_=(oce)
+ filebase_=(${exp_name}_${ns}_restart_oce)
+ ofile_=(${OCEAN_ini}_${OCEAN_ini_member}_${nemo_timestamp}_restart.nc.gz)
+ odir_=(${outdir}/ic/ocean/${nem_grid}/${OCEAN_ini})
+ has_config lim3
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' lim3 == any ']'
+ [[ lim3 == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ lim3 ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ lim3 ]]
+ for __m in '$config'
+ [[ lim3 =~ lim3 ]]
+ [[ lim3 =~ lim3 ]]
+ __f=1
+ for __m in '$config'
+ [[ rnfmapper =~ lim3 ]]
+ for __m in '$config'
+ [[ xios:detached =~ lim3 ]]
+ for __m in '$config'
+ [[ oasis =~ lim3 ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ lim3 ]]
+ ((  __f  ))
+ return 0
+ ftype_+=(ice)
+ filebase_+=(${exp_name}_${ns}_restart_ice)
+ ofile_+=(${ICE_ini}_${ICE_ini_member}_${nemo_timestamp}_restart_ice.nc.gz)
+ odir_+=(${outdir}/ic/ice/${nem_grid_wol}_LIM3/${ICE_ini})
+ has_config pisces
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' pisces == any ']'
+ [[ pisces == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ pisces ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ pisces ]]
+ for __m in '$config'
+ [[ lim3 =~ pisces ]]
+ for __m in '$config'
+ [[ rnfmapper =~ pisces ]]
+ for __m in '$config'
+ [[ xios:detached =~ pisces ]]
+ for __m in '$config'
+ [[ oasis =~ pisces ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ pisces ]]
+ ((  __f  ))
+ return 1
+ echo filebase_oce: t0ni_00002880_restart_oce filebase_ice: t0ni_00002880_restart_ice
+ (( j = 0 ))
+ (( j < 2 ))
+ filebase=t0ni_00002880_restart_oce
+ ofile=t0ni_fc0_19931130_restart.nc.gz
+ odir=/gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/ocean/ORCA025L75/t0ni
+ ifiles=(${filebase}_*)
+ ndomain=1391
+ '[' 1391 -gt 1 ']'
+ rm -f /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/nemo/t0ni_00002880_restart_oce_ok
+ echo found 1391 files matching pattern t0ni_00002880_restart_oce
+ rm -f t0ni_00002880_restart_oce.nc t0ni_00002880_restart_oce.nc.gz /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/nemo/t0ni_00002880_restart_oce_ok
+ rebuild_nemo -t48 t0ni_00002880_restart_oce 1391
+ rm -f nam_rebuild
+ gzip t0ni_00002880_restart_oce.nc
+ rm -rf /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/ocean/ORCA025L75/t0ni
+ mkdir -p /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/ocean/ORCA025L75/t0ni
+ mv t0ni_00002880_restart_oce.nc.gz /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/ocean/ORCA025L75/t0ni/t0ni_fc0_19931130_restart.nc.gz
+ touch /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/nemo/t0ni_00002880_restart_oce_ok
+ mv t0ni_00002880_restart_oce_0000.nc t0ni_00002880_restart_oce_0001.nc t0ni_00002880_restart_oce_0002.nc t0ni_00002880_restart_oce_0003.nc t0ni_00002880_restart_oce_0004.nc t0ni_00002880_restart_oce_0005.nc t0ni_00002880_restart_oce_0006.nc t0ni_00002880_restart_oce_0007.nc t0ni_00002880_restart_oce_0008.nc t0ni_00002880_restart_oce_0009.nc t0ni_00002880_restart_oce_0010.nc t0ni_00002880_restart_oce_0011.nc t0ni_00002880_restart_oce_0012.nc t0ni_00002880_restart_oce_0013.nc t0ni_00002880_restart_oce_0014.nc t0ni_00002880_restart_oce_0015.nc t0ni_00002880_restart_oce_0016.nc t0ni_00002880_restart_oce_0017.nc t0ni_00002880_restart_oce_0018.nc t0ni_00002880_restart_oce_0019.nc t0ni_00002880_restart_oce_0020.nc t0ni_00002880_restart_oce_0021.nc t0ni_00002880_restart_oce_0022.nc t0ni_00002880_restart_oce_0023.nc t0ni_00002880_restart_oce_0024.nc t0ni_00002880_restart_oce_0025.nc t0ni_00002880_restart_oce_0026.nc t0ni_00002880_restart_oce_0027.nc t0ni_00002880_restart_oce_0028.nc t0ni_00002880_restart_oce_0029.nc t0ni_00002880_restart_oce_0030.nc t0ni_00002880_restart_oce_0031.nc t0ni_00002880_restart_oce_0032.nc t0ni_00002880_restart_oce_0033.nc t0ni_00002880_restart_oce_0034.nc t0ni_00002880_restart_oce_0035.nc t0ni_00002880_restart_oce_0036.nc t0ni_00002880_restart_oce_0037.nc t0ni_00002880_restart_oce_0038.nc t0ni_00002880_restart_oce_0039.nc t0ni_00002880_restart_oce_0040.nc t0ni_00002880_restart_oce_0041.nc t0ni_00002880_restart_oce_0042.nc t0ni_00002880_restart_oce_0043.nc t0ni_00002880_restart_oce_0044.nc t0ni_00002880_restart_oce_0045.nc t0ni_00002880_restart_oce_0046.nc t0ni_00002880_restart_oce_0047.nc t0ni_00002880_restart_oce_0048.nc t0ni_00002880_restart_oce_0049.nc t0ni_00002880_restart_oce_0050.nc t0ni_00002880_restart_oce_0051.nc t0ni_00002880_restart_oce_0052.nc t0ni_00002880_restart_oce_0053.nc t0ni_00002880_restart_oce_0054.nc t0ni_00002880_restart_oce_0055.nc t0ni_00002880_restart_oce_0056.nc t0ni_00002880_restart_oce_0057.nc t0ni_00002880_restart_oce_0058.nc t0ni_00002880_restart_oce_0059.nc t0ni_00002880_restart_oce_0060.nc t0ni_00002880_restart_oce_0061.nc t0ni_00002880_restart_oce_0062.nc t0ni_00002880_restart_oce_0063.nc t0ni_00002880_restart_oce_0064.nc t0ni_00002880_restart_oce_0065.nc t0ni_00002880_restart_oce_0066.nc t0ni_00002880_restart_oce_0067.nc t0ni_00002880_restart_oce_0068.nc t0ni_00002880_restart_oce_0069.nc t0ni_00002880_restart_oce_0070.nc t0ni_00002880_restart_oce_0071.nc t0ni_00002880_restart_oce_0072.nc t0ni_00002880_restart_oce_0073.nc t0ni_00002880_restart_oce_0074.nc t0ni_00002880_restart_oce_0075.nc t0ni_00002880_restart_oce_0076.nc t0ni_00002880_restart_oce_0077.nc t0ni_00002880_restart_oce_0078.nc t0ni_00002880_restart_oce_0079.nc t0ni_00002880_restart_oce_0080.nc t0ni_00002880_restart_oce_0081.nc t0ni_00002880_restart_oce_0082.nc t0ni_00002880_restart_oce_0083.nc t0ni_00002880_restart_oce_0084.nc t0ni_00002880_restart_oce_0085.nc t0ni_00002880_restart_oce_0086.nc t0ni_00002880_restart_oce_0087.nc t0ni_00002880_restart_oce_0088.nc t0ni_00002880_restart_oce_0089.nc t0ni_00002880_restart_oce_0090.nc t0ni_00002880_restart_oce_0091.nc t0ni_00002880_restart_oce_0092.nc t0ni_00002880_restart_oce_0093.nc t0ni_00002880_restart_oce_0094.nc t0ni_00002880_restart_oce_0095.nc t0ni_00002880_restart_oce_0096.nc t0ni_00002880_restart_oce_0097.nc t0ni_00002880_restart_oce_0098.nc t0ni_00002880_restart_oce_0099.nc t0ni_00002880_restart_oce_0100.nc t0ni_00002880_restart_oce_0101.nc t0ni_00002880_restart_oce_0102.nc t0ni_00002880_restart_oce_0103.nc t0ni_00002880_restart_oce_0104.nc t0ni_00002880_restart_oce_0105.nc t0ni_00002880_restart_oce_0106.nc t0ni_00002880_restart_oce_0107.nc t0ni_00002880_restart_oce_0108.nc t0ni_00002880_restart_oce_0109.nc t0ni_00002880_restart_oce_0110.nc t0ni_00002880_restart_oce_0111.nc t0ni_00002880_restart_oce_0112.nc t0ni_00002880_restart_oce_0113.nc t0ni_00002880_restart_oce_0114.nc t0ni_00002880_restart_oce_0115.nc t0ni_00002880_restart_oce_0116.nc t0ni_00002880_restart_oce_0117.nc t0ni_00002880_restart_oce_0118.nc t0ni_00002880_restart_oce_0119.nc t0ni_00002880_restart_oce_0120.nc t0ni_00002880_restart_oce_0121.nc t0ni_00002880_restart_oce_0122.nc t0ni_00002880_restart_oce_0123.nc t0ni_00002880_restart_oce_0124.nc t0ni_00002880_restart_oce_0125.nc t0ni_00002880_restart_oce_0126.nc t0ni_00002880_restart_oce_0127.nc t0ni_00002880_restart_oce_0128.nc t0ni_00002880_restart_oce_0129.nc t0ni_00002880_restart_oce_0130.nc t0ni_00002880_restart_oce_0131.nc t0ni_00002880_restart_oce_0132.nc t0ni_00002880_restart_oce_0133.nc t0ni_00002880_restart_oce_0134.nc t0ni_00002880_restart_oce_0135.nc t0ni_00002880_restart_oce_0136.nc t0ni_00002880_restart_oce_0137.nc t0ni_00002880_restart_oce_0138.nc t0ni_00002880_restart_oce_0139.nc t0ni_00002880_restart_oce_0140.nc t0ni_00002880_restart_oce_0141.nc t0ni_00002880_restart_oce_0142.nc t0ni_00002880_restart_oce_0143.nc t0ni_00002880_restart_oce_0144.nc t0ni_00002880_restart_oce_0145.nc t0ni_00002880_restart_oce_0146.nc t0ni_00002880_restart_oce_0147.nc t0ni_00002880_restart_oce_0148.nc t0ni_00002880_restart_oce_0149.nc t0ni_00002880_restart_oce_0150.nc t0ni_00002880_restart_oce_0151.nc t0ni_00002880_restart_oce_0152.nc t0ni_00002880_restart_oce_0153.nc t0ni_00002880_restart_oce_0154.nc t0ni_00002880_restart_oce_0155.nc t0ni_00002880_restart_oce_0156.nc t0ni_00002880_restart_oce_0157.nc t0ni_00002880_restart_oce_0158.nc t0ni_00002880_restart_oce_0159.nc t0ni_00002880_restart_oce_0160.nc t0ni_00002880_restart_oce_0161.nc t0ni_00002880_restart_oce_0162.nc t0ni_00002880_restart_oce_0163.nc t0ni_00002880_restart_oce_0164.nc t0ni_00002880_restart_oce_0165.nc t0ni_00002880_restart_oce_0166.nc t0ni_00002880_restart_oce_0167.nc t0ni_00002880_restart_oce_0168.nc t0ni_00002880_restart_oce_0169.nc t0ni_00002880_restart_oce_0170.nc t0ni_00002880_restart_oce_0171.nc t0ni_00002880_restart_oce_0172.nc t0ni_00002880_restart_oce_0173.nc t0ni_00002880_restart_oce_0174.nc t0ni_00002880_restart_oce_0175.nc t0ni_00002880_restart_oce_0176.nc t0ni_00002880_restart_oce_0177.nc t0ni_00002880_restart_oce_0178.nc t0ni_00002880_restart_oce_0179.nc t0ni_00002880_restart_oce_0180.nc t0ni_00002880_restart_oce_0181.nc t0ni_00002880_restart_oce_0182.nc t0ni_00002880_restart_oce_0183.nc t0ni_00002880_restart_oce_0184.nc t0ni_00002880_restart_oce_0185.nc t0ni_00002880_restart_oce_0186.nc t0ni_00002880_restart_oce_0187.nc t0ni_00002880_restart_oce_0188.nc t0ni_00002880_restart_oce_0189.nc t0ni_00002880_restart_oce_0190.nc t0ni_00002880_restart_oce_0191.nc t0ni_00002880_restart_oce_0192.nc t0ni_00002880_restart_oce_0193.nc t0ni_00002880_restart_oce_0194.nc t0ni_00002880_restart_oce_0195.nc t0ni_00002880_restart_oce_0196.nc t0ni_00002880_restart_oce_0197.nc t0ni_00002880_restart_oce_0198.nc t0ni_00002880_restart_oce_0199.nc t0ni_00002880_restart_oce_0200.nc t0ni_00002880_restart_oce_0201.nc t0ni_00002880_restart_oce_0202.nc t0ni_00002880_restart_oce_0203.nc t0ni_00002880_restart_oce_0204.nc t0ni_00002880_restart_oce_0205.nc t0ni_00002880_restart_oce_0206.nc t0ni_00002880_restart_oce_0207.nc t0ni_00002880_restart_oce_0208.nc t0ni_00002880_restart_oce_0209.nc t0ni_00002880_restart_oce_0210.nc t0ni_00002880_restart_oce_0211.nc t0ni_00002880_restart_oce_0212.nc t0ni_00002880_restart_oce_0213.nc t0ni_00002880_restart_oce_0214.nc t0ni_00002880_restart_oce_0215.nc t0ni_00002880_restart_oce_0216.nc t0ni_00002880_restart_oce_0217.nc t0ni_00002880_restart_oce_0218.nc t0ni_00002880_restart_oce_0219.nc t0ni_00002880_restart_oce_0220.nc t0ni_00002880_restart_oce_0221.nc t0ni_00002880_restart_oce_0222.nc t0ni_00002880_restart_oce_0223.nc t0ni_00002880_restart_oce_0224.nc t0ni_00002880_restart_oce_0225.nc t0ni_00002880_restart_oce_0226.nc t0ni_00002880_restart_oce_0227.nc t0ni_00002880_restart_oce_0228.nc t0ni_00002880_restart_oce_0229.nc t0ni_00002880_restart_oce_0230.nc t0ni_00002880_restart_oce_0231.nc t0ni_00002880_restart_oce_0232.nc t0ni_00002880_restart_oce_0233.nc t0ni_00002880_restart_oce_0234.nc t0ni_00002880_restart_oce_0235.nc t0ni_00002880_restart_oce_0236.nc t0ni_00002880_restart_oce_0237.nc t0ni_00002880_restart_oce_0238.nc t0ni_00002880_restart_oce_0239.nc t0ni_00002880_restart_oce_0240.nc t0ni_00002880_restart_oce_0241.nc t0ni_00002880_restart_oce_0242.nc t0ni_00002880_restart_oce_0243.nc t0ni_00002880_restart_oce_0244.nc t0ni_00002880_restart_oce_0245.nc t0ni_00002880_restart_oce_0246.nc t0ni_00002880_restart_oce_0247.nc t0ni_00002880_restart_oce_0248.nc t0ni_00002880_restart_oce_0249.nc t0ni_00002880_restart_oce_0250.nc t0ni_00002880_restart_oce_0251.nc t0ni_00002880_restart_oce_0252.nc t0ni_00002880_restart_oce_0253.nc t0ni_00002880_restart_oce_0254.nc t0ni_00002880_restart_oce_0255.nc t0ni_00002880_restart_oce_0256.nc t0ni_00002880_restart_oce_0257.nc t0ni_00002880_restart_oce_0258.nc t0ni_00002880_restart_oce_0259.nc t0ni_00002880_restart_oce_0260.nc t0ni_00002880_restart_oce_0261.nc t0ni_00002880_restart_oce_0262.nc t0ni_00002880_restart_oce_0263.nc t0ni_00002880_restart_oce_0264.nc t0ni_00002880_restart_oce_0265.nc t0ni_00002880_restart_oce_0266.nc t0ni_00002880_restart_oce_0267.nc t0ni_00002880_restart_oce_0268.nc t0ni_00002880_restart_oce_0269.nc t0ni_00002880_restart_oce_0270.nc t0ni_00002880_restart_oce_0271.nc t0ni_00002880_restart_oce_0272.nc t0ni_00002880_restart_oce_0273.nc t0ni_00002880_restart_oce_0274.nc t0ni_00002880_restart_oce_0275.nc t0ni_00002880_restart_oce_0276.nc t0ni_00002880_restart_oce_0277.nc t0ni_00002880_restart_oce_0278.nc t0ni_00002880_restart_oce_0279.nc t0ni_00002880_restart_oce_0280.nc t0ni_00002880_restart_oce_0281.nc t0ni_00002880_restart_oce_0282.nc t0ni_00002880_restart_oce_0283.nc t0ni_00002880_restart_oce_0284.nc t0ni_00002880_restart_oce_0285.nc t0ni_00002880_restart_oce_0286.nc t0ni_00002880_restart_oce_0287.nc t0ni_00002880_restart_oce_0288.nc t0ni_00002880_restart_oce_0289.nc t0ni_00002880_restart_oce_0290.nc t0ni_00002880_restart_oce_0291.nc t0ni_00002880_restart_oce_0292.nc t0ni_00002880_restart_oce_0293.nc t0ni_00002880_restart_oce_0294.nc t0ni_00002880_restart_oce_0295.nc t0ni_00002880_restart_oce_0296.nc t0ni_00002880_restart_oce_0297.nc t0ni_00002880_restart_oce_0298.nc t0ni_00002880_restart_oce_0299.nc t0ni_00002880_restart_oce_0300.nc t0ni_00002880_restart_oce_0301.nc t0ni_00002880_restart_oce_0302.nc t0ni_00002880_restart_oce_0303.nc t0ni_00002880_restart_oce_0304.nc t0ni_00002880_restart_oce_0305.nc t0ni_00002880_restart_oce_0306.nc t0ni_00002880_restart_oce_0307.nc t0ni_00002880_restart_oce_0308.nc t0ni_00002880_restart_oce_0309.nc t0ni_00002880_restart_oce_0310.nc t0ni_00002880_restart_oce_0311.nc t0ni_00002880_restart_oce_0312.nc t0ni_00002880_restart_oce_0313.nc t0ni_00002880_restart_oce_0314.nc t0ni_00002880_restart_oce_0315.nc t0ni_00002880_restart_oce_0316.nc t0ni_00002880_restart_oce_0317.nc t0ni_00002880_restart_oce_0318.nc t0ni_00002880_restart_oce_0319.nc t0ni_00002880_restart_oce_0320.nc t0ni_00002880_restart_oce_0321.nc t0ni_00002880_restart_oce_0322.nc t0ni_00002880_restart_oce_0323.nc t0ni_00002880_restart_oce_0324.nc t0ni_00002880_restart_oce_0325.nc t0ni_00002880_restart_oce_0326.nc t0ni_00002880_restart_oce_0327.nc t0ni_00002880_restart_oce_0328.nc t0ni_00002880_restart_oce_0329.nc t0ni_00002880_restart_oce_0330.nc t0ni_00002880_restart_oce_0331.nc t0ni_00002880_restart_oce_0332.nc t0ni_00002880_restart_oce_0333.nc t0ni_00002880_restart_oce_0334.nc t0ni_00002880_restart_oce_0335.nc t0ni_00002880_restart_oce_0336.nc t0ni_00002880_restart_oce_0337.nc t0ni_00002880_restart_oce_0338.nc t0ni_00002880_restart_oce_0339.nc t0ni_00002880_restart_oce_0340.nc t0ni_00002880_restart_oce_0341.nc t0ni_00002880_restart_oce_0342.nc t0ni_00002880_restart_oce_0343.nc t0ni_00002880_restart_oce_0344.nc t0ni_00002880_restart_oce_0345.nc t0ni_00002880_restart_oce_0346.nc t0ni_00002880_restart_oce_0347.nc t0ni_00002880_restart_oce_0348.nc t0ni_00002880_restart_oce_0349.nc t0ni_00002880_restart_oce_0350.nc t0ni_00002880_restart_oce_0351.nc t0ni_00002880_restart_oce_0352.nc t0ni_00002880_restart_oce_0353.nc t0ni_00002880_restart_oce_0354.nc t0ni_00002880_restart_oce_0355.nc t0ni_00002880_restart_oce_0356.nc t0ni_00002880_restart_oce_0357.nc t0ni_00002880_restart_oce_0358.nc t0ni_00002880_restart_oce_0359.nc t0ni_00002880_restart_oce_0360.nc t0ni_00002880_restart_oce_0361.nc t0ni_00002880_restart_oce_0362.nc t0ni_00002880_restart_oce_0363.nc t0ni_00002880_restart_oce_0364.nc t0ni_00002880_restart_oce_0365.nc t0ni_00002880_restart_oce_0366.nc t0ni_00002880_restart_oce_0367.nc t0ni_00002880_restart_oce_0368.nc t0ni_00002880_restart_oce_0369.nc t0ni_00002880_restart_oce_0370.nc t0ni_00002880_restart_oce_0371.nc t0ni_00002880_restart_oce_0372.nc t0ni_00002880_restart_oce_0373.nc t0ni_00002880_restart_oce_0374.nc t0ni_00002880_restart_oce_0375.nc t0ni_00002880_restart_oce_0376.nc t0ni_00002880_restart_oce_0377.nc t0ni_00002880_restart_oce_0378.nc t0ni_00002880_restart_oce_0379.nc t0ni_00002880_restart_oce_0380.nc t0ni_00002880_restart_oce_0381.nc t0ni_00002880_restart_oce_0382.nc t0ni_00002880_restart_oce_0383.nc t0ni_00002880_restart_oce_0384.nc t0ni_00002880_restart_oce_0385.nc t0ni_00002880_restart_oce_0386.nc t0ni_00002880_restart_oce_0387.nc t0ni_00002880_restart_oce_0388.nc t0ni_00002880_restart_oce_0389.nc t0ni_00002880_restart_oce_0390.nc t0ni_00002880_restart_oce_0391.nc t0ni_00002880_restart_oce_0392.nc t0ni_00002880_restart_oce_0393.nc t0ni_00002880_restart_oce_0394.nc t0ni_00002880_restart_oce_0395.nc t0ni_00002880_restart_oce_0396.nc t0ni_00002880_restart_oce_0397.nc t0ni_00002880_restart_oce_0398.nc t0ni_00002880_restart_oce_0399.nc t0ni_00002880_restart_oce_0400.nc t0ni_00002880_restart_oce_0401.nc t0ni_00002880_restart_oce_0402.nc t0ni_00002880_restart_oce_0403.nc t0ni_00002880_restart_oce_0404.nc t0ni_00002880_restart_oce_0405.nc t0ni_00002880_restart_oce_0406.nc t0ni_00002880_restart_oce_0407.nc t0ni_00002880_restart_oce_0408.nc t0ni_00002880_restart_oce_0409.nc t0ni_00002880_restart_oce_0410.nc t0ni_00002880_restart_oce_0411.nc t0ni_00002880_restart_oce_0412.nc t0ni_00002880_restart_oce_0413.nc t0ni_00002880_restart_oce_0414.nc t0ni_00002880_restart_oce_0415.nc t0ni_00002880_restart_oce_0416.nc t0ni_00002880_restart_oce_0417.nc t0ni_00002880_restart_oce_0418.nc t0ni_00002880_restart_oce_0419.nc t0ni_00002880_restart_oce_0420.nc t0ni_00002880_restart_oce_0421.nc t0ni_00002880_restart_oce_0422.nc t0ni_00002880_restart_oce_0423.nc t0ni_00002880_restart_oce_0424.nc t0ni_00002880_restart_oce_0425.nc t0ni_00002880_restart_oce_0426.nc t0ni_00002880_restart_oce_0427.nc t0ni_00002880_restart_oce_0428.nc t0ni_00002880_restart_oce_0429.nc t0ni_00002880_restart_oce_0430.nc t0ni_00002880_restart_oce_0431.nc t0ni_00002880_restart_oce_0432.nc t0ni_00002880_restart_oce_0433.nc t0ni_00002880_restart_oce_0434.nc t0ni_00002880_restart_oce_0435.nc t0ni_00002880_restart_oce_0436.nc t0ni_00002880_restart_oce_0437.nc t0ni_00002880_restart_oce_0438.nc t0ni_00002880_restart_oce_0439.nc t0ni_00002880_restart_oce_0440.nc t0ni_00002880_restart_oce_0441.nc t0ni_00002880_restart_oce_0442.nc t0ni_00002880_restart_oce_0443.nc t0ni_00002880_restart_oce_0444.nc t0ni_00002880_restart_oce_0445.nc t0ni_00002880_restart_oce_0446.nc t0ni_00002880_restart_oce_0447.nc t0ni_00002880_restart_oce_0448.nc t0ni_00002880_restart_oce_0449.nc t0ni_00002880_restart_oce_0450.nc t0ni_00002880_restart_oce_0451.nc t0ni_00002880_restart_oce_0452.nc t0ni_00002880_restart_oce_0453.nc t0ni_00002880_restart_oce_0454.nc t0ni_00002880_restart_oce_0455.nc t0ni_00002880_restart_oce_0456.nc t0ni_00002880_restart_oce_0457.nc t0ni_00002880_restart_oce_0458.nc t0ni_00002880_restart_oce_0459.nc t0ni_00002880_restart_oce_0460.nc t0ni_00002880_restart_oce_0461.nc t0ni_00002880_restart_oce_0462.nc t0ni_00002880_restart_oce_0463.nc t0ni_00002880_restart_oce_0464.nc t0ni_00002880_restart_oce_0465.nc t0ni_00002880_restart_oce_0466.nc t0ni_00002880_restart_oce_0467.nc t0ni_00002880_restart_oce_0468.nc t0ni_00002880_restart_oce_0469.nc t0ni_00002880_restart_oce_0470.nc t0ni_00002880_restart_oce_0471.nc t0ni_00002880_restart_oce_0472.nc t0ni_00002880_restart_oce_0473.nc t0ni_00002880_restart_oce_0474.nc t0ni_00002880_restart_oce_0475.nc t0ni_00002880_restart_oce_0476.nc t0ni_00002880_restart_oce_0477.nc t0ni_00002880_restart_oce_0478.nc t0ni_00002880_restart_oce_0479.nc t0ni_00002880_restart_oce_0480.nc t0ni_00002880_restart_oce_0481.nc t0ni_00002880_restart_oce_0482.nc t0ni_00002880_restart_oce_0483.nc t0ni_00002880_restart_oce_0484.nc t0ni_00002880_restart_oce_0485.nc t0ni_00002880_restart_oce_0486.nc t0ni_00002880_restart_oce_0487.nc t0ni_00002880_restart_oce_0488.nc t0ni_00002880_restart_oce_0489.nc t0ni_00002880_restart_oce_0490.nc t0ni_00002880_restart_oce_0491.nc t0ni_00002880_restart_oce_0492.nc t0ni_00002880_restart_oce_0493.nc t0ni_00002880_restart_oce_0494.nc t0ni_00002880_restart_oce_0495.nc t0ni_00002880_restart_oce_0496.nc t0ni_00002880_restart_oce_0497.nc t0ni_00002880_restart_oce_0498.nc t0ni_00002880_restart_oce_0499.nc t0ni_00002880_restart_oce_0500.nc t0ni_00002880_restart_oce_0501.nc t0ni_00002880_restart_oce_0502.nc t0ni_00002880_restart_oce_0503.nc t0ni_00002880_restart_oce_0504.nc t0ni_00002880_restart_oce_0505.nc t0ni_00002880_restart_oce_0506.nc t0ni_00002880_restart_oce_0507.nc t0ni_00002880_restart_oce_0508.nc t0ni_00002880_restart_oce_0509.nc t0ni_00002880_restart_oce_0510.nc t0ni_00002880_restart_oce_0511.nc t0ni_00002880_restart_oce_0512.nc t0ni_00002880_restart_oce_0513.nc t0ni_00002880_restart_oce_0514.nc t0ni_00002880_restart_oce_0515.nc t0ni_00002880_restart_oce_0516.nc t0ni_00002880_restart_oce_0517.nc t0ni_00002880_restart_oce_0518.nc t0ni_00002880_restart_oce_0519.nc t0ni_00002880_restart_oce_0520.nc t0ni_00002880_restart_oce_0521.nc t0ni_00002880_restart_oce_0522.nc t0ni_00002880_restart_oce_0523.nc t0ni_00002880_restart_oce_0524.nc t0ni_00002880_restart_oce_0525.nc t0ni_00002880_restart_oce_0526.nc t0ni_00002880_restart_oce_0527.nc t0ni_00002880_restart_oce_0528.nc t0ni_00002880_restart_oce_0529.nc t0ni_00002880_restart_oce_0530.nc t0ni_00002880_restart_oce_0531.nc t0ni_00002880_restart_oce_0532.nc t0ni_00002880_restart_oce_0533.nc t0ni_00002880_restart_oce_0534.nc t0ni_00002880_restart_oce_0535.nc t0ni_00002880_restart_oce_0536.nc t0ni_00002880_restart_oce_0537.nc t0ni_00002880_restart_oce_0538.nc t0ni_00002880_restart_oce_0539.nc t0ni_00002880_restart_oce_0540.nc t0ni_00002880_restart_oce_0541.nc t0ni_00002880_restart_oce_0542.nc t0ni_00002880_restart_oce_0543.nc t0ni_00002880_restart_oce_0544.nc t0ni_00002880_restart_oce_0545.nc t0ni_00002880_restart_oce_0546.nc t0ni_00002880_restart_oce_0547.nc t0ni_00002880_restart_oce_0548.nc t0ni_00002880_restart_oce_0549.nc t0ni_00002880_restart_oce_0550.nc t0ni_00002880_restart_oce_0551.nc t0ni_00002880_restart_oce_0552.nc t0ni_00002880_restart_oce_0553.nc t0ni_00002880_restart_oce_0554.nc t0ni_00002880_restart_oce_0555.nc t0ni_00002880_restart_oce_0556.nc t0ni_00002880_restart_oce_0557.nc t0ni_00002880_restart_oce_0558.nc t0ni_00002880_restart_oce_0559.nc t0ni_00002880_restart_oce_0560.nc t0ni_00002880_restart_oce_0561.nc t0ni_00002880_restart_oce_0562.nc t0ni_00002880_restart_oce_0563.nc t0ni_00002880_restart_oce_0564.nc t0ni_00002880_restart_oce_0565.nc t0ni_00002880_restart_oce_0566.nc t0ni_00002880_restart_oce_0567.nc t0ni_00002880_restart_oce_0568.nc t0ni_00002880_restart_oce_0569.nc t0ni_00002880_restart_oce_0570.nc t0ni_00002880_restart_oce_0571.nc t0ni_00002880_restart_oce_0572.nc t0ni_00002880_restart_oce_0573.nc t0ni_00002880_restart_oce_0574.nc t0ni_00002880_restart_oce_0575.nc t0ni_00002880_restart_oce_0576.nc t0ni_00002880_restart_oce_0577.nc t0ni_00002880_restart_oce_0578.nc t0ni_00002880_restart_oce_0579.nc t0ni_00002880_restart_oce_0580.nc t0ni_00002880_restart_oce_0581.nc t0ni_00002880_restart_oce_0582.nc t0ni_00002880_restart_oce_0583.nc t0ni_00002880_restart_oce_0584.nc t0ni_00002880_restart_oce_0585.nc t0ni_00002880_restart_oce_0586.nc t0ni_00002880_restart_oce_0587.nc t0ni_00002880_restart_oce_0588.nc t0ni_00002880_restart_oce_0589.nc t0ni_00002880_restart_oce_0590.nc t0ni_00002880_restart_oce_0591.nc t0ni_00002880_restart_oce_0592.nc t0ni_00002880_restart_oce_0593.nc t0ni_00002880_restart_oce_0594.nc t0ni_00002880_restart_oce_0595.nc t0ni_00002880_restart_oce_0596.nc t0ni_00002880_restart_oce_0597.nc t0ni_00002880_restart_oce_0598.nc t0ni_00002880_restart_oce_0599.nc t0ni_00002880_restart_oce_0600.nc t0ni_00002880_restart_oce_0601.nc t0ni_00002880_restart_oce_0602.nc t0ni_00002880_restart_oce_0603.nc t0ni_00002880_restart_oce_0604.nc t0ni_00002880_restart_oce_0605.nc t0ni_00002880_restart_oce_0606.nc t0ni_00002880_restart_oce_0607.nc t0ni_00002880_restart_oce_0608.nc t0ni_00002880_restart_oce_0609.nc t0ni_00002880_restart_oce_0610.nc t0ni_00002880_restart_oce_0611.nc t0ni_00002880_restart_oce_0612.nc t0ni_00002880_restart_oce_0613.nc t0ni_00002880_restart_oce_0614.nc t0ni_00002880_restart_oce_0615.nc t0ni_00002880_restart_oce_0616.nc t0ni_00002880_restart_oce_0617.nc t0ni_00002880_restart_oce_0618.nc t0ni_00002880_restart_oce_0619.nc t0ni_00002880_restart_oce_0620.nc t0ni_00002880_restart_oce_0621.nc t0ni_00002880_restart_oce_0622.nc t0ni_00002880_restart_oce_0623.nc t0ni_00002880_restart_oce_0624.nc t0ni_00002880_restart_oce_0625.nc t0ni_00002880_restart_oce_0626.nc t0ni_00002880_restart_oce_0627.nc t0ni_00002880_restart_oce_0628.nc t0ni_00002880_restart_oce_0629.nc t0ni_00002880_restart_oce_0630.nc t0ni_00002880_restart_oce_0631.nc t0ni_00002880_restart_oce_0632.nc t0ni_00002880_restart_oce_0633.nc t0ni_00002880_restart_oce_0634.nc t0ni_00002880_restart_oce_0635.nc t0ni_00002880_restart_oce_0636.nc t0ni_00002880_restart_oce_0637.nc t0ni_00002880_restart_oce_0638.nc t0ni_00002880_restart_oce_0639.nc t0ni_00002880_restart_oce_0640.nc t0ni_00002880_restart_oce_0641.nc t0ni_00002880_restart_oce_0642.nc t0ni_00002880_restart_oce_0643.nc t0ni_00002880_restart_oce_0644.nc t0ni_00002880_restart_oce_0645.nc t0ni_00002880_restart_oce_0646.nc t0ni_00002880_restart_oce_0647.nc t0ni_00002880_restart_oce_0648.nc t0ni_00002880_restart_oce_0649.nc t0ni_00002880_restart_oce_0650.nc t0ni_00002880_restart_oce_0651.nc t0ni_00002880_restart_oce_0652.nc t0ni_00002880_restart_oce_0653.nc t0ni_00002880_restart_oce_0654.nc t0ni_00002880_restart_oce_0655.nc t0ni_00002880_restart_oce_0656.nc t0ni_00002880_restart_oce_0657.nc t0ni_00002880_restart_oce_0658.nc t0ni_00002880_restart_oce_0659.nc t0ni_00002880_restart_oce_0660.nc t0ni_00002880_restart_oce_0661.nc t0ni_00002880_restart_oce_0662.nc t0ni_00002880_restart_oce_0663.nc t0ni_00002880_restart_oce_0664.nc t0ni_00002880_restart_oce_0665.nc t0ni_00002880_restart_oce_0666.nc t0ni_00002880_restart_oce_0667.nc t0ni_00002880_restart_oce_0668.nc t0ni_00002880_restart_oce_0669.nc t0ni_00002880_restart_oce_0670.nc t0ni_00002880_restart_oce_0671.nc t0ni_00002880_restart_oce_0672.nc t0ni_00002880_restart_oce_0673.nc t0ni_00002880_restart_oce_0674.nc t0ni_00002880_restart_oce_0675.nc t0ni_00002880_restart_oce_0676.nc t0ni_00002880_restart_oce_0677.nc t0ni_00002880_restart_oce_0678.nc t0ni_00002880_restart_oce_0679.nc t0ni_00002880_restart_oce_0680.nc t0ni_00002880_restart_oce_0681.nc t0ni_00002880_restart_oce_0682.nc t0ni_00002880_restart_oce_0683.nc t0ni_00002880_restart_oce_0684.nc t0ni_00002880_restart_oce_0685.nc t0ni_00002880_restart_oce_0686.nc t0ni_00002880_restart_oce_0687.nc t0ni_00002880_restart_oce_0688.nc t0ni_00002880_restart_oce_0689.nc t0ni_00002880_restart_oce_0690.nc t0ni_00002880_restart_oce_0691.nc t0ni_00002880_restart_oce_0692.nc t0ni_00002880_restart_oce_0693.nc t0ni_00002880_restart_oce_0694.nc t0ni_00002880_restart_oce_0695.nc t0ni_00002880_restart_oce_0696.nc t0ni_00002880_restart_oce_0697.nc t0ni_00002880_restart_oce_0698.nc t0ni_00002880_restart_oce_0699.nc t0ni_00002880_restart_oce_0700.nc t0ni_00002880_restart_oce_0701.nc t0ni_00002880_restart_oce_0702.nc t0ni_00002880_restart_oce_0703.nc t0ni_00002880_restart_oce_0704.nc t0ni_00002880_restart_oce_0705.nc t0ni_00002880_restart_oce_0706.nc t0ni_00002880_restart_oce_0707.nc t0ni_00002880_restart_oce_0708.nc t0ni_00002880_restart_oce_0709.nc t0ni_00002880_restart_oce_0710.nc t0ni_00002880_restart_oce_0711.nc t0ni_00002880_restart_oce_0712.nc t0ni_00002880_restart_oce_0713.nc t0ni_00002880_restart_oce_0714.nc t0ni_00002880_restart_oce_0715.nc t0ni_00002880_restart_oce_0716.nc t0ni_00002880_restart_oce_0717.nc t0ni_00002880_restart_oce_0718.nc t0ni_00002880_restart_oce_0719.nc t0ni_00002880_restart_oce_0720.nc t0ni_00002880_restart_oce_0721.nc t0ni_00002880_restart_oce_0722.nc t0ni_00002880_restart_oce_0723.nc t0ni_00002880_restart_oce_0724.nc t0ni_00002880_restart_oce_0725.nc t0ni_00002880_restart_oce_0726.nc t0ni_00002880_restart_oce_0727.nc t0ni_00002880_restart_oce_0728.nc t0ni_00002880_restart_oce_0729.nc t0ni_00002880_restart_oce_0730.nc t0ni_00002880_restart_oce_0731.nc t0ni_00002880_restart_oce_0732.nc t0ni_00002880_restart_oce_0733.nc t0ni_00002880_restart_oce_0734.nc t0ni_00002880_restart_oce_0735.nc t0ni_00002880_restart_oce_0736.nc t0ni_00002880_restart_oce_0737.nc t0ni_00002880_restart_oce_0738.nc t0ni_00002880_restart_oce_0739.nc t0ni_00002880_restart_oce_0740.nc t0ni_00002880_restart_oce_0741.nc t0ni_00002880_restart_oce_0742.nc t0ni_00002880_restart_oce_0743.nc t0ni_00002880_restart_oce_0744.nc t0ni_00002880_restart_oce_0745.nc t0ni_00002880_restart_oce_0746.nc t0ni_00002880_restart_oce_0747.nc t0ni_00002880_restart_oce_0748.nc t0ni_00002880_restart_oce_0749.nc t0ni_00002880_restart_oce_0750.nc t0ni_00002880_restart_oce_0751.nc t0ni_00002880_restart_oce_0752.nc t0ni_00002880_restart_oce_0753.nc t0ni_00002880_restart_oce_0754.nc t0ni_00002880_restart_oce_0755.nc t0ni_00002880_restart_oce_0756.nc t0ni_00002880_restart_oce_0757.nc t0ni_00002880_restart_oce_0758.nc t0ni_00002880_restart_oce_0759.nc t0ni_00002880_restart_oce_0760.nc t0ni_00002880_restart_oce_0761.nc t0ni_00002880_restart_oce_0762.nc t0ni_00002880_restart_oce_0763.nc t0ni_00002880_restart_oce_0764.nc t0ni_00002880_restart_oce_0765.nc t0ni_00002880_restart_oce_0766.nc t0ni_00002880_restart_oce_0767.nc t0ni_00002880_restart_oce_0768.nc t0ni_00002880_restart_oce_0769.nc t0ni_00002880_restart_oce_0770.nc t0ni_00002880_restart_oce_0771.nc t0ni_00002880_restart_oce_0772.nc t0ni_00002880_restart_oce_0773.nc t0ni_00002880_restart_oce_0774.nc t0ni_00002880_restart_oce_0775.nc t0ni_00002880_restart_oce_0776.nc t0ni_00002880_restart_oce_0777.nc t0ni_00002880_restart_oce_0778.nc t0ni_00002880_restart_oce_0779.nc t0ni_00002880_restart_oce_0780.nc t0ni_00002880_restart_oce_0781.nc t0ni_00002880_restart_oce_0782.nc t0ni_00002880_restart_oce_0783.nc t0ni_00002880_restart_oce_0784.nc t0ni_00002880_restart_oce_0785.nc t0ni_00002880_restart_oce_0786.nc t0ni_00002880_restart_oce_0787.nc t0ni_00002880_restart_oce_0788.nc t0ni_00002880_restart_oce_0789.nc t0ni_00002880_restart_oce_0790.nc t0ni_00002880_restart_oce_0791.nc t0ni_00002880_restart_oce_0792.nc t0ni_00002880_restart_oce_0793.nc t0ni_00002880_restart_oce_0794.nc t0ni_00002880_restart_oce_0795.nc t0ni_00002880_restart_oce_0796.nc t0ni_00002880_restart_oce_0797.nc t0ni_00002880_restart_oce_0798.nc t0ni_00002880_restart_oce_0799.nc t0ni_00002880_restart_oce_0800.nc t0ni_00002880_restart_oce_0801.nc t0ni_00002880_restart_oce_0802.nc t0ni_00002880_restart_oce_0803.nc t0ni_00002880_restart_oce_0804.nc t0ni_00002880_restart_oce_0805.nc t0ni_00002880_restart_oce_0806.nc t0ni_00002880_restart_oce_0807.nc t0ni_00002880_restart_oce_0808.nc t0ni_00002880_restart_oce_0809.nc t0ni_00002880_restart_oce_0810.nc t0ni_00002880_restart_oce_0811.nc t0ni_00002880_restart_oce_0812.nc t0ni_00002880_restart_oce_0813.nc t0ni_00002880_restart_oce_0814.nc t0ni_00002880_restart_oce_0815.nc t0ni_00002880_restart_oce_0816.nc t0ni_00002880_restart_oce_0817.nc t0ni_00002880_restart_oce_0818.nc t0ni_00002880_restart_oce_0819.nc t0ni_00002880_restart_oce_0820.nc t0ni_00002880_restart_oce_0821.nc t0ni_00002880_restart_oce_0822.nc t0ni_00002880_restart_oce_0823.nc t0ni_00002880_restart_oce_0824.nc t0ni_00002880_restart_oce_0825.nc t0ni_00002880_restart_oce_0826.nc t0ni_00002880_restart_oce_0827.nc t0ni_00002880_restart_oce_0828.nc t0ni_00002880_restart_oce_0829.nc t0ni_00002880_restart_oce_0830.nc t0ni_00002880_restart_oce_0831.nc t0ni_00002880_restart_oce_0832.nc t0ni_00002880_restart_oce_0833.nc t0ni_00002880_restart_oce_0834.nc t0ni_00002880_restart_oce_0835.nc t0ni_00002880_restart_oce_0836.nc t0ni_00002880_restart_oce_0837.nc t0ni_00002880_restart_oce_0838.nc t0ni_00002880_restart_oce_0839.nc t0ni_00002880_restart_oce_0840.nc t0ni_00002880_restart_oce_0841.nc t0ni_00002880_restart_oce_0842.nc t0ni_00002880_restart_oce_0843.nc t0ni_00002880_restart_oce_0844.nc t0ni_00002880_restart_oce_0845.nc t0ni_00002880_restart_oce_0846.nc t0ni_00002880_restart_oce_0847.nc t0ni_00002880_restart_oce_0848.nc t0ni_00002880_restart_oce_0849.nc t0ni_00002880_restart_oce_0850.nc t0ni_00002880_restart_oce_0851.nc t0ni_00002880_restart_oce_0852.nc t0ni_00002880_restart_oce_0853.nc t0ni_00002880_restart_oce_0854.nc t0ni_00002880_restart_oce_0855.nc t0ni_00002880_restart_oce_0856.nc t0ni_00002880_restart_oce_0857.nc t0ni_00002880_restart_oce_0858.nc t0ni_00002880_restart_oce_0859.nc t0ni_00002880_restart_oce_0860.nc t0ni_00002880_restart_oce_0861.nc t0ni_00002880_restart_oce_0862.nc t0ni_00002880_restart_oce_0863.nc t0ni_00002880_restart_oce_0864.nc t0ni_00002880_restart_oce_0865.nc t0ni_00002880_restart_oce_0866.nc t0ni_00002880_restart_oce_0867.nc t0ni_00002880_restart_oce_0868.nc t0ni_00002880_restart_oce_0869.nc t0ni_00002880_restart_oce_0870.nc t0ni_00002880_restart_oce_0871.nc t0ni_00002880_restart_oce_0872.nc t0ni_00002880_restart_oce_0873.nc t0ni_00002880_restart_oce_0874.nc t0ni_00002880_restart_oce_0875.nc t0ni_00002880_restart_oce_0876.nc t0ni_00002880_restart_oce_0877.nc t0ni_00002880_restart_oce_0878.nc t0ni_00002880_restart_oce_0879.nc t0ni_00002880_restart_oce_0880.nc t0ni_00002880_restart_oce_0881.nc t0ni_00002880_restart_oce_0882.nc t0ni_00002880_restart_oce_0883.nc t0ni_00002880_restart_oce_0884.nc t0ni_00002880_restart_oce_0885.nc t0ni_00002880_restart_oce_0886.nc t0ni_00002880_restart_oce_0887.nc t0ni_00002880_restart_oce_0888.nc t0ni_00002880_restart_oce_0889.nc t0ni_00002880_restart_oce_0890.nc t0ni_00002880_restart_oce_0891.nc t0ni_00002880_restart_oce_0892.nc t0ni_00002880_restart_oce_0893.nc t0ni_00002880_restart_oce_0894.nc t0ni_00002880_restart_oce_0895.nc t0ni_00002880_restart_oce_0896.nc t0ni_00002880_restart_oce_0897.nc t0ni_00002880_restart_oce_0898.nc t0ni_00002880_restart_oce_0899.nc t0ni_00002880_restart_oce_0900.nc t0ni_00002880_restart_oce_0901.nc t0ni_00002880_restart_oce_0902.nc t0ni_00002880_restart_oce_0903.nc t0ni_00002880_restart_oce_0904.nc t0ni_00002880_restart_oce_0905.nc t0ni_00002880_restart_oce_0906.nc t0ni_00002880_restart_oce_0907.nc t0ni_00002880_restart_oce_0908.nc t0ni_00002880_restart_oce_0909.nc t0ni_00002880_restart_oce_0910.nc t0ni_00002880_restart_oce_0911.nc t0ni_00002880_restart_oce_0912.nc t0ni_00002880_restart_oce_0913.nc t0ni_00002880_restart_oce_0914.nc t0ni_00002880_restart_oce_0915.nc t0ni_00002880_restart_oce_0916.nc t0ni_00002880_restart_oce_0917.nc t0ni_00002880_restart_oce_0918.nc t0ni_00002880_restart_oce_0919.nc t0ni_00002880_restart_oce_0920.nc t0ni_00002880_restart_oce_0921.nc t0ni_00002880_restart_oce_0922.nc t0ni_00002880_restart_oce_0923.nc t0ni_00002880_restart_oce_0924.nc t0ni_00002880_restart_oce_0925.nc t0ni_00002880_restart_oce_0926.nc t0ni_00002880_restart_oce_0927.nc t0ni_00002880_restart_oce_0928.nc t0ni_00002880_restart_oce_0929.nc t0ni_00002880_restart_oce_0930.nc t0ni_00002880_restart_oce_0931.nc t0ni_00002880_restart_oce_0932.nc t0ni_00002880_restart_oce_0933.nc t0ni_00002880_restart_oce_0934.nc t0ni_00002880_restart_oce_0935.nc t0ni_00002880_restart_oce_0936.nc t0ni_00002880_restart_oce_0937.nc t0ni_00002880_restart_oce_0938.nc t0ni_00002880_restart_oce_0939.nc t0ni_00002880_restart_oce_0940.nc t0ni_00002880_restart_oce_0941.nc t0ni_00002880_restart_oce_0942.nc t0ni_00002880_restart_oce_0943.nc t0ni_00002880_restart_oce_0944.nc t0ni_00002880_restart_oce_0945.nc t0ni_00002880_restart_oce_0946.nc t0ni_00002880_restart_oce_0947.nc t0ni_00002880_restart_oce_0948.nc t0ni_00002880_restart_oce_0949.nc t0ni_00002880_restart_oce_0950.nc t0ni_00002880_restart_oce_0951.nc t0ni_00002880_restart_oce_0952.nc t0ni_00002880_restart_oce_0953.nc t0ni_00002880_restart_oce_0954.nc t0ni_00002880_restart_oce_0955.nc t0ni_00002880_restart_oce_0956.nc t0ni_00002880_restart_oce_0957.nc t0ni_00002880_restart_oce_0958.nc t0ni_00002880_restart_oce_0959.nc t0ni_00002880_restart_oce_0960.nc t0ni_00002880_restart_oce_0961.nc t0ni_00002880_restart_oce_0962.nc t0ni_00002880_restart_oce_0963.nc t0ni_00002880_restart_oce_0964.nc t0ni_00002880_restart_oce_0965.nc t0ni_00002880_restart_oce_0966.nc t0ni_00002880_restart_oce_0967.nc t0ni_00002880_restart_oce_0968.nc t0ni_00002880_restart_oce_0969.nc t0ni_00002880_restart_oce_0970.nc t0ni_00002880_restart_oce_0971.nc t0ni_00002880_restart_oce_0972.nc t0ni_00002880_restart_oce_0973.nc t0ni_00002880_restart_oce_0974.nc t0ni_00002880_restart_oce_0975.nc t0ni_00002880_restart_oce_0976.nc t0ni_00002880_restart_oce_0977.nc t0ni_00002880_restart_oce_0978.nc t0ni_00002880_restart_oce_0979.nc t0ni_00002880_restart_oce_0980.nc t0ni_00002880_restart_oce_0981.nc t0ni_00002880_restart_oce_0982.nc t0ni_00002880_restart_oce_0983.nc t0ni_00002880_restart_oce_0984.nc t0ni_00002880_restart_oce_0985.nc t0ni_00002880_restart_oce_0986.nc t0ni_00002880_restart_oce_0987.nc t0ni_00002880_restart_oce_0988.nc t0ni_00002880_restart_oce_0989.nc t0ni_00002880_restart_oce_0990.nc t0ni_00002880_restart_oce_0991.nc t0ni_00002880_restart_oce_0992.nc t0ni_00002880_restart_oce_0993.nc t0ni_00002880_restart_oce_0994.nc t0ni_00002880_restart_oce_0995.nc t0ni_00002880_restart_oce_0996.nc t0ni_00002880_restart_oce_0997.nc t0ni_00002880_restart_oce_0998.nc t0ni_00002880_restart_oce_0999.nc t0ni_00002880_restart_oce_1000.nc t0ni_00002880_restart_oce_1001.nc t0ni_00002880_restart_oce_1002.nc t0ni_00002880_restart_oce_1003.nc t0ni_00002880_restart_oce_1004.nc t0ni_00002880_restart_oce_1005.nc t0ni_00002880_restart_oce_1006.nc t0ni_00002880_restart_oce_1007.nc t0ni_00002880_restart_oce_1008.nc t0ni_00002880_restart_oce_1009.nc t0ni_00002880_restart_oce_1010.nc t0ni_00002880_restart_oce_1011.nc t0ni_00002880_restart_oce_1012.nc t0ni_00002880_restart_oce_1013.nc t0ni_00002880_restart_oce_1014.nc t0ni_00002880_restart_oce_1015.nc t0ni_00002880_restart_oce_1016.nc t0ni_00002880_restart_oce_1017.nc t0ni_00002880_restart_oce_1018.nc t0ni_00002880_restart_oce_1019.nc t0ni_00002880_restart_oce_1020.nc t0ni_00002880_restart_oce_1021.nc t0ni_00002880_restart_oce_1022.nc t0ni_00002880_restart_oce_1023.nc t0ni_00002880_restart_oce_1024.nc t0ni_00002880_restart_oce_1025.nc t0ni_00002880_restart_oce_1026.nc t0ni_00002880_restart_oce_1027.nc t0ni_00002880_restart_oce_1028.nc t0ni_00002880_restart_oce_1029.nc t0ni_00002880_restart_oce_1030.nc t0ni_00002880_restart_oce_1031.nc t0ni_00002880_restart_oce_1032.nc t0ni_00002880_restart_oce_1033.nc t0ni_00002880_restart_oce_1034.nc t0ni_00002880_restart_oce_1035.nc t0ni_00002880_restart_oce_1036.nc t0ni_00002880_restart_oce_1037.nc t0ni_00002880_restart_oce_1038.nc t0ni_00002880_restart_oce_1039.nc t0ni_00002880_restart_oce_1040.nc t0ni_00002880_restart_oce_1041.nc t0ni_00002880_restart_oce_1042.nc t0ni_00002880_restart_oce_1043.nc t0ni_00002880_restart_oce_1044.nc t0ni_00002880_restart_oce_1045.nc t0ni_00002880_restart_oce_1046.nc t0ni_00002880_restart_oce_1047.nc t0ni_00002880_restart_oce_1048.nc t0ni_00002880_restart_oce_1049.nc t0ni_00002880_restart_oce_1050.nc t0ni_00002880_restart_oce_1051.nc t0ni_00002880_restart_oce_1052.nc t0ni_00002880_restart_oce_1053.nc t0ni_00002880_restart_oce_1054.nc t0ni_00002880_restart_oce_1055.nc t0ni_00002880_restart_oce_1056.nc t0ni_00002880_restart_oce_1057.nc t0ni_00002880_restart_oce_1058.nc t0ni_00002880_restart_oce_1059.nc t0ni_00002880_restart_oce_1060.nc t0ni_00002880_restart_oce_1061.nc t0ni_00002880_restart_oce_1062.nc t0ni_00002880_restart_oce_1063.nc t0ni_00002880_restart_oce_1064.nc t0ni_00002880_restart_oce_1065.nc t0ni_00002880_restart_oce_1066.nc t0ni_00002880_restart_oce_1067.nc t0ni_00002880_restart_oce_1068.nc t0ni_00002880_restart_oce_1069.nc t0ni_00002880_restart_oce_1070.nc t0ni_00002880_restart_oce_1071.nc t0ni_00002880_restart_oce_1072.nc t0ni_00002880_restart_oce_1073.nc t0ni_00002880_restart_oce_1074.nc t0ni_00002880_restart_oce_1075.nc t0ni_00002880_restart_oce_1076.nc t0ni_00002880_restart_oce_1077.nc t0ni_00002880_restart_oce_1078.nc t0ni_00002880_restart_oce_1079.nc t0ni_00002880_restart_oce_1080.nc t0ni_00002880_restart_oce_1081.nc t0ni_00002880_restart_oce_1082.nc t0ni_00002880_restart_oce_1083.nc t0ni_00002880_restart_oce_1084.nc t0ni_00002880_restart_oce_1085.nc t0ni_00002880_restart_oce_1086.nc t0ni_00002880_restart_oce_1087.nc t0ni_00002880_restart_oce_1088.nc t0ni_00002880_restart_oce_1089.nc t0ni_00002880_restart_oce_1090.nc t0ni_00002880_restart_oce_1091.nc t0ni_00002880_restart_oce_1092.nc t0ni_00002880_restart_oce_1093.nc t0ni_00002880_restart_oce_1094.nc t0ni_00002880_restart_oce_1095.nc t0ni_00002880_restart_oce_1096.nc t0ni_00002880_restart_oce_1097.nc t0ni_00002880_restart_oce_1098.nc t0ni_00002880_restart_oce_1099.nc t0ni_00002880_restart_oce_1100.nc t0ni_00002880_restart_oce_1101.nc t0ni_00002880_restart_oce_1102.nc t0ni_00002880_restart_oce_1103.nc t0ni_00002880_restart_oce_1104.nc t0ni_00002880_restart_oce_1105.nc t0ni_00002880_restart_oce_1106.nc t0ni_00002880_restart_oce_1107.nc t0ni_00002880_restart_oce_1108.nc t0ni_00002880_restart_oce_1109.nc t0ni_00002880_restart_oce_1110.nc t0ni_00002880_restart_oce_1111.nc t0ni_00002880_restart_oce_1112.nc t0ni_00002880_restart_oce_1113.nc t0ni_00002880_restart_oce_1114.nc t0ni_00002880_restart_oce_1115.nc t0ni_00002880_restart_oce_1116.nc t0ni_00002880_restart_oce_1117.nc t0ni_00002880_restart_oce_1118.nc t0ni_00002880_restart_oce_1119.nc t0ni_00002880_restart_oce_1120.nc t0ni_00002880_restart_oce_1121.nc t0ni_00002880_restart_oce_1122.nc t0ni_00002880_restart_oce_1123.nc t0ni_00002880_restart_oce_1124.nc t0ni_00002880_restart_oce_1125.nc t0ni_00002880_restart_oce_1126.nc t0ni_00002880_restart_oce_1127.nc t0ni_00002880_restart_oce_1128.nc t0ni_00002880_restart_oce_1129.nc t0ni_00002880_restart_oce_1130.nc t0ni_00002880_restart_oce_1131.nc t0ni_00002880_restart_oce_1132.nc t0ni_00002880_restart_oce_1133.nc t0ni_00002880_restart_oce_1134.nc t0ni_00002880_restart_oce_1135.nc t0ni_00002880_restart_oce_1136.nc t0ni_00002880_restart_oce_1137.nc t0ni_00002880_restart_oce_1138.nc t0ni_00002880_restart_oce_1139.nc t0ni_00002880_restart_oce_1140.nc t0ni_00002880_restart_oce_1141.nc t0ni_00002880_restart_oce_1142.nc t0ni_00002880_restart_oce_1143.nc t0ni_00002880_restart_oce_1144.nc t0ni_00002880_restart_oce_1145.nc t0ni_00002880_restart_oce_1146.nc t0ni_00002880_restart_oce_1147.nc t0ni_00002880_restart_oce_1148.nc t0ni_00002880_restart_oce_1149.nc t0ni_00002880_restart_oce_1150.nc t0ni_00002880_restart_oce_1151.nc t0ni_00002880_restart_oce_1152.nc t0ni_00002880_restart_oce_1153.nc t0ni_00002880_restart_oce_1154.nc t0ni_00002880_restart_oce_1155.nc t0ni_00002880_restart_oce_1156.nc t0ni_00002880_restart_oce_1157.nc t0ni_00002880_restart_oce_1158.nc t0ni_00002880_restart_oce_1159.nc t0ni_00002880_restart_oce_1160.nc t0ni_00002880_restart_oce_1161.nc t0ni_00002880_restart_oce_1162.nc t0ni_00002880_restart_oce_1163.nc t0ni_00002880_restart_oce_1164.nc t0ni_00002880_restart_oce_1165.nc t0ni_00002880_restart_oce_1166.nc t0ni_00002880_restart_oce_1167.nc t0ni_00002880_restart_oce_1168.nc t0ni_00002880_restart_oce_1169.nc t0ni_00002880_restart_oce_1170.nc t0ni_00002880_restart_oce_1171.nc t0ni_00002880_restart_oce_1172.nc t0ni_00002880_restart_oce_1173.nc t0ni_00002880_restart_oce_1174.nc t0ni_00002880_restart_oce_1175.nc t0ni_00002880_restart_oce_1176.nc t0ni_00002880_restart_oce_1177.nc t0ni_00002880_restart_oce_1178.nc t0ni_00002880_restart_oce_1179.nc t0ni_00002880_restart_oce_1180.nc t0ni_00002880_restart_oce_1181.nc t0ni_00002880_restart_oce_1182.nc t0ni_00002880_restart_oce_1183.nc t0ni_00002880_restart_oce_1184.nc t0ni_00002880_restart_oce_1185.nc t0ni_00002880_restart_oce_1186.nc t0ni_00002880_restart_oce_1187.nc t0ni_00002880_restart_oce_1188.nc t0ni_00002880_restart_oce_1189.nc t0ni_00002880_restart_oce_1190.nc t0ni_00002880_restart_oce_1191.nc t0ni_00002880_restart_oce_1192.nc t0ni_00002880_restart_oce_1193.nc t0ni_00002880_restart_oce_1194.nc t0ni_00002880_restart_oce_1195.nc t0ni_00002880_restart_oce_1196.nc t0ni_00002880_restart_oce_1197.nc t0ni_00002880_restart_oce_1198.nc t0ni_00002880_restart_oce_1199.nc t0ni_00002880_restart_oce_1200.nc t0ni_00002880_restart_oce_1201.nc t0ni_00002880_restart_oce_1202.nc t0ni_00002880_restart_oce_1203.nc t0ni_00002880_restart_oce_1204.nc t0ni_00002880_restart_oce_1205.nc t0ni_00002880_restart_oce_1206.nc t0ni_00002880_restart_oce_1207.nc t0ni_00002880_restart_oce_1208.nc t0ni_00002880_restart_oce_1209.nc t0ni_00002880_restart_oce_1210.nc t0ni_00002880_restart_oce_1211.nc t0ni_00002880_restart_oce_1212.nc t0ni_00002880_restart_oce_1213.nc t0ni_00002880_restart_oce_1214.nc t0ni_00002880_restart_oce_1215.nc t0ni_00002880_restart_oce_1216.nc t0ni_00002880_restart_oce_1217.nc t0ni_00002880_restart_oce_1218.nc t0ni_00002880_restart_oce_1219.nc t0ni_00002880_restart_oce_1220.nc t0ni_00002880_restart_oce_1221.nc t0ni_00002880_restart_oce_1222.nc t0ni_00002880_restart_oce_1223.nc t0ni_00002880_restart_oce_1224.nc t0ni_00002880_restart_oce_1225.nc t0ni_00002880_restart_oce_1226.nc t0ni_00002880_restart_oce_1227.nc t0ni_00002880_restart_oce_1228.nc t0ni_00002880_restart_oce_1229.nc t0ni_00002880_restart_oce_1230.nc t0ni_00002880_restart_oce_1231.nc t0ni_00002880_restart_oce_1232.nc t0ni_00002880_restart_oce_1233.nc t0ni_00002880_restart_oce_1234.nc t0ni_00002880_restart_oce_1235.nc t0ni_00002880_restart_oce_1236.nc t0ni_00002880_restart_oce_1237.nc t0ni_00002880_restart_oce_1238.nc t0ni_00002880_restart_oce_1239.nc t0ni_00002880_restart_oce_1240.nc t0ni_00002880_restart_oce_1241.nc t0ni_00002880_restart_oce_1242.nc t0ni_00002880_restart_oce_1243.nc t0ni_00002880_restart_oce_1244.nc t0ni_00002880_restart_oce_1245.nc t0ni_00002880_restart_oce_1246.nc t0ni_00002880_restart_oce_1247.nc t0ni_00002880_restart_oce_1248.nc t0ni_00002880_restart_oce_1249.nc t0ni_00002880_restart_oce_1250.nc t0ni_00002880_restart_oce_1251.nc t0ni_00002880_restart_oce_1252.nc t0ni_00002880_restart_oce_1253.nc t0ni_00002880_restart_oce_1254.nc t0ni_00002880_restart_oce_1255.nc t0ni_00002880_restart_oce_1256.nc t0ni_00002880_restart_oce_1257.nc t0ni_00002880_restart_oce_1258.nc t0ni_00002880_restart_oce_1259.nc t0ni_00002880_restart_oce_1260.nc t0ni_00002880_restart_oce_1261.nc t0ni_00002880_restart_oce_1262.nc t0ni_00002880_restart_oce_1263.nc t0ni_00002880_restart_oce_1264.nc t0ni_00002880_restart_oce_1265.nc t0ni_00002880_restart_oce_1266.nc t0ni_00002880_restart_oce_1267.nc t0ni_00002880_restart_oce_1268.nc t0ni_00002880_restart_oce_1269.nc t0ni_00002880_restart_oce_1270.nc t0ni_00002880_restart_oce_1271.nc t0ni_00002880_restart_oce_1272.nc t0ni_00002880_restart_oce_1273.nc t0ni_00002880_restart_oce_1274.nc t0ni_00002880_restart_oce_1275.nc t0ni_00002880_restart_oce_1276.nc t0ni_00002880_restart_oce_1277.nc t0ni_00002880_restart_oce_1278.nc t0ni_00002880_restart_oce_1279.nc t0ni_00002880_restart_oce_1280.nc t0ni_00002880_restart_oce_1281.nc t0ni_00002880_restart_oce_1282.nc t0ni_00002880_restart_oce_1283.nc t0ni_00002880_restart_oce_1284.nc t0ni_00002880_restart_oce_1285.nc t0ni_00002880_restart_oce_1286.nc t0ni_00002880_restart_oce_1287.nc t0ni_00002880_restart_oce_1288.nc t0ni_00002880_restart_oce_1289.nc t0ni_00002880_restart_oce_1290.nc t0ni_00002880_restart_oce_1291.nc t0ni_00002880_restart_oce_1292.nc t0ni_00002880_restart_oce_1293.nc t0ni_00002880_restart_oce_1294.nc t0ni_00002880_restart_oce_1295.nc t0ni_00002880_restart_oce_1296.nc t0ni_00002880_restart_oce_1297.nc t0ni_00002880_restart_oce_1298.nc t0ni_00002880_restart_oce_1299.nc t0ni_00002880_restart_oce_1300.nc t0ni_00002880_restart_oce_1301.nc t0ni_00002880_restart_oce_1302.nc t0ni_00002880_restart_oce_1303.nc t0ni_00002880_restart_oce_1304.nc t0ni_00002880_restart_oce_1305.nc t0ni_00002880_restart_oce_1306.nc t0ni_00002880_restart_oce_1307.nc t0ni_00002880_restart_oce_1308.nc t0ni_00002880_restart_oce_1309.nc t0ni_00002880_restart_oce_1310.nc t0ni_00002880_restart_oce_1311.nc t0ni_00002880_restart_oce_1312.nc t0ni_00002880_restart_oce_1313.nc t0ni_00002880_restart_oce_1314.nc t0ni_00002880_restart_oce_1315.nc t0ni_00002880_restart_oce_1316.nc t0ni_00002880_restart_oce_1317.nc t0ni_00002880_restart_oce_1318.nc t0ni_00002880_restart_oce_1319.nc t0ni_00002880_restart_oce_1320.nc t0ni_00002880_restart_oce_1321.nc t0ni_00002880_restart_oce_1322.nc t0ni_00002880_restart_oce_1323.nc t0ni_00002880_restart_oce_1324.nc t0ni_00002880_restart_oce_1325.nc t0ni_00002880_restart_oce_1326.nc t0ni_00002880_restart_oce_1327.nc t0ni_00002880_restart_oce_1328.nc t0ni_00002880_restart_oce_1329.nc t0ni_00002880_restart_oce_1330.nc t0ni_00002880_restart_oce_1331.nc t0ni_00002880_restart_oce_1332.nc t0ni_00002880_restart_oce_1333.nc t0ni_00002880_restart_oce_1334.nc t0ni_00002880_restart_oce_1335.nc t0ni_00002880_restart_oce_1336.nc t0ni_00002880_restart_oce_1337.nc t0ni_00002880_restart_oce_1338.nc t0ni_00002880_restart_oce_1339.nc t0ni_00002880_restart_oce_1340.nc t0ni_00002880_restart_oce_1341.nc t0ni_00002880_restart_oce_1342.nc t0ni_00002880_restart_oce_1343.nc t0ni_00002880_restart_oce_1344.nc t0ni_00002880_restart_oce_1345.nc t0ni_00002880_restart_oce_1346.nc t0ni_00002880_restart_oce_1347.nc t0ni_00002880_restart_oce_1348.nc t0ni_00002880_restart_oce_1349.nc t0ni_00002880_restart_oce_1350.nc t0ni_00002880_restart_oce_1351.nc t0ni_00002880_restart_oce_1352.nc t0ni_00002880_restart_oce_1353.nc t0ni_00002880_restart_oce_1354.nc t0ni_00002880_restart_oce_1355.nc t0ni_00002880_restart_oce_1356.nc t0ni_00002880_restart_oce_1357.nc t0ni_00002880_restart_oce_1358.nc t0ni_00002880_restart_oce_1359.nc t0ni_00002880_restart_oce_1360.nc t0ni_00002880_restart_oce_1361.nc t0ni_00002880_restart_oce_1362.nc t0ni_00002880_restart_oce_1363.nc t0ni_00002880_restart_oce_1364.nc t0ni_00002880_restart_oce_1365.nc t0ni_00002880_restart_oce_1366.nc t0ni_00002880_restart_oce_1367.nc t0ni_00002880_restart_oce_1368.nc t0ni_00002880_restart_oce_1369.nc t0ni_00002880_restart_oce_1370.nc t0ni_00002880_restart_oce_1371.nc t0ni_00002880_restart_oce_1372.nc t0ni_00002880_restart_oce_1373.nc t0ni_00002880_restart_oce_1374.nc t0ni_00002880_restart_oce_1375.nc t0ni_00002880_restart_oce_1376.nc t0ni_00002880_restart_oce_1377.nc t0ni_00002880_restart_oce_1378.nc t0ni_00002880_restart_oce_1379.nc t0ni_00002880_restart_oce_1380.nc t0ni_00002880_restart_oce_1381.nc t0ni_00002880_restart_oce_1382.nc t0ni_00002880_restart_oce_1383.nc t0ni_00002880_restart_oce_1384.nc t0ni_00002880_restart_oce_1385.nc t0ni_00002880_restart_oce_1386.nc t0ni_00002880_restart_oce_1387.nc t0ni_00002880_restart_oce_1388.nc t0ni_00002880_restart_oce_1389.nc t0ni_00002880_restart_oce_1390.nc ../delete
+ (( j++ ))
+ (( j < 2 ))
+ filebase=t0ni_00002880_restart_ice
+ ofile=t0ni_fc0_19931130_restart_ice.nc.gz
+ odir=/gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/ice/ORCA025_LIM3/t0ni
+ ifiles=(${filebase}_*)
+ ndomain=1391
+ '[' 1391 -gt 1 ']'
+ rm -f /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/nemo/t0ni_00002880_restart_ice_ok
+ echo found 1391 files matching pattern t0ni_00002880_restart_ice
+ rm -f t0ni_00002880_restart_ice.nc t0ni_00002880_restart_ice.nc.gz /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/nemo/t0ni_00002880_restart_ice_ok
+ rebuild_nemo -t48 t0ni_00002880_restart_ice 1391
+ rm -f nam_rebuild
+ gzip t0ni_00002880_restart_ice.nc
+ rm -rf /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/ice/ORCA025_LIM3/t0ni
+ mkdir -p /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/ice/ORCA025_LIM3/t0ni
+ mv t0ni_00002880_restart_ice.nc.gz /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/ice/ORCA025_LIM3/t0ni/t0ni_fc0_19931130_restart_ice.nc.gz
+ touch /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/nemo/t0ni_00002880_restart_ice_ok
+ mv t0ni_00002880_restart_ice_0000.nc t0ni_00002880_restart_ice_0001.nc t0ni_00002880_restart_ice_0002.nc t0ni_00002880_restart_ice_0003.nc t0ni_00002880_restart_ice_0004.nc t0ni_00002880_restart_ice_0005.nc t0ni_00002880_restart_ice_0006.nc t0ni_00002880_restart_ice_0007.nc t0ni_00002880_restart_ice_0008.nc t0ni_00002880_restart_ice_0009.nc t0ni_00002880_restart_ice_0010.nc t0ni_00002880_restart_ice_0011.nc t0ni_00002880_restart_ice_0012.nc t0ni_00002880_restart_ice_0013.nc t0ni_00002880_restart_ice_0014.nc t0ni_00002880_restart_ice_0015.nc t0ni_00002880_restart_ice_0016.nc t0ni_00002880_restart_ice_0017.nc t0ni_00002880_restart_ice_0018.nc t0ni_00002880_restart_ice_0019.nc t0ni_00002880_restart_ice_0020.nc t0ni_00002880_restart_ice_0021.nc t0ni_00002880_restart_ice_0022.nc t0ni_00002880_restart_ice_0023.nc t0ni_00002880_restart_ice_0024.nc t0ni_00002880_restart_ice_0025.nc t0ni_00002880_restart_ice_0026.nc t0ni_00002880_restart_ice_0027.nc t0ni_00002880_restart_ice_0028.nc t0ni_00002880_restart_ice_0029.nc t0ni_00002880_restart_ice_0030.nc t0ni_00002880_restart_ice_0031.nc t0ni_00002880_restart_ice_0032.nc t0ni_00002880_restart_ice_0033.nc t0ni_00002880_restart_ice_0034.nc t0ni_00002880_restart_ice_0035.nc t0ni_00002880_restart_ice_0036.nc t0ni_00002880_restart_ice_0037.nc t0ni_00002880_restart_ice_0038.nc t0ni_00002880_restart_ice_0039.nc t0ni_00002880_restart_ice_0040.nc t0ni_00002880_restart_ice_0041.nc t0ni_00002880_restart_ice_0042.nc t0ni_00002880_restart_ice_0043.nc t0ni_00002880_restart_ice_0044.nc t0ni_00002880_restart_ice_0045.nc t0ni_00002880_restart_ice_0046.nc t0ni_00002880_restart_ice_0047.nc t0ni_00002880_restart_ice_0048.nc t0ni_00002880_restart_ice_0049.nc t0ni_00002880_restart_ice_0050.nc t0ni_00002880_restart_ice_0051.nc t0ni_00002880_restart_ice_0052.nc t0ni_00002880_restart_ice_0053.nc t0ni_00002880_restart_ice_0054.nc t0ni_00002880_restart_ice_0055.nc t0ni_00002880_restart_ice_0056.nc t0ni_00002880_restart_ice_0057.nc t0ni_00002880_restart_ice_0058.nc t0ni_00002880_restart_ice_0059.nc t0ni_00002880_restart_ice_0060.nc t0ni_00002880_restart_ice_0061.nc t0ni_00002880_restart_ice_0062.nc t0ni_00002880_restart_ice_0063.nc t0ni_00002880_restart_ice_0064.nc t0ni_00002880_restart_ice_0065.nc t0ni_00002880_restart_ice_0066.nc t0ni_00002880_restart_ice_0067.nc t0ni_00002880_restart_ice_0068.nc t0ni_00002880_restart_ice_0069.nc t0ni_00002880_restart_ice_0070.nc t0ni_00002880_restart_ice_0071.nc t0ni_00002880_restart_ice_0072.nc t0ni_00002880_restart_ice_0073.nc t0ni_00002880_restart_ice_0074.nc t0ni_00002880_restart_ice_0075.nc t0ni_00002880_restart_ice_0076.nc t0ni_00002880_restart_ice_0077.nc t0ni_00002880_restart_ice_0078.nc t0ni_00002880_restart_ice_0079.nc t0ni_00002880_restart_ice_0080.nc t0ni_00002880_restart_ice_0081.nc t0ni_00002880_restart_ice_0082.nc t0ni_00002880_restart_ice_0083.nc t0ni_00002880_restart_ice_0084.nc t0ni_00002880_restart_ice_0085.nc t0ni_00002880_restart_ice_0086.nc t0ni_00002880_restart_ice_0087.nc t0ni_00002880_restart_ice_0088.nc t0ni_00002880_restart_ice_0089.nc t0ni_00002880_restart_ice_0090.nc t0ni_00002880_restart_ice_0091.nc t0ni_00002880_restart_ice_0092.nc t0ni_00002880_restart_ice_0093.nc t0ni_00002880_restart_ice_0094.nc t0ni_00002880_restart_ice_0095.nc t0ni_00002880_restart_ice_0096.nc t0ni_00002880_restart_ice_0097.nc t0ni_00002880_restart_ice_0098.nc t0ni_00002880_restart_ice_0099.nc t0ni_00002880_restart_ice_0100.nc t0ni_00002880_restart_ice_0101.nc t0ni_00002880_restart_ice_0102.nc t0ni_00002880_restart_ice_0103.nc t0ni_00002880_restart_ice_0104.nc t0ni_00002880_restart_ice_0105.nc t0ni_00002880_restart_ice_0106.nc t0ni_00002880_restart_ice_0107.nc t0ni_00002880_restart_ice_0108.nc t0ni_00002880_restart_ice_0109.nc t0ni_00002880_restart_ice_0110.nc t0ni_00002880_restart_ice_0111.nc t0ni_00002880_restart_ice_0112.nc t0ni_00002880_restart_ice_0113.nc t0ni_00002880_restart_ice_0114.nc t0ni_00002880_restart_ice_0115.nc t0ni_00002880_restart_ice_0116.nc t0ni_00002880_restart_ice_0117.nc t0ni_00002880_restart_ice_0118.nc t0ni_00002880_restart_ice_0119.nc t0ni_00002880_restart_ice_0120.nc t0ni_00002880_restart_ice_0121.nc t0ni_00002880_restart_ice_0122.nc t0ni_00002880_restart_ice_0123.nc t0ni_00002880_restart_ice_0124.nc t0ni_00002880_restart_ice_0125.nc t0ni_00002880_restart_ice_0126.nc t0ni_00002880_restart_ice_0127.nc t0ni_00002880_restart_ice_0128.nc t0ni_00002880_restart_ice_0129.nc t0ni_00002880_restart_ice_0130.nc t0ni_00002880_restart_ice_0131.nc t0ni_00002880_restart_ice_0132.nc t0ni_00002880_restart_ice_0133.nc t0ni_00002880_restart_ice_0134.nc t0ni_00002880_restart_ice_0135.nc t0ni_00002880_restart_ice_0136.nc t0ni_00002880_restart_ice_0137.nc t0ni_00002880_restart_ice_0138.nc t0ni_00002880_restart_ice_0139.nc t0ni_00002880_restart_ice_0140.nc t0ni_00002880_restart_ice_0141.nc t0ni_00002880_restart_ice_0142.nc t0ni_00002880_restart_ice_0143.nc t0ni_00002880_restart_ice_0144.nc t0ni_00002880_restart_ice_0145.nc t0ni_00002880_restart_ice_0146.nc t0ni_00002880_restart_ice_0147.nc t0ni_00002880_restart_ice_0148.nc t0ni_00002880_restart_ice_0149.nc t0ni_00002880_restart_ice_0150.nc t0ni_00002880_restart_ice_0151.nc t0ni_00002880_restart_ice_0152.nc t0ni_00002880_restart_ice_0153.nc t0ni_00002880_restart_ice_0154.nc t0ni_00002880_restart_ice_0155.nc t0ni_00002880_restart_ice_0156.nc t0ni_00002880_restart_ice_0157.nc t0ni_00002880_restart_ice_0158.nc t0ni_00002880_restart_ice_0159.nc t0ni_00002880_restart_ice_0160.nc t0ni_00002880_restart_ice_0161.nc t0ni_00002880_restart_ice_0162.nc t0ni_00002880_restart_ice_0163.nc t0ni_00002880_restart_ice_0164.nc t0ni_00002880_restart_ice_0165.nc t0ni_00002880_restart_ice_0166.nc t0ni_00002880_restart_ice_0167.nc t0ni_00002880_restart_ice_0168.nc t0ni_00002880_restart_ice_0169.nc t0ni_00002880_restart_ice_0170.nc t0ni_00002880_restart_ice_0171.nc t0ni_00002880_restart_ice_0172.nc t0ni_00002880_restart_ice_0173.nc t0ni_00002880_restart_ice_0174.nc t0ni_00002880_restart_ice_0175.nc t0ni_00002880_restart_ice_0176.nc t0ni_00002880_restart_ice_0177.nc t0ni_00002880_restart_ice_0178.nc t0ni_00002880_restart_ice_0179.nc t0ni_00002880_restart_ice_0180.nc t0ni_00002880_restart_ice_0181.nc t0ni_00002880_restart_ice_0182.nc t0ni_00002880_restart_ice_0183.nc t0ni_00002880_restart_ice_0184.nc t0ni_00002880_restart_ice_0185.nc t0ni_00002880_restart_ice_0186.nc t0ni_00002880_restart_ice_0187.nc t0ni_00002880_restart_ice_0188.nc t0ni_00002880_restart_ice_0189.nc t0ni_00002880_restart_ice_0190.nc t0ni_00002880_restart_ice_0191.nc t0ni_00002880_restart_ice_0192.nc t0ni_00002880_restart_ice_0193.nc t0ni_00002880_restart_ice_0194.nc t0ni_00002880_restart_ice_0195.nc t0ni_00002880_restart_ice_0196.nc t0ni_00002880_restart_ice_0197.nc t0ni_00002880_restart_ice_0198.nc t0ni_00002880_restart_ice_0199.nc t0ni_00002880_restart_ice_0200.nc t0ni_00002880_restart_ice_0201.nc t0ni_00002880_restart_ice_0202.nc t0ni_00002880_restart_ice_0203.nc t0ni_00002880_restart_ice_0204.nc t0ni_00002880_restart_ice_0205.nc t0ni_00002880_restart_ice_0206.nc t0ni_00002880_restart_ice_0207.nc t0ni_00002880_restart_ice_0208.nc t0ni_00002880_restart_ice_0209.nc t0ni_00002880_restart_ice_0210.nc t0ni_00002880_restart_ice_0211.nc t0ni_00002880_restart_ice_0212.nc t0ni_00002880_restart_ice_0213.nc t0ni_00002880_restart_ice_0214.nc t0ni_00002880_restart_ice_0215.nc t0ni_00002880_restart_ice_0216.nc t0ni_00002880_restart_ice_0217.nc t0ni_00002880_restart_ice_0218.nc t0ni_00002880_restart_ice_0219.nc t0ni_00002880_restart_ice_0220.nc t0ni_00002880_restart_ice_0221.nc t0ni_00002880_restart_ice_0222.nc t0ni_00002880_restart_ice_0223.nc t0ni_00002880_restart_ice_0224.nc t0ni_00002880_restart_ice_0225.nc t0ni_00002880_restart_ice_0226.nc t0ni_00002880_restart_ice_0227.nc t0ni_00002880_restart_ice_0228.nc t0ni_00002880_restart_ice_0229.nc t0ni_00002880_restart_ice_0230.nc t0ni_00002880_restart_ice_0231.nc t0ni_00002880_restart_ice_0232.nc t0ni_00002880_restart_ice_0233.nc t0ni_00002880_restart_ice_0234.nc t0ni_00002880_restart_ice_0235.nc t0ni_00002880_restart_ice_0236.nc t0ni_00002880_restart_ice_0237.nc t0ni_00002880_restart_ice_0238.nc t0ni_00002880_restart_ice_0239.nc t0ni_00002880_restart_ice_0240.nc t0ni_00002880_restart_ice_0241.nc t0ni_00002880_restart_ice_0242.nc t0ni_00002880_restart_ice_0243.nc t0ni_00002880_restart_ice_0244.nc t0ni_00002880_restart_ice_0245.nc t0ni_00002880_restart_ice_0246.nc t0ni_00002880_restart_ice_0247.nc t0ni_00002880_restart_ice_0248.nc t0ni_00002880_restart_ice_0249.nc t0ni_00002880_restart_ice_0250.nc t0ni_00002880_restart_ice_0251.nc t0ni_00002880_restart_ice_0252.nc t0ni_00002880_restart_ice_0253.nc t0ni_00002880_restart_ice_0254.nc t0ni_00002880_restart_ice_0255.nc t0ni_00002880_restart_ice_0256.nc t0ni_00002880_restart_ice_0257.nc t0ni_00002880_restart_ice_0258.nc t0ni_00002880_restart_ice_0259.nc t0ni_00002880_restart_ice_0260.nc t0ni_00002880_restart_ice_0261.nc t0ni_00002880_restart_ice_0262.nc t0ni_00002880_restart_ice_0263.nc t0ni_00002880_restart_ice_0264.nc t0ni_00002880_restart_ice_0265.nc t0ni_00002880_restart_ice_0266.nc t0ni_00002880_restart_ice_0267.nc t0ni_00002880_restart_ice_0268.nc t0ni_00002880_restart_ice_0269.nc t0ni_00002880_restart_ice_0270.nc t0ni_00002880_restart_ice_0271.nc t0ni_00002880_restart_ice_0272.nc t0ni_00002880_restart_ice_0273.nc t0ni_00002880_restart_ice_0274.nc t0ni_00002880_restart_ice_0275.nc t0ni_00002880_restart_ice_0276.nc t0ni_00002880_restart_ice_0277.nc t0ni_00002880_restart_ice_0278.nc t0ni_00002880_restart_ice_0279.nc t0ni_00002880_restart_ice_0280.nc t0ni_00002880_restart_ice_0281.nc t0ni_00002880_restart_ice_0282.nc t0ni_00002880_restart_ice_0283.nc t0ni_00002880_restart_ice_0284.nc t0ni_00002880_restart_ice_0285.nc t0ni_00002880_restart_ice_0286.nc t0ni_00002880_restart_ice_0287.nc t0ni_00002880_restart_ice_0288.nc t0ni_00002880_restart_ice_0289.nc t0ni_00002880_restart_ice_0290.nc t0ni_00002880_restart_ice_0291.nc t0ni_00002880_restart_ice_0292.nc t0ni_00002880_restart_ice_0293.nc t0ni_00002880_restart_ice_0294.nc t0ni_00002880_restart_ice_0295.nc t0ni_00002880_restart_ice_0296.nc t0ni_00002880_restart_ice_0297.nc t0ni_00002880_restart_ice_0298.nc t0ni_00002880_restart_ice_0299.nc t0ni_00002880_restart_ice_0300.nc t0ni_00002880_restart_ice_0301.nc t0ni_00002880_restart_ice_0302.nc t0ni_00002880_restart_ice_0303.nc t0ni_00002880_restart_ice_0304.nc t0ni_00002880_restart_ice_0305.nc t0ni_00002880_restart_ice_0306.nc t0ni_00002880_restart_ice_0307.nc t0ni_00002880_restart_ice_0308.nc t0ni_00002880_restart_ice_0309.nc t0ni_00002880_restart_ice_0310.nc t0ni_00002880_restart_ice_0311.nc t0ni_00002880_restart_ice_0312.nc t0ni_00002880_restart_ice_0313.nc t0ni_00002880_restart_ice_0314.nc t0ni_00002880_restart_ice_0315.nc t0ni_00002880_restart_ice_0316.nc t0ni_00002880_restart_ice_0317.nc t0ni_00002880_restart_ice_0318.nc t0ni_00002880_restart_ice_0319.nc t0ni_00002880_restart_ice_0320.nc t0ni_00002880_restart_ice_0321.nc t0ni_00002880_restart_ice_0322.nc t0ni_00002880_restart_ice_0323.nc t0ni_00002880_restart_ice_0324.nc t0ni_00002880_restart_ice_0325.nc t0ni_00002880_restart_ice_0326.nc t0ni_00002880_restart_ice_0327.nc t0ni_00002880_restart_ice_0328.nc t0ni_00002880_restart_ice_0329.nc t0ni_00002880_restart_ice_0330.nc t0ni_00002880_restart_ice_0331.nc t0ni_00002880_restart_ice_0332.nc t0ni_00002880_restart_ice_0333.nc t0ni_00002880_restart_ice_0334.nc t0ni_00002880_restart_ice_0335.nc t0ni_00002880_restart_ice_0336.nc t0ni_00002880_restart_ice_0337.nc t0ni_00002880_restart_ice_0338.nc t0ni_00002880_restart_ice_0339.nc t0ni_00002880_restart_ice_0340.nc t0ni_00002880_restart_ice_0341.nc t0ni_00002880_restart_ice_0342.nc t0ni_00002880_restart_ice_0343.nc t0ni_00002880_restart_ice_0344.nc t0ni_00002880_restart_ice_0345.nc t0ni_00002880_restart_ice_0346.nc t0ni_00002880_restart_ice_0347.nc t0ni_00002880_restart_ice_0348.nc t0ni_00002880_restart_ice_0349.nc t0ni_00002880_restart_ice_0350.nc t0ni_00002880_restart_ice_0351.nc t0ni_00002880_restart_ice_0352.nc t0ni_00002880_restart_ice_0353.nc t0ni_00002880_restart_ice_0354.nc t0ni_00002880_restart_ice_0355.nc t0ni_00002880_restart_ice_0356.nc t0ni_00002880_restart_ice_0357.nc t0ni_00002880_restart_ice_0358.nc t0ni_00002880_restart_ice_0359.nc t0ni_00002880_restart_ice_0360.nc t0ni_00002880_restart_ice_0361.nc t0ni_00002880_restart_ice_0362.nc t0ni_00002880_restart_ice_0363.nc t0ni_00002880_restart_ice_0364.nc t0ni_00002880_restart_ice_0365.nc t0ni_00002880_restart_ice_0366.nc t0ni_00002880_restart_ice_0367.nc t0ni_00002880_restart_ice_0368.nc t0ni_00002880_restart_ice_0369.nc t0ni_00002880_restart_ice_0370.nc t0ni_00002880_restart_ice_0371.nc t0ni_00002880_restart_ice_0372.nc t0ni_00002880_restart_ice_0373.nc t0ni_00002880_restart_ice_0374.nc t0ni_00002880_restart_ice_0375.nc t0ni_00002880_restart_ice_0376.nc t0ni_00002880_restart_ice_0377.nc t0ni_00002880_restart_ice_0378.nc t0ni_00002880_restart_ice_0379.nc t0ni_00002880_restart_ice_0380.nc t0ni_00002880_restart_ice_0381.nc t0ni_00002880_restart_ice_0382.nc t0ni_00002880_restart_ice_0383.nc t0ni_00002880_restart_ice_0384.nc t0ni_00002880_restart_ice_0385.nc t0ni_00002880_restart_ice_0386.nc t0ni_00002880_restart_ice_0387.nc t0ni_00002880_restart_ice_0388.nc t0ni_00002880_restart_ice_0389.nc t0ni_00002880_restart_ice_0390.nc t0ni_00002880_restart_ice_0391.nc t0ni_00002880_restart_ice_0392.nc t0ni_00002880_restart_ice_0393.nc t0ni_00002880_restart_ice_0394.nc t0ni_00002880_restart_ice_0395.nc t0ni_00002880_restart_ice_0396.nc t0ni_00002880_restart_ice_0397.nc t0ni_00002880_restart_ice_0398.nc t0ni_00002880_restart_ice_0399.nc t0ni_00002880_restart_ice_0400.nc t0ni_00002880_restart_ice_0401.nc t0ni_00002880_restart_ice_0402.nc t0ni_00002880_restart_ice_0403.nc t0ni_00002880_restart_ice_0404.nc t0ni_00002880_restart_ice_0405.nc t0ni_00002880_restart_ice_0406.nc t0ni_00002880_restart_ice_0407.nc t0ni_00002880_restart_ice_0408.nc t0ni_00002880_restart_ice_0409.nc t0ni_00002880_restart_ice_0410.nc t0ni_00002880_restart_ice_0411.nc t0ni_00002880_restart_ice_0412.nc t0ni_00002880_restart_ice_0413.nc t0ni_00002880_restart_ice_0414.nc t0ni_00002880_restart_ice_0415.nc t0ni_00002880_restart_ice_0416.nc t0ni_00002880_restart_ice_0417.nc t0ni_00002880_restart_ice_0418.nc t0ni_00002880_restart_ice_0419.nc t0ni_00002880_restart_ice_0420.nc t0ni_00002880_restart_ice_0421.nc t0ni_00002880_restart_ice_0422.nc t0ni_00002880_restart_ice_0423.nc t0ni_00002880_restart_ice_0424.nc t0ni_00002880_restart_ice_0425.nc t0ni_00002880_restart_ice_0426.nc t0ni_00002880_restart_ice_0427.nc t0ni_00002880_restart_ice_0428.nc t0ni_00002880_restart_ice_0429.nc t0ni_00002880_restart_ice_0430.nc t0ni_00002880_restart_ice_0431.nc t0ni_00002880_restart_ice_0432.nc t0ni_00002880_restart_ice_0433.nc t0ni_00002880_restart_ice_0434.nc t0ni_00002880_restart_ice_0435.nc t0ni_00002880_restart_ice_0436.nc t0ni_00002880_restart_ice_0437.nc t0ni_00002880_restart_ice_0438.nc t0ni_00002880_restart_ice_0439.nc t0ni_00002880_restart_ice_0440.nc t0ni_00002880_restart_ice_0441.nc t0ni_00002880_restart_ice_0442.nc t0ni_00002880_restart_ice_0443.nc t0ni_00002880_restart_ice_0444.nc t0ni_00002880_restart_ice_0445.nc t0ni_00002880_restart_ice_0446.nc t0ni_00002880_restart_ice_0447.nc t0ni_00002880_restart_ice_0448.nc t0ni_00002880_restart_ice_0449.nc t0ni_00002880_restart_ice_0450.nc t0ni_00002880_restart_ice_0451.nc t0ni_00002880_restart_ice_0452.nc t0ni_00002880_restart_ice_0453.nc t0ni_00002880_restart_ice_0454.nc t0ni_00002880_restart_ice_0455.nc t0ni_00002880_restart_ice_0456.nc t0ni_00002880_restart_ice_0457.nc t0ni_00002880_restart_ice_0458.nc t0ni_00002880_restart_ice_0459.nc t0ni_00002880_restart_ice_0460.nc t0ni_00002880_restart_ice_0461.nc t0ni_00002880_restart_ice_0462.nc t0ni_00002880_restart_ice_0463.nc t0ni_00002880_restart_ice_0464.nc t0ni_00002880_restart_ice_0465.nc t0ni_00002880_restart_ice_0466.nc t0ni_00002880_restart_ice_0467.nc t0ni_00002880_restart_ice_0468.nc t0ni_00002880_restart_ice_0469.nc t0ni_00002880_restart_ice_0470.nc t0ni_00002880_restart_ice_0471.nc t0ni_00002880_restart_ice_0472.nc t0ni_00002880_restart_ice_0473.nc t0ni_00002880_restart_ice_0474.nc t0ni_00002880_restart_ice_0475.nc t0ni_00002880_restart_ice_0476.nc t0ni_00002880_restart_ice_0477.nc t0ni_00002880_restart_ice_0478.nc t0ni_00002880_restart_ice_0479.nc t0ni_00002880_restart_ice_0480.nc t0ni_00002880_restart_ice_0481.nc t0ni_00002880_restart_ice_0482.nc t0ni_00002880_restart_ice_0483.nc t0ni_00002880_restart_ice_0484.nc t0ni_00002880_restart_ice_0485.nc t0ni_00002880_restart_ice_0486.nc t0ni_00002880_restart_ice_0487.nc t0ni_00002880_restart_ice_0488.nc t0ni_00002880_restart_ice_0489.nc t0ni_00002880_restart_ice_0490.nc t0ni_00002880_restart_ice_0491.nc t0ni_00002880_restart_ice_0492.nc t0ni_00002880_restart_ice_0493.nc t0ni_00002880_restart_ice_0494.nc t0ni_00002880_restart_ice_0495.nc t0ni_00002880_restart_ice_0496.nc t0ni_00002880_restart_ice_0497.nc t0ni_00002880_restart_ice_0498.nc t0ni_00002880_restart_ice_0499.nc t0ni_00002880_restart_ice_0500.nc t0ni_00002880_restart_ice_0501.nc t0ni_00002880_restart_ice_0502.nc t0ni_00002880_restart_ice_0503.nc t0ni_00002880_restart_ice_0504.nc t0ni_00002880_restart_ice_0505.nc t0ni_00002880_restart_ice_0506.nc t0ni_00002880_restart_ice_0507.nc t0ni_00002880_restart_ice_0508.nc t0ni_00002880_restart_ice_0509.nc t0ni_00002880_restart_ice_0510.nc t0ni_00002880_restart_ice_0511.nc t0ni_00002880_restart_ice_0512.nc t0ni_00002880_restart_ice_0513.nc t0ni_00002880_restart_ice_0514.nc t0ni_00002880_restart_ice_0515.nc t0ni_00002880_restart_ice_0516.nc t0ni_00002880_restart_ice_0517.nc t0ni_00002880_restart_ice_0518.nc t0ni_00002880_restart_ice_0519.nc t0ni_00002880_restart_ice_0520.nc t0ni_00002880_restart_ice_0521.nc t0ni_00002880_restart_ice_0522.nc t0ni_00002880_restart_ice_0523.nc t0ni_00002880_restart_ice_0524.nc t0ni_00002880_restart_ice_0525.nc t0ni_00002880_restart_ice_0526.nc t0ni_00002880_restart_ice_0527.nc t0ni_00002880_restart_ice_0528.nc t0ni_00002880_restart_ice_0529.nc t0ni_00002880_restart_ice_0530.nc t0ni_00002880_restart_ice_0531.nc t0ni_00002880_restart_ice_0532.nc t0ni_00002880_restart_ice_0533.nc t0ni_00002880_restart_ice_0534.nc t0ni_00002880_restart_ice_0535.nc t0ni_00002880_restart_ice_0536.nc t0ni_00002880_restart_ice_0537.nc t0ni_00002880_restart_ice_0538.nc t0ni_00002880_restart_ice_0539.nc t0ni_00002880_restart_ice_0540.nc t0ni_00002880_restart_ice_0541.nc t0ni_00002880_restart_ice_0542.nc t0ni_00002880_restart_ice_0543.nc t0ni_00002880_restart_ice_0544.nc t0ni_00002880_restart_ice_0545.nc t0ni_00002880_restart_ice_0546.nc t0ni_00002880_restart_ice_0547.nc t0ni_00002880_restart_ice_0548.nc t0ni_00002880_restart_ice_0549.nc t0ni_00002880_restart_ice_0550.nc t0ni_00002880_restart_ice_0551.nc t0ni_00002880_restart_ice_0552.nc t0ni_00002880_restart_ice_0553.nc t0ni_00002880_restart_ice_0554.nc t0ni_00002880_restart_ice_0555.nc t0ni_00002880_restart_ice_0556.nc t0ni_00002880_restart_ice_0557.nc t0ni_00002880_restart_ice_0558.nc t0ni_00002880_restart_ice_0559.nc t0ni_00002880_restart_ice_0560.nc t0ni_00002880_restart_ice_0561.nc t0ni_00002880_restart_ice_0562.nc t0ni_00002880_restart_ice_0563.nc t0ni_00002880_restart_ice_0564.nc t0ni_00002880_restart_ice_0565.nc t0ni_00002880_restart_ice_0566.nc t0ni_00002880_restart_ice_0567.nc t0ni_00002880_restart_ice_0568.nc t0ni_00002880_restart_ice_0569.nc t0ni_00002880_restart_ice_0570.nc t0ni_00002880_restart_ice_0571.nc t0ni_00002880_restart_ice_0572.nc t0ni_00002880_restart_ice_0573.nc t0ni_00002880_restart_ice_0574.nc t0ni_00002880_restart_ice_0575.nc t0ni_00002880_restart_ice_0576.nc t0ni_00002880_restart_ice_0577.nc t0ni_00002880_restart_ice_0578.nc t0ni_00002880_restart_ice_0579.nc t0ni_00002880_restart_ice_0580.nc t0ni_00002880_restart_ice_0581.nc t0ni_00002880_restart_ice_0582.nc t0ni_00002880_restart_ice_0583.nc t0ni_00002880_restart_ice_0584.nc t0ni_00002880_restart_ice_0585.nc t0ni_00002880_restart_ice_0586.nc t0ni_00002880_restart_ice_0587.nc t0ni_00002880_restart_ice_0588.nc t0ni_00002880_restart_ice_0589.nc t0ni_00002880_restart_ice_0590.nc t0ni_00002880_restart_ice_0591.nc t0ni_00002880_restart_ice_0592.nc t0ni_00002880_restart_ice_0593.nc t0ni_00002880_restart_ice_0594.nc t0ni_00002880_restart_ice_0595.nc t0ni_00002880_restart_ice_0596.nc t0ni_00002880_restart_ice_0597.nc t0ni_00002880_restart_ice_0598.nc t0ni_00002880_restart_ice_0599.nc t0ni_00002880_restart_ice_0600.nc t0ni_00002880_restart_ice_0601.nc t0ni_00002880_restart_ice_0602.nc t0ni_00002880_restart_ice_0603.nc t0ni_00002880_restart_ice_0604.nc t0ni_00002880_restart_ice_0605.nc t0ni_00002880_restart_ice_0606.nc t0ni_00002880_restart_ice_0607.nc t0ni_00002880_restart_ice_0608.nc t0ni_00002880_restart_ice_0609.nc t0ni_00002880_restart_ice_0610.nc t0ni_00002880_restart_ice_0611.nc t0ni_00002880_restart_ice_0612.nc t0ni_00002880_restart_ice_0613.nc t0ni_00002880_restart_ice_0614.nc t0ni_00002880_restart_ice_0615.nc t0ni_00002880_restart_ice_0616.nc t0ni_00002880_restart_ice_0617.nc t0ni_00002880_restart_ice_0618.nc t0ni_00002880_restart_ice_0619.nc t0ni_00002880_restart_ice_0620.nc t0ni_00002880_restart_ice_0621.nc t0ni_00002880_restart_ice_0622.nc t0ni_00002880_restart_ice_0623.nc t0ni_00002880_restart_ice_0624.nc t0ni_00002880_restart_ice_0625.nc t0ni_00002880_restart_ice_0626.nc t0ni_00002880_restart_ice_0627.nc t0ni_00002880_restart_ice_0628.nc t0ni_00002880_restart_ice_0629.nc t0ni_00002880_restart_ice_0630.nc t0ni_00002880_restart_ice_0631.nc t0ni_00002880_restart_ice_0632.nc t0ni_00002880_restart_ice_0633.nc t0ni_00002880_restart_ice_0634.nc t0ni_00002880_restart_ice_0635.nc t0ni_00002880_restart_ice_0636.nc t0ni_00002880_restart_ice_0637.nc t0ni_00002880_restart_ice_0638.nc t0ni_00002880_restart_ice_0639.nc t0ni_00002880_restart_ice_0640.nc t0ni_00002880_restart_ice_0641.nc t0ni_00002880_restart_ice_0642.nc t0ni_00002880_restart_ice_0643.nc t0ni_00002880_restart_ice_0644.nc t0ni_00002880_restart_ice_0645.nc t0ni_00002880_restart_ice_0646.nc t0ni_00002880_restart_ice_0647.nc t0ni_00002880_restart_ice_0648.nc t0ni_00002880_restart_ice_0649.nc t0ni_00002880_restart_ice_0650.nc t0ni_00002880_restart_ice_0651.nc t0ni_00002880_restart_ice_0652.nc t0ni_00002880_restart_ice_0653.nc t0ni_00002880_restart_ice_0654.nc t0ni_00002880_restart_ice_0655.nc t0ni_00002880_restart_ice_0656.nc t0ni_00002880_restart_ice_0657.nc t0ni_00002880_restart_ice_0658.nc t0ni_00002880_restart_ice_0659.nc t0ni_00002880_restart_ice_0660.nc t0ni_00002880_restart_ice_0661.nc t0ni_00002880_restart_ice_0662.nc t0ni_00002880_restart_ice_0663.nc t0ni_00002880_restart_ice_0664.nc t0ni_00002880_restart_ice_0665.nc t0ni_00002880_restart_ice_0666.nc t0ni_00002880_restart_ice_0667.nc t0ni_00002880_restart_ice_0668.nc t0ni_00002880_restart_ice_0669.nc t0ni_00002880_restart_ice_0670.nc t0ni_00002880_restart_ice_0671.nc t0ni_00002880_restart_ice_0672.nc t0ni_00002880_restart_ice_0673.nc t0ni_00002880_restart_ice_0674.nc t0ni_00002880_restart_ice_0675.nc t0ni_00002880_restart_ice_0676.nc t0ni_00002880_restart_ice_0677.nc t0ni_00002880_restart_ice_0678.nc t0ni_00002880_restart_ice_0679.nc t0ni_00002880_restart_ice_0680.nc t0ni_00002880_restart_ice_0681.nc t0ni_00002880_restart_ice_0682.nc t0ni_00002880_restart_ice_0683.nc t0ni_00002880_restart_ice_0684.nc t0ni_00002880_restart_ice_0685.nc t0ni_00002880_restart_ice_0686.nc t0ni_00002880_restart_ice_0687.nc t0ni_00002880_restart_ice_0688.nc t0ni_00002880_restart_ice_0689.nc t0ni_00002880_restart_ice_0690.nc t0ni_00002880_restart_ice_0691.nc t0ni_00002880_restart_ice_0692.nc t0ni_00002880_restart_ice_0693.nc t0ni_00002880_restart_ice_0694.nc t0ni_00002880_restart_ice_0695.nc t0ni_00002880_restart_ice_0696.nc t0ni_00002880_restart_ice_0697.nc t0ni_00002880_restart_ice_0698.nc t0ni_00002880_restart_ice_0699.nc t0ni_00002880_restart_ice_0700.nc t0ni_00002880_restart_ice_0701.nc t0ni_00002880_restart_ice_0702.nc t0ni_00002880_restart_ice_0703.nc t0ni_00002880_restart_ice_0704.nc t0ni_00002880_restart_ice_0705.nc t0ni_00002880_restart_ice_0706.nc t0ni_00002880_restart_ice_0707.nc t0ni_00002880_restart_ice_0708.nc t0ni_00002880_restart_ice_0709.nc t0ni_00002880_restart_ice_0710.nc t0ni_00002880_restart_ice_0711.nc t0ni_00002880_restart_ice_0712.nc t0ni_00002880_restart_ice_0713.nc t0ni_00002880_restart_ice_0714.nc t0ni_00002880_restart_ice_0715.nc t0ni_00002880_restart_ice_0716.nc t0ni_00002880_restart_ice_0717.nc t0ni_00002880_restart_ice_0718.nc t0ni_00002880_restart_ice_0719.nc t0ni_00002880_restart_ice_0720.nc t0ni_00002880_restart_ice_0721.nc t0ni_00002880_restart_ice_0722.nc t0ni_00002880_restart_ice_0723.nc t0ni_00002880_restart_ice_0724.nc t0ni_00002880_restart_ice_0725.nc t0ni_00002880_restart_ice_0726.nc t0ni_00002880_restart_ice_0727.nc t0ni_00002880_restart_ice_0728.nc t0ni_00002880_restart_ice_0729.nc t0ni_00002880_restart_ice_0730.nc t0ni_00002880_restart_ice_0731.nc t0ni_00002880_restart_ice_0732.nc t0ni_00002880_restart_ice_0733.nc t0ni_00002880_restart_ice_0734.nc t0ni_00002880_restart_ice_0735.nc t0ni_00002880_restart_ice_0736.nc t0ni_00002880_restart_ice_0737.nc t0ni_00002880_restart_ice_0738.nc t0ni_00002880_restart_ice_0739.nc t0ni_00002880_restart_ice_0740.nc t0ni_00002880_restart_ice_0741.nc t0ni_00002880_restart_ice_0742.nc t0ni_00002880_restart_ice_0743.nc t0ni_00002880_restart_ice_0744.nc t0ni_00002880_restart_ice_0745.nc t0ni_00002880_restart_ice_0746.nc t0ni_00002880_restart_ice_0747.nc t0ni_00002880_restart_ice_0748.nc t0ni_00002880_restart_ice_0749.nc t0ni_00002880_restart_ice_0750.nc t0ni_00002880_restart_ice_0751.nc t0ni_00002880_restart_ice_0752.nc t0ni_00002880_restart_ice_0753.nc t0ni_00002880_restart_ice_0754.nc t0ni_00002880_restart_ice_0755.nc t0ni_00002880_restart_ice_0756.nc t0ni_00002880_restart_ice_0757.nc t0ni_00002880_restart_ice_0758.nc t0ni_00002880_restart_ice_0759.nc t0ni_00002880_restart_ice_0760.nc t0ni_00002880_restart_ice_0761.nc t0ni_00002880_restart_ice_0762.nc t0ni_00002880_restart_ice_0763.nc t0ni_00002880_restart_ice_0764.nc t0ni_00002880_restart_ice_0765.nc t0ni_00002880_restart_ice_0766.nc t0ni_00002880_restart_ice_0767.nc t0ni_00002880_restart_ice_0768.nc t0ni_00002880_restart_ice_0769.nc t0ni_00002880_restart_ice_0770.nc t0ni_00002880_restart_ice_0771.nc t0ni_00002880_restart_ice_0772.nc t0ni_00002880_restart_ice_0773.nc t0ni_00002880_restart_ice_0774.nc t0ni_00002880_restart_ice_0775.nc t0ni_00002880_restart_ice_0776.nc t0ni_00002880_restart_ice_0777.nc t0ni_00002880_restart_ice_0778.nc t0ni_00002880_restart_ice_0779.nc t0ni_00002880_restart_ice_0780.nc t0ni_00002880_restart_ice_0781.nc t0ni_00002880_restart_ice_0782.nc t0ni_00002880_restart_ice_0783.nc t0ni_00002880_restart_ice_0784.nc t0ni_00002880_restart_ice_0785.nc t0ni_00002880_restart_ice_0786.nc t0ni_00002880_restart_ice_0787.nc t0ni_00002880_restart_ice_0788.nc t0ni_00002880_restart_ice_0789.nc t0ni_00002880_restart_ice_0790.nc t0ni_00002880_restart_ice_0791.nc t0ni_00002880_restart_ice_0792.nc t0ni_00002880_restart_ice_0793.nc t0ni_00002880_restart_ice_0794.nc t0ni_00002880_restart_ice_0795.nc t0ni_00002880_restart_ice_0796.nc t0ni_00002880_restart_ice_0797.nc t0ni_00002880_restart_ice_0798.nc t0ni_00002880_restart_ice_0799.nc t0ni_00002880_restart_ice_0800.nc t0ni_00002880_restart_ice_0801.nc t0ni_00002880_restart_ice_0802.nc t0ni_00002880_restart_ice_0803.nc t0ni_00002880_restart_ice_0804.nc t0ni_00002880_restart_ice_0805.nc t0ni_00002880_restart_ice_0806.nc t0ni_00002880_restart_ice_0807.nc t0ni_00002880_restart_ice_0808.nc t0ni_00002880_restart_ice_0809.nc t0ni_00002880_restart_ice_0810.nc t0ni_00002880_restart_ice_0811.nc t0ni_00002880_restart_ice_0812.nc t0ni_00002880_restart_ice_0813.nc t0ni_00002880_restart_ice_0814.nc t0ni_00002880_restart_ice_0815.nc t0ni_00002880_restart_ice_0816.nc t0ni_00002880_restart_ice_0817.nc t0ni_00002880_restart_ice_0818.nc t0ni_00002880_restart_ice_0819.nc t0ni_00002880_restart_ice_0820.nc t0ni_00002880_restart_ice_0821.nc t0ni_00002880_restart_ice_0822.nc t0ni_00002880_restart_ice_0823.nc t0ni_00002880_restart_ice_0824.nc t0ni_00002880_restart_ice_0825.nc t0ni_00002880_restart_ice_0826.nc t0ni_00002880_restart_ice_0827.nc t0ni_00002880_restart_ice_0828.nc t0ni_00002880_restart_ice_0829.nc t0ni_00002880_restart_ice_0830.nc t0ni_00002880_restart_ice_0831.nc t0ni_00002880_restart_ice_0832.nc t0ni_00002880_restart_ice_0833.nc t0ni_00002880_restart_ice_0834.nc t0ni_00002880_restart_ice_0835.nc t0ni_00002880_restart_ice_0836.nc t0ni_00002880_restart_ice_0837.nc t0ni_00002880_restart_ice_0838.nc t0ni_00002880_restart_ice_0839.nc t0ni_00002880_restart_ice_0840.nc t0ni_00002880_restart_ice_0841.nc t0ni_00002880_restart_ice_0842.nc t0ni_00002880_restart_ice_0843.nc t0ni_00002880_restart_ice_0844.nc t0ni_00002880_restart_ice_0845.nc t0ni_00002880_restart_ice_0846.nc t0ni_00002880_restart_ice_0847.nc t0ni_00002880_restart_ice_0848.nc t0ni_00002880_restart_ice_0849.nc t0ni_00002880_restart_ice_0850.nc t0ni_00002880_restart_ice_0851.nc t0ni_00002880_restart_ice_0852.nc t0ni_00002880_restart_ice_0853.nc t0ni_00002880_restart_ice_0854.nc t0ni_00002880_restart_ice_0855.nc t0ni_00002880_restart_ice_0856.nc t0ni_00002880_restart_ice_0857.nc t0ni_00002880_restart_ice_0858.nc t0ni_00002880_restart_ice_0859.nc t0ni_00002880_restart_ice_0860.nc t0ni_00002880_restart_ice_0861.nc t0ni_00002880_restart_ice_0862.nc t0ni_00002880_restart_ice_0863.nc t0ni_00002880_restart_ice_0864.nc t0ni_00002880_restart_ice_0865.nc t0ni_00002880_restart_ice_0866.nc t0ni_00002880_restart_ice_0867.nc t0ni_00002880_restart_ice_0868.nc t0ni_00002880_restart_ice_0869.nc t0ni_00002880_restart_ice_0870.nc t0ni_00002880_restart_ice_0871.nc t0ni_00002880_restart_ice_0872.nc t0ni_00002880_restart_ice_0873.nc t0ni_00002880_restart_ice_0874.nc t0ni_00002880_restart_ice_0875.nc t0ni_00002880_restart_ice_0876.nc t0ni_00002880_restart_ice_0877.nc t0ni_00002880_restart_ice_0878.nc t0ni_00002880_restart_ice_0879.nc t0ni_00002880_restart_ice_0880.nc t0ni_00002880_restart_ice_0881.nc t0ni_00002880_restart_ice_0882.nc t0ni_00002880_restart_ice_0883.nc t0ni_00002880_restart_ice_0884.nc t0ni_00002880_restart_ice_0885.nc t0ni_00002880_restart_ice_0886.nc t0ni_00002880_restart_ice_0887.nc t0ni_00002880_restart_ice_0888.nc t0ni_00002880_restart_ice_0889.nc t0ni_00002880_restart_ice_0890.nc t0ni_00002880_restart_ice_0891.nc t0ni_00002880_restart_ice_0892.nc t0ni_00002880_restart_ice_0893.nc t0ni_00002880_restart_ice_0894.nc t0ni_00002880_restart_ice_0895.nc t0ni_00002880_restart_ice_0896.nc t0ni_00002880_restart_ice_0897.nc t0ni_00002880_restart_ice_0898.nc t0ni_00002880_restart_ice_0899.nc t0ni_00002880_restart_ice_0900.nc t0ni_00002880_restart_ice_0901.nc t0ni_00002880_restart_ice_0902.nc t0ni_00002880_restart_ice_0903.nc t0ni_00002880_restart_ice_0904.nc t0ni_00002880_restart_ice_0905.nc t0ni_00002880_restart_ice_0906.nc t0ni_00002880_restart_ice_0907.nc t0ni_00002880_restart_ice_0908.nc t0ni_00002880_restart_ice_0909.nc t0ni_00002880_restart_ice_0910.nc t0ni_00002880_restart_ice_0911.nc t0ni_00002880_restart_ice_0912.nc t0ni_00002880_restart_ice_0913.nc t0ni_00002880_restart_ice_0914.nc t0ni_00002880_restart_ice_0915.nc t0ni_00002880_restart_ice_0916.nc t0ni_00002880_restart_ice_0917.nc t0ni_00002880_restart_ice_0918.nc t0ni_00002880_restart_ice_0919.nc t0ni_00002880_restart_ice_0920.nc t0ni_00002880_restart_ice_0921.nc t0ni_00002880_restart_ice_0922.nc t0ni_00002880_restart_ice_0923.nc t0ni_00002880_restart_ice_0924.nc t0ni_00002880_restart_ice_0925.nc t0ni_00002880_restart_ice_0926.nc t0ni_00002880_restart_ice_0927.nc t0ni_00002880_restart_ice_0928.nc t0ni_00002880_restart_ice_0929.nc t0ni_00002880_restart_ice_0930.nc t0ni_00002880_restart_ice_0931.nc t0ni_00002880_restart_ice_0932.nc t0ni_00002880_restart_ice_0933.nc t0ni_00002880_restart_ice_0934.nc t0ni_00002880_restart_ice_0935.nc t0ni_00002880_restart_ice_0936.nc t0ni_00002880_restart_ice_0937.nc t0ni_00002880_restart_ice_0938.nc t0ni_00002880_restart_ice_0939.nc t0ni_00002880_restart_ice_0940.nc t0ni_00002880_restart_ice_0941.nc t0ni_00002880_restart_ice_0942.nc t0ni_00002880_restart_ice_0943.nc t0ni_00002880_restart_ice_0944.nc t0ni_00002880_restart_ice_0945.nc t0ni_00002880_restart_ice_0946.nc t0ni_00002880_restart_ice_0947.nc t0ni_00002880_restart_ice_0948.nc t0ni_00002880_restart_ice_0949.nc t0ni_00002880_restart_ice_0950.nc t0ni_00002880_restart_ice_0951.nc t0ni_00002880_restart_ice_0952.nc t0ni_00002880_restart_ice_0953.nc t0ni_00002880_restart_ice_0954.nc t0ni_00002880_restart_ice_0955.nc t0ni_00002880_restart_ice_0956.nc t0ni_00002880_restart_ice_0957.nc t0ni_00002880_restart_ice_0958.nc t0ni_00002880_restart_ice_0959.nc t0ni_00002880_restart_ice_0960.nc t0ni_00002880_restart_ice_0961.nc t0ni_00002880_restart_ice_0962.nc t0ni_00002880_restart_ice_0963.nc t0ni_00002880_restart_ice_0964.nc t0ni_00002880_restart_ice_0965.nc t0ni_00002880_restart_ice_0966.nc t0ni_00002880_restart_ice_0967.nc t0ni_00002880_restart_ice_0968.nc t0ni_00002880_restart_ice_0969.nc t0ni_00002880_restart_ice_0970.nc t0ni_00002880_restart_ice_0971.nc t0ni_00002880_restart_ice_0972.nc t0ni_00002880_restart_ice_0973.nc t0ni_00002880_restart_ice_0974.nc t0ni_00002880_restart_ice_0975.nc t0ni_00002880_restart_ice_0976.nc t0ni_00002880_restart_ice_0977.nc t0ni_00002880_restart_ice_0978.nc t0ni_00002880_restart_ice_0979.nc t0ni_00002880_restart_ice_0980.nc t0ni_00002880_restart_ice_0981.nc t0ni_00002880_restart_ice_0982.nc t0ni_00002880_restart_ice_0983.nc t0ni_00002880_restart_ice_0984.nc t0ni_00002880_restart_ice_0985.nc t0ni_00002880_restart_ice_0986.nc t0ni_00002880_restart_ice_0987.nc t0ni_00002880_restart_ice_0988.nc t0ni_00002880_restart_ice_0989.nc t0ni_00002880_restart_ice_0990.nc t0ni_00002880_restart_ice_0991.nc t0ni_00002880_restart_ice_0992.nc t0ni_00002880_restart_ice_0993.nc t0ni_00002880_restart_ice_0994.nc t0ni_00002880_restart_ice_0995.nc t0ni_00002880_restart_ice_0996.nc t0ni_00002880_restart_ice_0997.nc t0ni_00002880_restart_ice_0998.nc t0ni_00002880_restart_ice_0999.nc t0ni_00002880_restart_ice_1000.nc t0ni_00002880_restart_ice_1001.nc t0ni_00002880_restart_ice_1002.nc t0ni_00002880_restart_ice_1003.nc t0ni_00002880_restart_ice_1004.nc t0ni_00002880_restart_ice_1005.nc t0ni_00002880_restart_ice_1006.nc t0ni_00002880_restart_ice_1007.nc t0ni_00002880_restart_ice_1008.nc t0ni_00002880_restart_ice_1009.nc t0ni_00002880_restart_ice_1010.nc t0ni_00002880_restart_ice_1011.nc t0ni_00002880_restart_ice_1012.nc t0ni_00002880_restart_ice_1013.nc t0ni_00002880_restart_ice_1014.nc t0ni_00002880_restart_ice_1015.nc t0ni_00002880_restart_ice_1016.nc t0ni_00002880_restart_ice_1017.nc t0ni_00002880_restart_ice_1018.nc t0ni_00002880_restart_ice_1019.nc t0ni_00002880_restart_ice_1020.nc t0ni_00002880_restart_ice_1021.nc t0ni_00002880_restart_ice_1022.nc t0ni_00002880_restart_ice_1023.nc t0ni_00002880_restart_ice_1024.nc t0ni_00002880_restart_ice_1025.nc t0ni_00002880_restart_ice_1026.nc t0ni_00002880_restart_ice_1027.nc t0ni_00002880_restart_ice_1028.nc t0ni_00002880_restart_ice_1029.nc t0ni_00002880_restart_ice_1030.nc t0ni_00002880_restart_ice_1031.nc t0ni_00002880_restart_ice_1032.nc t0ni_00002880_restart_ice_1033.nc t0ni_00002880_restart_ice_1034.nc t0ni_00002880_restart_ice_1035.nc t0ni_00002880_restart_ice_1036.nc t0ni_00002880_restart_ice_1037.nc t0ni_00002880_restart_ice_1038.nc t0ni_00002880_restart_ice_1039.nc t0ni_00002880_restart_ice_1040.nc t0ni_00002880_restart_ice_1041.nc t0ni_00002880_restart_ice_1042.nc t0ni_00002880_restart_ice_1043.nc t0ni_00002880_restart_ice_1044.nc t0ni_00002880_restart_ice_1045.nc t0ni_00002880_restart_ice_1046.nc t0ni_00002880_restart_ice_1047.nc t0ni_00002880_restart_ice_1048.nc t0ni_00002880_restart_ice_1049.nc t0ni_00002880_restart_ice_1050.nc t0ni_00002880_restart_ice_1051.nc t0ni_00002880_restart_ice_1052.nc t0ni_00002880_restart_ice_1053.nc t0ni_00002880_restart_ice_1054.nc t0ni_00002880_restart_ice_1055.nc t0ni_00002880_restart_ice_1056.nc t0ni_00002880_restart_ice_1057.nc t0ni_00002880_restart_ice_1058.nc t0ni_00002880_restart_ice_1059.nc t0ni_00002880_restart_ice_1060.nc t0ni_00002880_restart_ice_1061.nc t0ni_00002880_restart_ice_1062.nc t0ni_00002880_restart_ice_1063.nc t0ni_00002880_restart_ice_1064.nc t0ni_00002880_restart_ice_1065.nc t0ni_00002880_restart_ice_1066.nc t0ni_00002880_restart_ice_1067.nc t0ni_00002880_restart_ice_1068.nc t0ni_00002880_restart_ice_1069.nc t0ni_00002880_restart_ice_1070.nc t0ni_00002880_restart_ice_1071.nc t0ni_00002880_restart_ice_1072.nc t0ni_00002880_restart_ice_1073.nc t0ni_00002880_restart_ice_1074.nc t0ni_00002880_restart_ice_1075.nc t0ni_00002880_restart_ice_1076.nc t0ni_00002880_restart_ice_1077.nc t0ni_00002880_restart_ice_1078.nc t0ni_00002880_restart_ice_1079.nc t0ni_00002880_restart_ice_1080.nc t0ni_00002880_restart_ice_1081.nc t0ni_00002880_restart_ice_1082.nc t0ni_00002880_restart_ice_1083.nc t0ni_00002880_restart_ice_1084.nc t0ni_00002880_restart_ice_1085.nc t0ni_00002880_restart_ice_1086.nc t0ni_00002880_restart_ice_1087.nc t0ni_00002880_restart_ice_1088.nc t0ni_00002880_restart_ice_1089.nc t0ni_00002880_restart_ice_1090.nc t0ni_00002880_restart_ice_1091.nc t0ni_00002880_restart_ice_1092.nc t0ni_00002880_restart_ice_1093.nc t0ni_00002880_restart_ice_1094.nc t0ni_00002880_restart_ice_1095.nc t0ni_00002880_restart_ice_1096.nc t0ni_00002880_restart_ice_1097.nc t0ni_00002880_restart_ice_1098.nc t0ni_00002880_restart_ice_1099.nc t0ni_00002880_restart_ice_1100.nc t0ni_00002880_restart_ice_1101.nc t0ni_00002880_restart_ice_1102.nc t0ni_00002880_restart_ice_1103.nc t0ni_00002880_restart_ice_1104.nc t0ni_00002880_restart_ice_1105.nc t0ni_00002880_restart_ice_1106.nc t0ni_00002880_restart_ice_1107.nc t0ni_00002880_restart_ice_1108.nc t0ni_00002880_restart_ice_1109.nc t0ni_00002880_restart_ice_1110.nc t0ni_00002880_restart_ice_1111.nc t0ni_00002880_restart_ice_1112.nc t0ni_00002880_restart_ice_1113.nc t0ni_00002880_restart_ice_1114.nc t0ni_00002880_restart_ice_1115.nc t0ni_00002880_restart_ice_1116.nc t0ni_00002880_restart_ice_1117.nc t0ni_00002880_restart_ice_1118.nc t0ni_00002880_restart_ice_1119.nc t0ni_00002880_restart_ice_1120.nc t0ni_00002880_restart_ice_1121.nc t0ni_00002880_restart_ice_1122.nc t0ni_00002880_restart_ice_1123.nc t0ni_00002880_restart_ice_1124.nc t0ni_00002880_restart_ice_1125.nc t0ni_00002880_restart_ice_1126.nc t0ni_00002880_restart_ice_1127.nc t0ni_00002880_restart_ice_1128.nc t0ni_00002880_restart_ice_1129.nc t0ni_00002880_restart_ice_1130.nc t0ni_00002880_restart_ice_1131.nc t0ni_00002880_restart_ice_1132.nc t0ni_00002880_restart_ice_1133.nc t0ni_00002880_restart_ice_1134.nc t0ni_00002880_restart_ice_1135.nc t0ni_00002880_restart_ice_1136.nc t0ni_00002880_restart_ice_1137.nc t0ni_00002880_restart_ice_1138.nc t0ni_00002880_restart_ice_1139.nc t0ni_00002880_restart_ice_1140.nc t0ni_00002880_restart_ice_1141.nc t0ni_00002880_restart_ice_1142.nc t0ni_00002880_restart_ice_1143.nc t0ni_00002880_restart_ice_1144.nc t0ni_00002880_restart_ice_1145.nc t0ni_00002880_restart_ice_1146.nc t0ni_00002880_restart_ice_1147.nc t0ni_00002880_restart_ice_1148.nc t0ni_00002880_restart_ice_1149.nc t0ni_00002880_restart_ice_1150.nc t0ni_00002880_restart_ice_1151.nc t0ni_00002880_restart_ice_1152.nc t0ni_00002880_restart_ice_1153.nc t0ni_00002880_restart_ice_1154.nc t0ni_00002880_restart_ice_1155.nc t0ni_00002880_restart_ice_1156.nc t0ni_00002880_restart_ice_1157.nc t0ni_00002880_restart_ice_1158.nc t0ni_00002880_restart_ice_1159.nc t0ni_00002880_restart_ice_1160.nc t0ni_00002880_restart_ice_1161.nc t0ni_00002880_restart_ice_1162.nc t0ni_00002880_restart_ice_1163.nc t0ni_00002880_restart_ice_1164.nc t0ni_00002880_restart_ice_1165.nc t0ni_00002880_restart_ice_1166.nc t0ni_00002880_restart_ice_1167.nc t0ni_00002880_restart_ice_1168.nc t0ni_00002880_restart_ice_1169.nc t0ni_00002880_restart_ice_1170.nc t0ni_00002880_restart_ice_1171.nc t0ni_00002880_restart_ice_1172.nc t0ni_00002880_restart_ice_1173.nc t0ni_00002880_restart_ice_1174.nc t0ni_00002880_restart_ice_1175.nc t0ni_00002880_restart_ice_1176.nc t0ni_00002880_restart_ice_1177.nc t0ni_00002880_restart_ice_1178.nc t0ni_00002880_restart_ice_1179.nc t0ni_00002880_restart_ice_1180.nc t0ni_00002880_restart_ice_1181.nc t0ni_00002880_restart_ice_1182.nc t0ni_00002880_restart_ice_1183.nc t0ni_00002880_restart_ice_1184.nc t0ni_00002880_restart_ice_1185.nc t0ni_00002880_restart_ice_1186.nc t0ni_00002880_restart_ice_1187.nc t0ni_00002880_restart_ice_1188.nc t0ni_00002880_restart_ice_1189.nc t0ni_00002880_restart_ice_1190.nc t0ni_00002880_restart_ice_1191.nc t0ni_00002880_restart_ice_1192.nc t0ni_00002880_restart_ice_1193.nc t0ni_00002880_restart_ice_1194.nc t0ni_00002880_restart_ice_1195.nc t0ni_00002880_restart_ice_1196.nc t0ni_00002880_restart_ice_1197.nc t0ni_00002880_restart_ice_1198.nc t0ni_00002880_restart_ice_1199.nc t0ni_00002880_restart_ice_1200.nc t0ni_00002880_restart_ice_1201.nc t0ni_00002880_restart_ice_1202.nc t0ni_00002880_restart_ice_1203.nc t0ni_00002880_restart_ice_1204.nc t0ni_00002880_restart_ice_1205.nc t0ni_00002880_restart_ice_1206.nc t0ni_00002880_restart_ice_1207.nc t0ni_00002880_restart_ice_1208.nc t0ni_00002880_restart_ice_1209.nc t0ni_00002880_restart_ice_1210.nc t0ni_00002880_restart_ice_1211.nc t0ni_00002880_restart_ice_1212.nc t0ni_00002880_restart_ice_1213.nc t0ni_00002880_restart_ice_1214.nc t0ni_00002880_restart_ice_1215.nc t0ni_00002880_restart_ice_1216.nc t0ni_00002880_restart_ice_1217.nc t0ni_00002880_restart_ice_1218.nc t0ni_00002880_restart_ice_1219.nc t0ni_00002880_restart_ice_1220.nc t0ni_00002880_restart_ice_1221.nc t0ni_00002880_restart_ice_1222.nc t0ni_00002880_restart_ice_1223.nc t0ni_00002880_restart_ice_1224.nc t0ni_00002880_restart_ice_1225.nc t0ni_00002880_restart_ice_1226.nc t0ni_00002880_restart_ice_1227.nc t0ni_00002880_restart_ice_1228.nc t0ni_00002880_restart_ice_1229.nc t0ni_00002880_restart_ice_1230.nc t0ni_00002880_restart_ice_1231.nc t0ni_00002880_restart_ice_1232.nc t0ni_00002880_restart_ice_1233.nc t0ni_00002880_restart_ice_1234.nc t0ni_00002880_restart_ice_1235.nc t0ni_00002880_restart_ice_1236.nc t0ni_00002880_restart_ice_1237.nc t0ni_00002880_restart_ice_1238.nc t0ni_00002880_restart_ice_1239.nc t0ni_00002880_restart_ice_1240.nc t0ni_00002880_restart_ice_1241.nc t0ni_00002880_restart_ice_1242.nc t0ni_00002880_restart_ice_1243.nc t0ni_00002880_restart_ice_1244.nc t0ni_00002880_restart_ice_1245.nc t0ni_00002880_restart_ice_1246.nc t0ni_00002880_restart_ice_1247.nc t0ni_00002880_restart_ice_1248.nc t0ni_00002880_restart_ice_1249.nc t0ni_00002880_restart_ice_1250.nc t0ni_00002880_restart_ice_1251.nc t0ni_00002880_restart_ice_1252.nc t0ni_00002880_restart_ice_1253.nc t0ni_00002880_restart_ice_1254.nc t0ni_00002880_restart_ice_1255.nc t0ni_00002880_restart_ice_1256.nc t0ni_00002880_restart_ice_1257.nc t0ni_00002880_restart_ice_1258.nc t0ni_00002880_restart_ice_1259.nc t0ni_00002880_restart_ice_1260.nc t0ni_00002880_restart_ice_1261.nc t0ni_00002880_restart_ice_1262.nc t0ni_00002880_restart_ice_1263.nc t0ni_00002880_restart_ice_1264.nc t0ni_00002880_restart_ice_1265.nc t0ni_00002880_restart_ice_1266.nc t0ni_00002880_restart_ice_1267.nc t0ni_00002880_restart_ice_1268.nc t0ni_00002880_restart_ice_1269.nc t0ni_00002880_restart_ice_1270.nc t0ni_00002880_restart_ice_1271.nc t0ni_00002880_restart_ice_1272.nc t0ni_00002880_restart_ice_1273.nc t0ni_00002880_restart_ice_1274.nc t0ni_00002880_restart_ice_1275.nc t0ni_00002880_restart_ice_1276.nc t0ni_00002880_restart_ice_1277.nc t0ni_00002880_restart_ice_1278.nc t0ni_00002880_restart_ice_1279.nc t0ni_00002880_restart_ice_1280.nc t0ni_00002880_restart_ice_1281.nc t0ni_00002880_restart_ice_1282.nc t0ni_00002880_restart_ice_1283.nc t0ni_00002880_restart_ice_1284.nc t0ni_00002880_restart_ice_1285.nc t0ni_00002880_restart_ice_1286.nc t0ni_00002880_restart_ice_1287.nc t0ni_00002880_restart_ice_1288.nc t0ni_00002880_restart_ice_1289.nc t0ni_00002880_restart_ice_1290.nc t0ni_00002880_restart_ice_1291.nc t0ni_00002880_restart_ice_1292.nc t0ni_00002880_restart_ice_1293.nc t0ni_00002880_restart_ice_1294.nc t0ni_00002880_restart_ice_1295.nc t0ni_00002880_restart_ice_1296.nc t0ni_00002880_restart_ice_1297.nc t0ni_00002880_restart_ice_1298.nc t0ni_00002880_restart_ice_1299.nc t0ni_00002880_restart_ice_1300.nc t0ni_00002880_restart_ice_1301.nc t0ni_00002880_restart_ice_1302.nc t0ni_00002880_restart_ice_1303.nc t0ni_00002880_restart_ice_1304.nc t0ni_00002880_restart_ice_1305.nc t0ni_00002880_restart_ice_1306.nc t0ni_00002880_restart_ice_1307.nc t0ni_00002880_restart_ice_1308.nc t0ni_00002880_restart_ice_1309.nc t0ni_00002880_restart_ice_1310.nc t0ni_00002880_restart_ice_1311.nc t0ni_00002880_restart_ice_1312.nc t0ni_00002880_restart_ice_1313.nc t0ni_00002880_restart_ice_1314.nc t0ni_00002880_restart_ice_1315.nc t0ni_00002880_restart_ice_1316.nc t0ni_00002880_restart_ice_1317.nc t0ni_00002880_restart_ice_1318.nc t0ni_00002880_restart_ice_1319.nc t0ni_00002880_restart_ice_1320.nc t0ni_00002880_restart_ice_1321.nc t0ni_00002880_restart_ice_1322.nc t0ni_00002880_restart_ice_1323.nc t0ni_00002880_restart_ice_1324.nc t0ni_00002880_restart_ice_1325.nc t0ni_00002880_restart_ice_1326.nc t0ni_00002880_restart_ice_1327.nc t0ni_00002880_restart_ice_1328.nc t0ni_00002880_restart_ice_1329.nc t0ni_00002880_restart_ice_1330.nc t0ni_00002880_restart_ice_1331.nc t0ni_00002880_restart_ice_1332.nc t0ni_00002880_restart_ice_1333.nc t0ni_00002880_restart_ice_1334.nc t0ni_00002880_restart_ice_1335.nc t0ni_00002880_restart_ice_1336.nc t0ni_00002880_restart_ice_1337.nc t0ni_00002880_restart_ice_1338.nc t0ni_00002880_restart_ice_1339.nc t0ni_00002880_restart_ice_1340.nc t0ni_00002880_restart_ice_1341.nc t0ni_00002880_restart_ice_1342.nc t0ni_00002880_restart_ice_1343.nc t0ni_00002880_restart_ice_1344.nc t0ni_00002880_restart_ice_1345.nc t0ni_00002880_restart_ice_1346.nc t0ni_00002880_restart_ice_1347.nc t0ni_00002880_restart_ice_1348.nc t0ni_00002880_restart_ice_1349.nc t0ni_00002880_restart_ice_1350.nc t0ni_00002880_restart_ice_1351.nc t0ni_00002880_restart_ice_1352.nc t0ni_00002880_restart_ice_1353.nc t0ni_00002880_restart_ice_1354.nc t0ni_00002880_restart_ice_1355.nc t0ni_00002880_restart_ice_1356.nc t0ni_00002880_restart_ice_1357.nc t0ni_00002880_restart_ice_1358.nc t0ni_00002880_restart_ice_1359.nc t0ni_00002880_restart_ice_1360.nc t0ni_00002880_restart_ice_1361.nc t0ni_00002880_restart_ice_1362.nc t0ni_00002880_restart_ice_1363.nc t0ni_00002880_restart_ice_1364.nc t0ni_00002880_restart_ice_1365.nc t0ni_00002880_restart_ice_1366.nc t0ni_00002880_restart_ice_1367.nc t0ni_00002880_restart_ice_1368.nc t0ni_00002880_restart_ice_1369.nc t0ni_00002880_restart_ice_1370.nc t0ni_00002880_restart_ice_1371.nc t0ni_00002880_restart_ice_1372.nc t0ni_00002880_restart_ice_1373.nc t0ni_00002880_restart_ice_1374.nc t0ni_00002880_restart_ice_1375.nc t0ni_00002880_restart_ice_1376.nc t0ni_00002880_restart_ice_1377.nc t0ni_00002880_restart_ice_1378.nc t0ni_00002880_restart_ice_1379.nc t0ni_00002880_restart_ice_1380.nc t0ni_00002880_restart_ice_1381.nc t0ni_00002880_restart_ice_1382.nc t0ni_00002880_restart_ice_1383.nc t0ni_00002880_restart_ice_1384.nc t0ni_00002880_restart_ice_1385.nc t0ni_00002880_restart_ice_1386.nc t0ni_00002880_restart_ice_1387.nc t0ni_00002880_restart_ice_1388.nc t0ni_00002880_restart_ice_1389.nc t0ni_00002880_restart_ice_1390.nc ../delete
+ (( j++ ))
+ (( j < 2 ))
+ cd ..
+ echo =====================================
+ has_config oasis
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' oasis == any ']'
+ [[ oasis == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ oasis ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ oasis ]]
+ for __m in '$config'
+ [[ lim3 =~ oasis ]]
+ for __m in '$config'
+ [[ rnfmapper =~ oasis ]]
+ for __m in '$config'
+ [[ xios:detached =~ oasis ]]
+ for __m in '$config'
+ [[ oasis =~ oasis ]]
+ [[ oasis =~ oasis ]]
+ __f=1
+ for __m in '$config'
+ [[ save_ic:end_leg =~ oasis ]]
+ ((  __f  ))
+ return 0
+ '[' -d oasis ']'
+ cd oasis
+ oas_rst_dir=/gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/oasis
+ oas_rst_ifs_nemo='rstas.nc rstos.nc'
+ has_config ifs nemo
+ ((  2  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' ifs == any ']'
+ [[ ifs == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ ifs ]]
+ [[ ifs =~ ifs ]]
+ __f=1
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ ifs ]]
+ for __m in '$config'
+ [[ lim3 =~ ifs ]]
+ for __m in '$config'
+ [[ rnfmapper =~ ifs ]]
+ for __m in '$config'
+ [[ xios:detached =~ ifs ]]
+ for __m in '$config'
+ [[ oasis =~ ifs ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ ifs ]]
+ ((  __f  ))
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ nemo ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ nemo ]]
+ [[ nemo:start_from_restart:elpin =~ nemo ]]
+ __f=1
+ for __m in '$config'
+ [[ lim3 =~ nemo ]]
+ for __m in '$config'
+ [[ rnfmapper =~ nemo ]]
+ for __m in '$config'
+ [[ xios:detached =~ nemo ]]
+ for __m in '$config'
+ [[ oasis =~ nemo ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ nemo ]]
+ ((  __f  ))
+ return 0
ls ${oas_rst_ifs_nemo} 2>/dev/null || true
++ ls rstas.nc rstos.nc
+ ifiles='rstas.nc
rstos.nc'
+ odir=/gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/coupler/T511-ORCA025/t0ni
+ [[ rstas.nc
rstos.nc != '' ]]
+ echo oasis ifs nemo restart files rstas.nc rstos.nc 'found!'
+ rm -f /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/oasis_ok
+ rm -rf /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/coupler/T511-ORCA025/t0ni
+ mkdir -p /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/coupler/T511-ORCA025/t0ni
ls $ifiles 2>/dev/null
++ ls rstas.nc rstos.nc
+ for ifile in '$(ls $ifiles 2>/dev/null)'
basename ${ifile}
++ basename rstas.nc
+ ofile=t0ni_fc0_1993120100_rstas.nc.gz
+ gzip -c rstas.nc
+ mv rstas.nc ../delete
+ for ifile in '$(ls $ifiles 2>/dev/null)'
basename ${ifile}
++ basename rstos.nc
+ ofile=t0ni_fc0_1993120100_rstos.nc.gz
+ gzip -c rstos.nc
+ mv rstos.nc ../delete
+ touch /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/oasis_ok
+ oas_rst_ifs_lpjg='vegin.nc lpjgv.nc'
+ has_config ifs lpjg
+ ((  2  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' ifs == any ']'
+ [[ ifs == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ ifs ]]
+ [[ ifs =~ ifs ]]
+ __f=1
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ ifs ]]
+ for __m in '$config'
+ [[ lim3 =~ ifs ]]
+ for __m in '$config'
+ [[ rnfmapper =~ ifs ]]
+ for __m in '$config'
+ [[ xios:detached =~ ifs ]]
+ for __m in '$config'
+ [[ oasis =~ ifs ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ ifs ]]
+ ((  __f  ))
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ lpjg ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ lpjg ]]
+ for __m in '$config'
+ [[ lim3 =~ lpjg ]]
+ for __m in '$config'
+ [[ rnfmapper =~ lpjg ]]
+ for __m in '$config'
+ [[ xios:detached =~ lpjg ]]
+ for __m in '$config'
+ [[ oasis =~ lpjg ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ lpjg ]]
+ ((  __f  ))
+ return 1
+ has_config ifs tm5
+ ((  2  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' ifs == any ']'
+ [[ ifs == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ ifs ]]
+ [[ ifs =~ ifs ]]
+ __f=1
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ ifs ]]
+ for __m in '$config'
+ [[ lim3 =~ ifs ]]
+ for __m in '$config'
+ [[ rnfmapper =~ ifs ]]
+ for __m in '$config'
+ [[ xios:detached =~ ifs ]]
+ for __m in '$config'
+ [[ oasis =~ ifs ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ ifs ]]
+ ((  __f  ))
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ tm5 ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ tm5 ]]
+ for __m in '$config'
+ [[ lim3 =~ tm5 ]]
+ for __m in '$config'
+ [[ rnfmapper =~ tm5 ]]
+ for __m in '$config'
+ [[ xios:detached =~ tm5 ]]
+ for __m in '$config'
+ [[ oasis =~ tm5 ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ tm5 ]]
+ ((  __f  ))
+ return 1
+ cd ..
+ echo =====================================
+ has_config lpjg
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' lpjg == any ']'
+ [[ lpjg == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ lpjg ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ lpjg ]]
+ for __m in '$config'
+ [[ lim3 =~ lpjg ]]
+ for __m in '$config'
+ [[ rnfmapper =~ lpjg ]]
+ for __m in '$config'
+ [[ xios:detached =~ lpjg ]]
+ for __m in '$config'
+ [[ oasis =~ lpjg ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ lpjg ]]
+ ((  __f  ))
+ return 1
+ echo =====================================
+ has_config tm5
+ ((  1  ))
+ [[ -z ifs    nemo:start_from_restart:elpin  lim3 rnfmapper xios:detached oasis   save_ic:end_leg ]]
+ local __c
+ local __m
+ '[' tm5 == any ']'
+ [[ tm5 == \a\l\l ]]
+ local __f
+ for __c in '"$@"'
+ __f=0
+ for __m in '$config'
+ [[ ifs =~ tm5 ]]
+ for __m in '$config'
+ [[ nemo:start_from_restart:elpin =~ tm5 ]]
+ for __m in '$config'
+ [[ lim3 =~ tm5 ]]
+ for __m in '$config'
+ [[ rnfmapper =~ tm5 ]]
+ for __m in '$config'
+ [[ xios:detached =~ tm5 ]]
+ for __m in '$config'
+ [[ oasis =~ tm5 ]]
+ for __m in '$config'
+ [[ save_ic:end_leg =~ tm5 ]]
+ ((  __f  ))
+ return 1
+ echo =====================================
+ '[' '!' -d /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic ']'
find ${outdir}/ic/ -type d
++ find /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/ -type d
+ chmod 775 /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/ /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/atmos /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/atmos/T511L91 /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/atmos/T511L91/t0ni /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/ocean /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/ocean/ORCA025L75 /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/ocean/ORCA025L75/t0ni /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/coupler /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/coupler/T511-ORCA025 /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/coupler/T511-ORCA025/t0ni /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/ice /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/ice/ORCA025_LIM3 /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/ice/ORCA025_LIM3/t0ni
find ${outdir}/ic/ -type f
++ find /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/ -type f
+ chmod 654 /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/atmos/T511L91/t0ni/t0ni_fc0_1993120100.tar.gz /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/ocean/ORCA025L75/t0ni/t0ni_fc0_19931130_restart.nc.gz /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/coupler/T511-ORCA025/t0ni/t0ni_fc0_1993120100_rstos.nc.gz /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/coupler/T511-ORCA025/t0ni/t0ni_fc0_1993120100_rstas.nc.gz /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic/ice/ORCA025_LIM3/t0ni/t0ni_fc0_19931130_restart_ice.nc.gz
+ [[ FALSE == \T\R\U\E ]]
+ pushd .
+ cd /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201/ic
+ tar -cvf ../IC_t0ni_19931101_fc0_001_0_19931101-19931130.tar atmos coupler ice ocean
+ chmod 774 ../IC_t0ni_19931101_fc0_001_0_19931101-19931130.tar
+ cd ..
+ mkdir -p /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/ic
+ mv -f IC_t0ni_19931101_fc0_001_0_19931101-19931130.tar /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/ic
+ echo moved IC_t0ni_19931101_fc0_001_0_19931101-19931130.tar to /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/ic
+ popd
+ rm -rf /gpfs/scratch/bsc32/bsc32627/t0ni/19931101/fc0/runtime/save_ic/19931201
+ (( i++ ))
+ (( i < 1 ))

###################
# Autosubmit tailer
###################
set -xuve
+ set -xuve
echo $(date +%s) >> ${job_name_ptrn}_STAT
date +%s
++ date +%s
+ echo 1639816010
touch ${job_name_ptrn}_COMPLETED
+ touch /gpfs/scratch/bsc32/bsc32627/t0ni/LOG_t0ni/t0ni_19931101_fc0_1_SAVEIC_COMPLETED
exit 0
+ exit 0
