#!/bin/ksh
###############################################################################
#                              RUN EC_EARTH
###############################################################################
#BSUB -J run_t159l62_uncoup
#BSUB -o run_t159l62_uncoup_%J.out
#BSUB -e run_t159l62_uncoup_%J.err
#BSUB -n 32
#BSUB -R "span[ptile=16]"
#BSUB -W 00:30
#

# modules setup
source /usr/share/modules/init/ksh
module switch openmpi impi
export I_MPI_PMI_EXTENSIONS=on
export DAPL_DBG_TYPE_CM_EST="0x8000"
export DAPL_DBG_TYPE_CM_WARN="0x10000"
export DAPL_CM_ROUTE_TIMEOUT_MS=20000
export DAPL_CM_ARP_TIMEOUT_MS=10000
export DAPL_UCM_REP_TIME=800
export DAPL_UCM_RTU_TIME=400
export DAPL_UCM_RETRY=30
export DAPL_ACK_RETRY=30
export DAPL_ACK_TIME=30
export I_MPI_DEVICE="rdssm:ofa-v2-ib0"

# General settings

set -xuv
date

ECEARTH=/gpfs/projects/ecm86/models/ecearth/v2.3.0
SCRIPTDIR=$ECEARTH/setup
BINDIR=$ECEARTH/bin
PATH=${SCRIPTDIR}:${PATH}
TESTDATADIR=$ECEARTH/inidata/testdata
WRITINGDIR=/gpfs/scratch/ecm86/$USER/test
RUN_dir=$WRITINGDIR/run_t159l62_uncoup_${LSB_JOBID}

# Resolution dependent part

IFS_resolution=T159L62
RUN_length_prev=0
FCLENGTH=48
WAM=false
NPROCIFS=32
COUPFREQ=6
NPROMA=40
LLAST=TRUE
NPRTRVIFS=1
EXPID=b0if
LFASTRES=FALSE
NFRP=6
LCMIP5=TRUE
NRCP=0

# Bin path

IFSEXE=$BINDIR/ifsMASTER

# Run setup

if [[ ${RUN_length_prev} = '0' ]]; then
  mkdir -p ${RUN_dir} || true
  rm -fr ${RUN_dir}/*
fi
cd ${RUN_dir}

export LOCAL_DEFINITION_TEMPLATES=$ECEARTH/sources/ifs/src/emos/local_definitions
export MBX_SIZE=512000000

EXPVER=$(setupifs.ksh -r ${IFS_resolution} -e $ECEARTH/sources -l $FCLENGTH -n $NPROCIFS -w $WAM -p $NPROMA -c $COUPFREQ -o .$LLAST. -0 $TESTDATADIR -1 $NPRTRVIFS -2 ${EXPID} -3 .$LFASTRES. -4 $NFRP -5 .$LCMIP5. -6 $NRCP)

# Execution

export OASIS3=simulate
export OASIS3DEBUGLEVEL=3
export DR_HOOK_IGNORE_SIGNALS='-1'

ulimit -s unlimited

t1=$(date +%s)

mpirun -np ${NPROCIFS} ${IFSEXE} -v ecmwf -e ${EXPID}

t2=$(date +%s)

tr=$(date -d "0 -$t1 sec + $t2 sec" +%T)

date
