#!/bin/ksh
###############################################################################
#                              RUN EC_EARTH
###############################################################################
#BSUB -J run_t159l62_orca1
#BSUB -o run_t159l62_orca1_%J.out
#BSUB -e run_t159l62_orca1_%J.err
#BSUB -n 64
#BSUB -R "span[ptile=16]"
#BSUB -W 00:30
#

# modules setup
source /usr/share/modules/init/ksh
module switch openmpi impi
export I_MPI_PMI_EXTENSIONS=on
export DAPL_DBG_TYPE_CM_EST="0x8000"
export DAPL_DBG_TYPE_CM_WARN="0x10000"
export DAPL_CM_ROUTE_TIMEOUT_MS=20000
export DAPL_CM_ARP_TIMEOUT_MS=10000
export DAPL_UCM_REP_TIME=800
export DAPL_UCM_RTU_TIME=400
export DAPL_UCM_RETRY=30
export DAPL_ACK_RETRY=30
export DAPL_ACK_TIME=30
export I_MPI_DEVICE="rdssm:ofa-v2-ib0"

# General settings

set -xuv
date

ECEARTH=/gpfs/projects/ecm86/models/ecearth/v2.3.0
SCRIPTDIR=$ECEARTH/setup
BINDIR=$ECEARTH/bin
PATH=${SCRIPTDIR}:${PATH}
TESTDATADIR=$ECEARTH/inidata/testdata
WRITINGDIR=/gpfs/scratch/ecm86/$USER/test
RUN_dir=$WRITINGDIR/run_t159l62_orca1_${LSB_JOBID}

# Resolution dependent part

IFS_resolution=T159L62
RUN_length_prev=0
FCLENGTH=48
WAM=false
NPROCIFS=32
NPROCNEMOX=4
NPROCNEMOY=4
NPROCNEMO=$((NPROCNEMOX*NPROCNEMOY))
NPROCOASIS=1
COUPFREQ=3
NPROMA=40
LLAST=TRUE
NPRTRVIFS=1
EXPID=b0if
LFASTRES=FALSE
STARTDATE=19900101
ADVSCH=cen2
NFRP=6
LCMIP5=TRUE
NRCP=0

OASIS3OLDWEIGHTS=true
OASIS3WEIGHTSDIR=$TESTDATADIR/oasis3/weights/big_endian/${IFS_resolution}_orca1

export LOCAL_DEFINITION_TEMPLATES=$ECEARTH/sources/ifs/src/emos/local_definitions
export MBX_SIZE=512000000

# Bin path

IFSEXE=$BINDIR/ifsMASTER
OASIS3EXE=$BINDIR/oasis3.MPI1.x
NEMOEXE=$BINDIR/opa_exe.ORCA1_OASIS3.$NPROCNEMOX.$NPROCNEMOY

# Run setup

if [[ ${RUN_length_prev} = '0' ]]; then
  mkdir -p ${RUN_dir} || true
  rm -fr ${RUN_dir}/*
else
  \rm -f ${RUN_dir}/guarf0000000000 ${RUN_dir}/gxtrarf0000000000 ${RUN_dir}/srf0000000000 ${RUN_dir}/surf0000000000 ${RUN_dir}/tmrf0000000000
  \rm -f ${RUN_dir}/anaisout
  \rm -f ${RUN_dir}/mesh_zgr.nc ${RUN_dir}/mesh_hgr.nc ${RUN_dir}/mask.nc ${RUN_dir}/damping.coeff_[0-9][0-9][0-9][0-9].nc
fi

cd ${RUN_dir}

EXPVER=$(setupifs.ksh -r ${IFS_resolution} -e $ECEARTH/sources -d ${RUN_length_prev} -l $FCLENGTH -n $NPROCIFS -w $WAM -c $COUPFREQ -p $NPROMA -o .$LLAST. -0 $TESTDATADIR -1 $NPRTRVIFS -2 ${EXPID} -3 .$LFASTRES. -4 $NFRP -5 .${LCMIP5}. -6 $NRCP)

if [[ $OASIS3OLDWEIGHTS = false ]]; then
  setupoasis3.ksh -r ${IFS_resolution} -g ORCA1 -e $ECEARTH/sources -l $FCLENGTH -n $NPROCIFS -p $NPROCNEMO -c $COUPFREQ -0 $TESTDATADIR -1 $STARTDATE
else
  setupoasis3.ksh -r ${IFS_resolution} -g ORCA1 -e $ECEARTH/sources -l $((${FCLENGTH}-${RUN_length_prev})) -n $NPROCIFS -p $NPROCNEMO -c $COUPFREQ -w -W $OASIS3WEIGHTSDIR -0 $TESTDATADIR -1 $STARTDATE
fi

if [[ ${RUN_length_prev} != '0' ]]; then
  setupnemo.ksh -g ORCA1 -e ${ECEARTH}/sources -l ${FCLENGTH} -p ${RUN_length_prev} -0 $TESTDATADIR -1 $STARTDATE -2 $ADVSCH
else
  setupnemo.ksh -g ORCA1 -e ${ECEARTH}/sources -l ${FCLENGTH} -0 $TESTDATADIR -1 $STARTDATE -2 $ADVSCH
fi

# Execution

export OASIS3=yes
export OASIS3DEBUGLEVEL=2
export DR_HOOK_IGNORE_SIGNALS='-1'

ulimit -s unlimited

t1=$(date +%s)

mpirun -np ${NPROCOASIS} ${OASIS3EXE} : \
       -np ${NPROCIFS} ${IFSEXE} -v ecmwf -e ${EXPID} : \
       -np ${NPROCNEMO} ${NEMOEXE}

t2=$(date +%s)

tr=$(date -d "0 -$t1 sec + $t2 sec" +%T)

date
