#!/bin/sh
###############################################################################
#                              RUN EC_EARTH
###############################################################################
#BSUB -J run-atm
#BSUB -o run-atm_%J.out
#BSUB -e run-atm_%J.err
#BSUB -n 48
#BSUB -R "span[ptile=16]"
#BSUB -W 00:30
#
###############################################################################

# modules setup
module purge
module load intel
module load openmpi
module load szip
module load hdf5
module load netcdf
module load mkl
export OMPI_MCA_mpool_rdma_rcache_size_limit=1000000000
export OMPI_MCA_btl_openib_receive_queues="S,4096,1024:S,12288,512:S,65536,512"
export OMPI_MCA_btl_openib_ib_min_rnr_timer=25
export OMPI_MCA_btl_openib_ib_timeout=20

# setup the grib_api env.
PATH=/gpfs/projects/ecm86/opt/grib_api-1.9.9/intel-13/bin:$PATH

set -xuve
module list

# =============================================================================
# *** BEGIN User configuration
# =============================================================================

# -----------------------------------------------------------------------------
# Switch between LOW, STANDARD and HIGH RESOLUTION by setting:
#
# --------------------------------------------------
#   Parameter          lowres     stdres    highres
# --------------------------------------------------
#   ifs_grid           T159L62      T255       T799
#   nem_grid          ORCA1L46  ORCA1L46 ORCA025L46
#   ifs_time_step_sec     3600      2700        720
#   nem_time_step_sec     3600      3600       1200
# --------------------------------------------------
#
# Change also:
#   exp_name
#   run_start_date (lo:1990/std:1996/hi:1996)
#   ifs_numproc
#   nem_numproc
#   run_dir
# and make sure to use the appropriate NEMO executable
# -----------------------------------------------------------------------------

# -----------------------------------------------------------------------------
# *** General configuration
# -----------------------------------------------------------------------------
# Experiment name (exactly 4 letters!)
exp_name=ECE3

# Simulation start and end date. Use any (reasonable) syntax you want.
run_start_date="1990-01-01"
run_end_date="${run_start_date} + 5 days"

# Set $force_run_from_scratch to 'true' if you want to force this run to start
# from scratch, possibly ignoring any restart files present in the run
# directory. Leave set to 'false' otherwise.
# NOTE: If set to 'true' the run directory $run_dir is cleaned!
force_run_from_scratch=false

# Resolution
ifs_grid=T255L62

# Restart frequency. Use any (reasonable) number and time unit you want.
# For runs without restart, leave this variable empty
rst_freq="1 month"

# Coupling frequency [hrs]
cpl_freq_hrs=0

# Directories
ecearth_base_dir=/gpfs/projects/ecm86/models/ecearth/v3.0
start_dir=${PWD}
ctrl_file_dir=${ecearth_base_dir}/setup/ctrl
run_dir=/gpfs/scratch/ecm86/$USER/test/run_${ifs_grid}_atm_${LSB_JOBID}
ini_data_dir=${ecearth_base_dir}/inidata
bin_dir=${ecearth_base_dir}/bin

# Architecture
build_arch=ecconf

# This file is used to store information about restarts
ece_info_file="ece.info"

# -----------------------------------------------------------------------------
# *** Read platform dependent configuration
# -----------------------------------------------------------------------------
grib_api_path=/gpfs/projects/ecm86/opt/grib_api-1.9.9/intel-13

export GRIB_DEFINITION_PATH=${grib_api_path}/share/definitions
export GRIB_SAMPLES_PATH=${grib_api_path}/ifs_samples/grib1
export GRIB_BIN_PATH=${grib_api_path}/bin

# -----------------------------------------------------------------------------
# *** IFS configuration
# -----------------------------------------------------------------------------

ifs_version=36r4

case ${ifs_grid} in
     T159*)  ifs_time_step_sec=3600 ;;
     T255*)  ifs_time_step_sec=2700 ;;
     T511*)  ifs_time_step_sec=900 ;;
     T799*)  ifs_time_step_sec=720  ;;
     *) error "Unsupported grid type: ${ifs_grid}"
     ;;
esac

ifs_output_freq=$(( 6 * 3600 / ifs_time_step_sec ))
ifs_di_freq=$(( 24 * 3600 / ifs_time_step_sec ))
ifs_ddh_freq=$(( 120 * 3600 / ifs_time_step_sec ))

ifs_res_hor=$(echo ${ifs_grid} | sed 's:T\([0-9]*\)L\([0-9]*\):\1:')
ifs_res_ver=$(echo ${ifs_grid} | sed 's:T\([0-9]*\)L\([0-9]*\):\2:')

ifs_numproc=48

ifs_exe_file=${bin_dir}/ifsmaster-${build_arch}

ifs_lastout=true

ifs_cmip5=FALSE
ifs_cmip5_rcp=0
ifs_volcanoes=FALSE
ifs_ncmip5fixyr=0

nprtrv=1

# =============================================================================
# *** END of User configuration
# =============================================================================

# =============================================================================
# *** This is where the code begins ...
# =============================================================================

# -----------------------------------------------------------------------------
# *** Define some helper functions
# -----------------------------------------------------------------------------

function info
{
    echo "*II* $@"
}

function error
{
    echo "*EE* $@"
    exit 1
}

function cleanup
{
    # Cleanup function, triggered by the traps set in the run-script
    set +u

    if [ -r ${tempfile} ]
    then
        rm -f ${tempfile}
    fi

    if [ -r ${stdout_file} -a -d ${run_dir} ]
    then
        cp ${stdout_file} ${run_dir}
    fi
}

# -----------------------------------------------------------------------------
# *** Make sure to clean up on exit
# -----------------------------------------------------------------------------

trap 'cleanup' EXIT SIGHUP SIGINT SIGTERM

# -----------------------------------------------------------------------------
# *** Create the run dir if necessary and go there
#     Everything is done from here.
# -----------------------------------------------------------------------------
if [ ! -d ${run_dir} ]
then
    mkdir -p ${run_dir}
fi
cd ${run_dir}

# -----------------------------------------------------------------------------
# *** Determine the time span of this run and whether it's a restart leg
# -----------------------------------------------------------------------------

# Regularise the format of the start and end date of the simulation
run_start_date=$(date -uR -d "${run_start_date}")
run_end_date=$(date -uR -d "${run_end_date}")

# Check for restart information file and set the current leg start date
#   Ignore restart information file if force_run_from_scratch is true
if ! $force_run_from_scratch && [ -r ${ece_info_file} ]
then
    leg_is_restart=true
    . ./${ece_info_file}
    leg_start_date=${leg_end_date}
    leg_number=$((leg_number+1))
else
    leg_is_restart=false
    leg_start_date=${run_start_date}
    leg_number=1
fi

# Compute the end date of the current leg
if [ -n "${rst_freq}" ]
then
    leg_end_date=$(date -uR -d "${leg_start_date} + ${rst_freq}")
else
    leg_end_date=${run_end_date}
fi

if [ $(date -d "${leg_end_date}" +%s) -gt $(date -d "${run_end_date}" +%s) ]
then
    leg_end_date=${run_end_date}
fi

# Some time variables needed later
leg_length_sec=$(( $(date -d "${leg_end_date}" +%s) - $(date -d "${leg_start_date}" +%s) ))
leg_start_sec=$(( $(date -d "${leg_start_date}" +%s) - $(date -d "${run_start_date}" +%s) ))
leg_end_sec=$(( $(date -d "${leg_end_date}" +%s) - $(date -d "${run_start_date}" +%s) ))
leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d)
leg_start_date_yyyy=$(date -u -d "${leg_start_date}" +%Y)
leg_end_date_yyyy=$(date -u -d "${leg_end_date}" +%Y)

# Check whether there's actually time left to simulate - exit otherwise
if [ ${leg_length_sec} -le 0 ]
then
    info "Leg start date equal to or after end of simulation."
    info "Nothing left to do. Exiting."
    exit 0
fi

# -----------------------------------------------------------------------------
# *** Prepare the run directory for a run from scratch
# -----------------------------------------------------------------------------
if ! $leg_is_restart
then
    # -------------------------------------------------------------------------
    # *** This is no restart, hence, clean the run dir and link the executables
    # -------------------------------------------------------------------------
    rm -fr ${run_dir}/*

    ln -s ${ifs_exe_file}

    # -------------------------------------------------------------------------
    # *** Files needed for IFS (linked)
    # -------------------------------------------------------------------------

    # Initial data
    ln -s \
    ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMGGECE3INIUA \
                                                        ICMGG${exp_name}INIUA
    ln -s \
    ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMGGECE3INIT \
                                                        ICMGG${exp_name}INIT
    ln -s \
    ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMSHECE3INIT \
                                                        ICMSH${exp_name}INIT
    # Other stuff
    ln -s ${ini_data_dir}/ifs/postins
    ln -s ${ini_data_dir}/ifs/dirlist
    ln -s ${ini_data_dir}/ifs/rtables/* .

    # Run-off file
    ln -s ${ini_data_dir}/ifs/${ifs_grid}/runoff_maps.txt

fi # ! $leg_is_restart

# -----------------------------------------------------------------------------
# *** Create some control files
# -----------------------------------------------------------------------------

# IFS, NEMO, LIM namelist and OASIS namcouple files
. ${ctrl_file_dir}/namelist.ifs-${ifs_grid}.sh > fort.4

# -----------------------------------------------------------------------------
# *** IFS climatology data
# -----------------------------------------------------------------------------
icmclfile=${ini_data_dir}/ifs/${ifs_grid}/climate/ICMCL
tempfile=tmp.$$

# Create data for december, the year before the leg starts
grib_set \
-s dataDate=$(printf "%04d" $((leg_start_date_yyyy-1)))1215 ${icmclfile}-12 \
                                                       ICMCL${exp_name}INIT

# Create data for all month in the years of the leg
for y in $(seq ${leg_start_date_yyyy} ${leg_end_date_yyyy})
do
    yy=$(printf "%04d" $y)
    for m in $(seq 1 12)
    do
        mm=$(printf "%02d" $m)
        grib_set -s dataDate=${yy}${mm}15 ${icmclfile}-${mm} ${tempfile}
        cat ${tempfile} >> ICMCL${exp_name}INIT
    done
done

# Create data for january, the year after the leg ends
grib_set -s dataDate=$(printf "%04d" $((leg_end_date_yyyy+1)))0115 ${icmclfile}-01 \
                                                                          ${tempfile}
cat ${tempfile} >> ICMCL${exp_name}INIT

ln -sf ${ini_data_dir}/ifs/${ifs_grid}/climate/ICMSEAECE3INIT ICMSEA${exp_name}INIT

# Clean up
rm -f ${tempfile}

# -----------------------------------------------------------------------------
# *** Start the run
# -----------------------------------------------------------------------------
export DR_HOOK_IGNORE_SIGNALS='-1'
export OASIS3=simulate
export OASIS3DEBUGLEVEL=3

### --- PLATFORM DEPENDENT CODE ---
ulimit -s unlimited
### --- END of PLATFORM DEPENDENT CODE ---

t1=$(date +%s)

### --- PLATFORM DEPENDENT CODE ---
mpirun -np ${ifs_numproc} ${ifs_exe_file} -v ecmwf -e ${exp_name}
### --- END of PLATFORM DEPENDENT CODE ---

t2=$(date +%s)

tr=$(date -d "0 -$t1 sec + $t2 sec" +%T)

# -----------------------------------------------------------------------------
# *** Write the restart control file
# -----------------------------------------------------------------------------
echo "#"                                             | tee -a ${ece_info_file}
echo "# Finished leg at `date '+%F %T'` after ${tr} (hh:mm:ss)" \
                                                     | tee -a ${ece_info_file}
echo "leg_number=${leg_number}"                      | tee -a ${ece_info_file}
echo "leg_start_date=\"${leg_start_date}\""          | tee -a ${ece_info_file}
echo "leg_end_date=\"${leg_end_date}\""              | tee -a ${ece_info_file}

# -----------------------------------------------------------------------------
# *** Platform independent finalising of the run
# -----------------------------------------------------------------------------
# Rename run-files so they are not overwritten in the next leg
filesToCopy="NODE.001_01   rcf   $oas_cpl_flds"
for file in ${filesToCopy}
do
    cp ${file} ${file}.$(printf %03d $((leg_number)))
done
rm -f NODE.001_01

# -----------------------------------------------------------------------------
# *** Platform dependent finalising of the run
# -----------------------------------------------------------------------------
finalise

exit 0
