#!/bin/bash
###############################################################################
#                              RUN EC_EARTH
###############################################################################
#$ -S /bin/sh
#$ -N run-oce
#$ -V
#$ -cwd
#$ -pe orte 16
#

# modules setup
source /opt/modules/init/bash
module load intel/13.0.1
module load openmpi/1.6-intel
# tell OpenMPI to only use infiniband
export OMPI_MCA_btl=sm,self,openib
module load szip/2.1
module load hdf5/1.8.10
module load netcdf/4.1.3

set -xuve

# =============================================================================
# *** BEGIN User configuration
# =============================================================================

# -----------------------------------------------------------------------------
# *** General configuration
# -----------------------------------------------------------------------------
# Experiment name (exactly 4 letters!)
exp_name=ECE3

# Simulation start and end date. Use any (reasonable) syntax you want.
run_start_date="1990-01-01"
run_end_date="${run_start_date} + 5 days"

# Set $force_run_from_scratch to 'true' if you want to force this run to start
# from scratch, possibly ignoring any restart files present in the run
# directory. Leave set to 'false' otherwise.
# NOTE: If set to 'true' the run directory $run_dir is cleaned!
force_run_from_scratch=false

# Resolution
nem_grid=ORCA1L46
lim=LIM3

# Restart frequency. Use any (reasonable) number and time unit you want.
# For runs without restart, leave this variable empty
rst_freq="1 month"

# Directories
ecearth_base_dir=/scratch/cfu/models/nemo/ece-v3.0.1
start_dir=${PWD}
ctrl_file_dir=${ecearth_base_dir}/setup/ctrl
run_dir=/scratch/cfu/$USER/test/run_${nem_grid}_oce_${JOB_ID}
ini_data_dir=${ecearth_base_dir}/inidata
bin_dir=${ecearth_base_dir}/bin

# Architecture
build_arch=ecconf

# This file is used to store information about restarts
ece_info_file="ece.info"

# -----------------------------------------------------------------------------
# *** Read platform dependent configuration
# -----------------------------------------------------------------------------

# -----------------------------------------------------------------------------
# *** NEMO/LIM configuration
# -----------------------------------------------------------------------------

nem_version=3.3.1
lim_version=$(echo $lim | cut -d 'M' -f 2)

case ${nem_grid} in
     ORCA1*)   nem_time_step_sec=3600 ;;
     ORCA025*) nem_time_step_sec=1200 ;;
     *) error "Unsupported grid type: ${nem_grid}"
    ;;
esac
lim_time_step_sec=3600

nem_res_hor=$(echo ${nem_grid} | sed 's:ORCA\([0-9]*\)L\([0-9]*\):\1:')
nem_res_ver=$(echo ${nem_grid} | sed 's:ORCA\([0-9]*\)L\([0-9]*\):\2:')

nem_exe_file=${bin_dir}/${nem_grid}_${lim}/nemo.exe

nem_numproc=16

advsch=tvd
rstctl=1


# =============================================================================
# *** END of User configuration
# =============================================================================

# =============================================================================
# *** This is where the code begins ...
# =============================================================================

# -----------------------------------------------------------------------------
# *** Define some helper functions
# -----------------------------------------------------------------------------

function info
{
    echo "*II* $@"
}

function error
{
    echo "*EE* $@"
    exit 1
}

function cleanup
{
    # Cleanup function, triggered by the traps set in the run-script
    set +u

    if [ -r ${tempfile} ]
    then
        rm -f ${tempfile}
    fi

    if [ -r ${stdout_file} -a -d ${run_dir} ]
    then
        cp ${stdout_file} ${run_dir}
    fi
}

# -----------------------------------------------------------------------------
# *** Make sure to clean up on exit
# -----------------------------------------------------------------------------

trap 'cleanup' EXIT SIGHUP SIGINT SIGTERM

# -----------------------------------------------------------------------------
# *** Create the run dir if necessary and go there
#     Everything is done from here.
# -----------------------------------------------------------------------------
if [ ! -d ${run_dir} ]
then
    mkdir -p ${run_dir}
fi
cd ${run_dir}

# -----------------------------------------------------------------------------
# *** Determine the time span of this run and whether it's a restart leg
# -----------------------------------------------------------------------------

# Regularise the format of the start and end date of the simulation
run_start_date=$(date -uR -d "${run_start_date}")
run_end_date=$(date -uR -d "${run_end_date}")

# Check for restart information file and set the current leg start date
#   Ignore restart information file if force_run_from_scratch is true
if ! $force_run_from_scratch && [ -r ${ece_info_file} ]
then
    leg_is_restart=true
    . ./${ece_info_file}
    leg_start_date=${leg_end_date}
    leg_number=$((leg_number+1))
else
    leg_is_restart=false
    leg_start_date=${run_start_date}
    leg_number=1
fi

# Compute the end date of the current leg
if [ -n "${rst_freq}" ]
then
    leg_end_date=$(date -uR -d "${leg_start_date} + ${rst_freq}")
else
    leg_end_date=${run_end_date}
fi

if [ $(date -d "${leg_end_date}" +%s) -gt $(date -d "${run_end_date}" +%s) ]
then
    leg_end_date=${run_end_date}
fi

# Some time variables needed later
leg_length_sec=$(( $(date -d "${leg_end_date}" +%s) - $(date -d "${leg_start_date}" +%s) ))
leg_start_sec=$(( $(date -d "${leg_start_date}" +%s) - $(date -d "${run_start_date}" +%s) ))
leg_end_sec=$(( $(date -d "${leg_end_date}" +%s) - $(date -d "${run_start_date}" +%s) ))
leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d)
leg_start_date_yyyy=$(date -u -d "${leg_start_date}" +%Y)
leg_end_date_yyyy=$(date -u -d "${leg_end_date}" +%Y)

# Check whether there's actually time left to simulate - exit otherwise
if [ ${leg_length_sec} -le 0 ]
then
    info "Leg start date equal to or after end of simulation."
    info "Nothing left to do. Exiting."
    exit 0
fi

# -----------------------------------------------------------------------------
# *** Prepare the run directory for a run from scratch
# -----------------------------------------------------------------------------
if ! $leg_is_restart
then
    # -------------------------------------------------------------------------
    # *** This is no restart, hence, clean the run dir and link the executables
    # -------------------------------------------------------------------------
    rm -fr ${run_dir}/*

    ln -s ${nem_exe_file}

    # -------------------------------------------------------------------------
    # *** Files needed for NEMO (linked)
    # -------------------------------------------------------------------------

    # Various stuff
    ln -s ${ini_data_dir}/nemo/${nem_grid}/bathy_meter.nc
    ln -s ${ini_data_dir}/nemo/${nem_grid}/coordinates.nc
    ln -s ${ini_data_dir}/nemo/${nem_grid}/ahmcoef

    # Initial data
    ln -s ${ini_data_dir}/nemo/${nem_grid}/temperature_*.nc .
    ln -s ${ini_data_dir}/nemo/${nem_grid}/salinity_*.nc .

    # IOM files
    ln -s ${ctrl_file_dir}/xmlio_server.def
    ln -s ${ctrl_file_dir}/iodef.xml

    # NEMO forcing files
    case ${nem_res_hor} in
          1)  nem_forcing=${ini_data_dir}/nemo/${nem_grid}/climate/DFS4.3
              if [ ${leg_start_date_yyyy} -le 1978 ]; then
                  ln -s ${nem_forcing}/precip/precip_DFS4.3_1958_to_1978.nc precip_DFS4.3_${leg_start_date_yyyy}.nc
              else
                  ln -s ${nem_forcing}/precip/precip_DFS4.3_${leg_start_date_yyyy}.nc
              fi

              if [ ${leg_start_date_yyyy} -le 1978 ]; then
                  ln -s ${nem_forcing}/snow/snow_DFS4.3_1958_to_1978.nc snow_DFS4.3_${leg_start_date_yyyy}.nc
              else
                  ln -s ${nem_forcing}/snow/snow_DFS4.3_${leg_start_date_yyyy}.nc
              fi

              if [ ${leg_start_date_yyyy} -le 1983 ]; then
                  ln -s ${nem_forcing}/radlw/radlw_DFS4.3_1958_to_1983.nc radlw_DFS4.3_${leg_start_date_yyyy}.nc
              else
                  ln -s ${nem_forcing}/radlw/radlw_DFS4.3_${leg_start_date_yyyy}.nc
              fi

              if [ ${leg_start_date_yyyy} -le 1983 ]; then
                  ln -s ${nem_forcing}/radsw/radsw_DFS4.3_1958_to_1983.nc radsw_DFS4.3_${leg_start_date_yyyy}.nc
              else
                  ln -s ${nem_forcing}/radsw/radsw_DFS4.3_${leg_start_date_yyyy}.nc
              fi

              ln -s ${nem_forcing}/q2/q2_DFS4.3_${leg_start_date_yyyy}_sht.nc*
              ln -s ${nem_forcing}/t2/t2_DFS4.3_${leg_start_date_yyyy}_sht.nc
              ln -s ${nem_forcing}/u10/u10_DFS4.3_${leg_start_date_yyyy}_sht.nc
              ln -s ${nem_forcing}/v10/v10_DFS4.3_${leg_start_date_yyyy}_sht.nc
              ln -s ${nem_forcing}/weights/weight_bicub_320x161-ORCA1.nc
              ln -s ${nem_forcing}/weights/weight_bilin_320x161-ORCA1.nc
              ln -s ${nem_forcing}/weights/weight_bilin_192x94-ORCA1.nc
              ;;
          *)  error "Unsupported horizontal resolution (NEMO-standalone): ${nem_res_hor}"
              ;;
    esac

fi # ! $leg_is_restart

# -----------------------------------------------------------------------------
# *** Create some control files
# -----------------------------------------------------------------------------

. ${ctrl_file_dir}/namelist.nemo-standalone-${nem_grid}.sh          > namelist
. ${ctrl_file_dir}/namelist.lim${lim_version}-ORCA${nem_res_hor}.sh > namelist_ice

# -----------------------------------------------------------------------------
# *** Link the appropriate NEMO restart files of the previous leg
# -----------------------------------------------------------------------------
if $leg_is_restart
then
    ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec )))
    for n in $(seq 0 $((nem_numproc-1)))
    do
        np=$(printf %04d ${n})
        ln -fs ${exp_name}_${ns}_restart_oce_${np}.nc restart_oce_${np}.nc
        ln -fs ${exp_name}_${ns}_restart_ice_${np}.nc restart_ice_${np}.nc
    done
fi

# -----------------------------------------------------------------------------
# *** Start the run
# -----------------------------------------------------------------------------

# Use the launch function from the platform configuration file
t1=$(date +%s)

### --- PLATFORM DEPENDENT CODE ---
mpirun -np ${nem_numproc} ${nem_exe_file}
### --- END of PLATFORM DEPENDENT CODE ---

t2=$(date +%s)

tr=$(date -d "0 -$t1 sec + $t2 sec" +%T)

# -----------------------------------------------------------------------------
# *** Write the restart control file
# -----------------------------------------------------------------------------
echo "#"                                             | tee -a ${ece_info_file}
echo "# Finished leg at `date '+%F %T'` after ${tr} (hh:mm:ss)" \
                                                     | tee -a ${ece_info_file}
echo "leg_number=${leg_number}"                      | tee -a ${ece_info_file}
echo "leg_start_date=\"${leg_start_date}\""          | tee -a ${ece_info_file}
echo "leg_end_date=\"${leg_end_date}\""              | tee -a ${ece_info_file}

# -----------------------------------------------------------------------------
# *** Platform independent finalising of the run
# -----------------------------------------------------------------------------
# Rename run-files so they are not overwritten in the next leg
filesToCopy="ocean.output"
for file in ${filesToCopy}
do
    cp ${file} ${file}.$(printf %03d $((leg_number)))
done
rm -f ocean.output

# -----------------------------------------------------------------------------
# *** Platform dependent finalising of the run
# -----------------------------------------------------------------------------
finalise

exit 0
