#@ shell           = /usr/bin/ksh 

        # Specifies the shell that parses the script. If not  
        # specified, your login shell will be used. 

#@ class           = np 

        # Specifies that your job should be run in the class (queue) 
        # np, which is used to run parallel jobs. 

#@ job_type        = parallel 

        # Specifies that your job is parallel 
       
#@ job_name        = ecearth3

        # Assigns the specified name to the request 
    
#@ output          = $(job_name).$(schedd_host).$(jobid).out 

        # Specifies the name and location of STDOUT. If not given, the 
        # default is /dev/null.  The file will be written in the 
        # submitting directory, by default.  

#@ error           = $(job_name).$(schedd_host).$(jobid).err 

        # Specifies the name and location of STDERR. If not given, the 
        # default is /dev/null.  The file will be written in the 
        # submitting directory, by default. 

#@ notification    = always  

        # Specifies that email should be sent only if the job fails.  
        # Other options include: always, complete, start, and 
        # never. The default is notification = complete. 

#@ resources       = ConsumableCpus(1) ConsumableMemory(1200mb) 

        # Specifies quantities of the resources which will be 'consumed' 
        # by each process/task of a job step. ConsumableCpus and 
        # ConsumableMemory must be specified with a value which is 
        # greater than zero.   
        # The value ConsumableCpus indicates the number of threads per 
        # (MPI) process/task. This job will be run using 1 thread 
        # per MPI process/task.   
        # The value ConsumableMemory specifies the (real) memory per 
        # (MPI) process/task which is shared between threads. This job 
        # will request 100 megabytes of real memory.   

#@ node             = 2 

        # Specifies that your job requires 2 nodes 
    
#@ tasks_per_node   = 32

        # Specifies the number of (MPI) processes/tasks of a parallel job  
        # you want to run per node. In this example your job requires  
        # 32 processes/tasks per node.   

#@ cpu_limit        = 12:00:00

        # Specifies the maximum CPU time in HH:MM:SS used by any single process.  
        # In this job 1 MPI process/task can use up to a total of 1 min CPU time. 
        # Note: All limits are capped by those specified in the class. 

#@ wall_clock_limit = 24:00:00

        # Specifies that your job requires HH:MM:SS of wall clock 
        # time. This job requires 3 min of wall clock time. 
        # Note: All limits are capped by those specified in the class. 

#@ queue 

set -vuex
alias gdate='/usr/local/apps/coreutils/8.20/bin/date'

# =============================================================================
# *** BEGIN User configuration
# =============================================================================

# -----------------------------------------------------------------------------
# *** General configuration
# -----------------------------------------------------------------------------
# Experiment name (exactly 4 letters!)
exp_name=ECE3

# Simulation start and end date. Use any (reasonable) syntax you want.
run_start_date="1990-01-01"
run_end_date="${run_start_date} + 1 years"

# Set $force_run_from_scratch to 'true' if you want to force this run to start
# from scratch, possibly ignoring any restart files present in the run
# directory. Leave set to 'false' otherwise.
# NOTE: If set to 'true' the run directory $run_dir is cleaned!
force_run_from_scratch=true

# Resolution
ifs_grid=T255L91
nem_grid=ORCA1L46

# Restart frequency. Use any (reasonable) number and time unit you want.
# For runs without restart, leave this variable empty
rst_freq="2 month"

# Coupling frequency [hrs]
cpl_freq_hrs=3

# Directories
start_dir=${PWD}
ctrl_file_dir=${start_dir}/ctrl

# Architecture
build_arch=ecconf

# This file is used to store information about restarts
ece_info_file="ece.info"

# -----------------------------------------------------------------------------
# *** Read platform dependent configuration
# -----------------------------------------------------------------------------
# Configure paths for building/running EC-Earth
ecearth_source_dir=${PERM}/ecearth3/sources
run_dir=${TEMP}/${USER}/run/${exp_name}
ini_data_dir=${PERM}/ecearth3-release

# File for standard output.
stdout_file=${start_dir}/out/out.out

# Configure grib api paths
export GRIB_DEFINITION_PATH=/usr/local/lib/metaps/lib/grib_api/1.9.16/share/definitions
export GRIB_SAMPLES_PATH=/usr/local/lib/metaps/lib/grib_api/1.9.16/ifs_samples/grib1
export GRIB_BIN_PATH=/usr/local/lib/metaps/lib/grib_api/1.9.16/bin64


# Configure and load modules
pre_load_modules_cmd=""
module_list=""


# -----------------------------------------------------------------------------
# *** IFS configuration
# -----------------------------------------------------------------------------

ifs_version=36r4

ifs_time_step_sec=2700

ifs_output_freq=$(( 6 * 3600 / ifs_time_step_sec ))
ifs_di_freq=$(( 24 * 3600 / ifs_time_step_sec ))
ifs_ddh_freq=$(( 120 * 3600 / ifs_time_step_sec ))

ifs_res_hor=$(echo ${ifs_grid} | sed 's:T\([0-9]*\)L\([0-9]*\):\1:')
ifs_res_ver=$(echo ${ifs_grid} | sed 's:T\([0-9]*\)L\([0-9]*\):\2:')

ifs_numproc=38

ifs_exe_file=${ecearth_source_dir}/ifs-${ifs_version}/bin/ifsmaster-${build_arch}

ifs_lastout=false

ifs_cmip5=TRUE
ifs_cmip5_rcp=0
ifs_volcanoes=TRUE

# -----------------------------------------------------------------------------
# *** NEMO/LIM configuration
# -----------------------------------------------------------------------------

nem_version=3.3.1
lim_version=3

nem_time_step_sec=3600
lim_time_step_sec=3600

nem_res_hor=$(echo ${nem_grid} | sed 's:ORCA\([0-9]*\)L\([0-9]*\):\1:')
nem_res_ver=$(echo ${nem_grid} | sed 's:ORCA\([0-9]*\)L\([0-9]*\):\2:')

nem_exe_file=${ecearth_source_dir}/nemo-${nem_version}/CONFIG/${nem_grid}_LIM${lim_version}/BLD/bin/nemo.exe

nem_numproc=18

# -----------------------------------------------------------------------------
# *** OASIS configuration
# -----------------------------------------------------------------------------

oas_namcut_script=${ecearth_source_dir}/util/namcouple-split/namsplit.pl

oas_numproc=8

# Flux correction that compensates for the mismatch between P-E over the ocean,
# and run-off from land (necessary to avoid sea level rise).
# The RUNOFF and CALVING coupling fields are multiplied by this factor. See
# documentation on the EC-Earth 3 Wiki for more details.
oas_qs_fluxcorr=0.985
oas_rnf_fluxcorr=0.87

oas_exe_file=${ecearth_source_dir}/oasis3/${build_arch}/bin/oasis3.MPI1.x

# Restart files for the coupling fields
oas_cpl_flds="TAUX_OCE TAUY_OCE TAUX_ICE TAUY_ICE \
              QS___OCE QS___ICE QNS__OCE QNS__ICE DQDT_ICE \
              PRCP_LIQ PRCP_SOL EVAP_TOT EVAP_ICE RNF__OCE CALV_OCE \
              O_SSTSST O_TepIce O_AlbIce OIceFrac O_IceTck O_SnwTck"

# =============================================================================
# *** END of User configuration
# =============================================================================

# =============================================================================
# *** This is where the code begins ...
# =============================================================================

# -----------------------------------------------------------------------------
# *** Define some helper functions
# -----------------------------------------------------------------------------

function info
{
    echo "*II* $@"
}

function error
{
    echo "*EE* $@"
    exit 1
}

function cleanup
{
    # Cleanup function, triggered by the traps set in the run-script
    set +u

    if [ -r ${tempfile} ]
    then
        rm -f ${tempfile}
    fi

    if [ -r ${stdout_file} -a -d ${run_dir} ]
    then
        cp ${stdout_file} ${run_dir}
    fi
}

# -----------------------------------------------------------------------------
# *** Make sure to clean up on exit
# -----------------------------------------------------------------------------

trap 'cleanup' EXIT SIGHUP SIGINT SIGTERM

# -----------------------------------------------------------------------------
# *** Create the run dir if necessary and go there
#     Everything is done from here.
# -----------------------------------------------------------------------------
if [ ! -d ${run_dir} ]
then
    mkdir -p ${run_dir}
fi
cd ${run_dir}

# -----------------------------------------------------------------------------
# *** Determine the time span of this run and whether it's a restart leg
# -----------------------------------------------------------------------------

# Regularise the format of the start and end date of the simulation
run_start_date=$(gdate -uR -d "${run_start_date}")
run_end_date=$(gdate -uR -d "${run_end_date}")

# Check for restart information file and set the current leg start date
#   Ignore restart information file if force_run_from_scratch is true
if ! $force_run_from_scratch && [ -r ${ece_info_file} ]
then
    leg_is_restart=true
    . ./${ece_info_file}
    leg_start_date=${leg_end_date}
    leg_number=$((leg_number+1))
else
    leg_is_restart=false
    leg_start_date=${run_start_date}
    leg_number=1
fi

# Compute the end date of the current leg
if [ -n "${rst_freq}" ]
then
    leg_end_date=$(gdate -uR -d "${leg_start_date} + ${rst_freq}")
else
    leg_end_date=${run_end_date}
fi

if [ $(gdate -d "${leg_end_date}" +%s) -gt $(gdate -d "${run_end_date}" +%s) ]
then
    leg_end_date=${run_end_date}
fi

# Some time variables needed later
leg_length_sec=$(( $(gdate -d "${leg_end_date}" +%s) - $(gdate -d "${leg_start_date}" +%s) ))
leg_start_sec=$(( $(gdate -d "${leg_start_date}" +%s) - $(gdate -d "${run_start_date}" +%s) ))
leg_end_sec=$(( $(gdate -d "${leg_end_date}" +%s) - $(gdate -d "${run_start_date}" +%s) ))
leg_start_date_yyyymmdd=$(gdate -d "${leg_start_date}" +%Y%m%d)
leg_start_date_yyyy=$(gdate -d "${leg_start_date}" +%Y)
leg_end_date_yyyy=$(gdate -d "${leg_end_date}" +%Y)

# Check whether there's actually time left to simulate - exit otherwise
if [ ${leg_length_sec} -le 0 ]
then
    info "Leg start date equal to or after end of simulation."
    info "Nothing left to do. Exiting."
    exit 0
fi

# -----------------------------------------------------------------------------
# *** Prepare the run directory for a run from scratch
# -----------------------------------------------------------------------------
if ! $leg_is_restart
then
    # -------------------------------------------------------------------------
    # *** This is no restart, hence, clean the run dir and link the executables
    # -------------------------------------------------------------------------
    rm -fr ${run_dir}/*

    ln -s ${ifs_exe_file}
    ln -s ${nem_exe_file}
    ln -s ${oas_exe_file}

    # -------------------------------------------------------------------------
    # *** Files needed for IFS (linked)
    # -------------------------------------------------------------------------

    # Initial data
    ln -s \
    ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMGGECE3INIUA \
                                                        ICMGG${exp_name}INIUA
    ln -s \
    ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMGGECE3INIT \
                                                        ICMGG${exp_name}INIT
    ln -s \
    ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMSHECE3INIT \
                                                        ICMSH${exp_name}INIT
    # Other stuff
    ln -s ${ini_data_dir}/ifs/rtables/* .

    # Output control (ppt files)
    mkdir postins
    cp ${ctrl_file_dir}/ppt* postins/
    f_min=$(( ifs_output_freq * ifs_time_step_sec / 60 ))
    for t in $(seq $f_min $f_min 1439)
    do
        hh=$(printf %02d $((t/60)) )
        mm=$(printf %02d $((t%60)) )
        ln -s pptdddddd0000 postins/pptdddddd$hh$mm
    done 
    /bin/ls -1 postins/* > dirlist

    # Run-off file
    ln -s ${ini_data_dir}/ifs/${ifs_grid}/runoff_maps.txt

    # -------------------------------------------------------------------------
    # *** Files needed for NEMO (linked)
    # -------------------------------------------------------------------------

    # Various stuff
    ln -s ${ini_data_dir}/nemo/${nem_grid}/bathy_meter.nc
    ln -s ${ini_data_dir}/nemo/${nem_grid}/coordinates.nc
    ln -s ${ini_data_dir}/nemo/${nem_grid}/ahmcoef

    # IOM files
    ln -s ${ctrl_file_dir}/xmlio_server.def
    ln -s ${ctrl_file_dir}/iodef.xml

    # -------------------------------------------------------------------------
    # *** Files needed for OASIS (linked)
    # -------------------------------------------------------------------------

    # Name table file
    ln -s ${ini_data_dir}/oasis/cf_name_table.txt

    oas_grid_dir=${ini_data_dir}/oasis/T${ifs_res_hor}-ORCA${nem_res_hor}

    # Grid definition files
    ln -s ${oas_grid_dir}/areas.nc
    ln -s ${oas_grid_dir}/grids.nc
    ln -s ${oas_grid_dir}/masks.nc

    # Weight files
    case ${ifs_res_hor} in
        159)  oas_agrd=080
              ;;
        255)  oas_agrd=128
              ;;
        511)  oas_agrd=256
              ;;
        799)  oas_agrd=400
              ;;
        *)    error "Unsupported horizontal resolution (IFS): ${ifs_res_hor}"
              ;;
    esac

    case ${nem_res_hor} in
          1)  oas_ogrd=O1t0
              ;;
        025)  oas_ogrd=Ot25
              ;;
          *)  error "Unsupported horizontal resolution (NEMO): ${nem_res_hor}"
              ;;
    esac

    for n in $(seq 0 $((oas_numproc-1)) )
    do
        ln -s \
        ${oas_grid_dir}/rmp_A${oas_agrd}_to_${oas_ogrd}_GAUSWGT.nc \
                        rmp_A${oas_agrd}_to_${oas_ogrd}_GAUSWGT_${n}.nc
        ln -s \
        ${oas_grid_dir}/rmp_${oas_ogrd}_to_L${oas_agrd}_GAUSWGT.nc \
                        rmp_${oas_ogrd}_to_L${oas_agrd}_GAUSWGT_${n}.nc
    done

    for f in ${oas_cpl_flds}
    do
        cp ${oas_grid_dir}/rst/$f .
    done

fi # ! $leg_is_restart

# -----------------------------------------------------------------------------
# *** Create some control files
# -----------------------------------------------------------------------------

# IFS, NEMO, LIM namelist and OASIS namcouple files
case ${ifs_grid} in
    T255L91) . ${ctrl_file_dir}/namelist.ifs-T255L91.sh > fort.4
             ;;
    *)       . ${ctrl_file_dir}/namelist.ifs-default.sh > fort.4
             ;;
esac
. ${ctrl_file_dir}/namelist.nemo-${nem_grid}.sh                     > namelist
. ${ctrl_file_dir}/namelist.lim${lim_version}-ORCA${nem_res_hor}.sh > namelist_ice
. ${ctrl_file_dir}/namcouple.sh                                     > namcouple

# Split OASIS namcouple file (if necessary)
if (( oas_numproc == 1 ))
then
    ln -s namcouple namcouple_0
elif [ -n "$oas_namcut_script" -a -x $oas_namcut_script ]
then
    ${oas_namcut_script} ${oas_numproc} namcouple
else
    error "Couldn't run script to split the OASIS namcouple file (\$oas_namcut_script)."
fi

# -----------------------------------------------------------------------------
# *** IFS climatology data
# -----------------------------------------------------------------------------
icmclfile=${ini_data_dir}/ifs/${ifs_grid}/climate/ICMCL
tempfile=tmp.$$
grib_set=${GRIB_BIN_PATH}${GRIB_BIN_PATH:+/}grib_set

# Create data for december, the year before the leg starts
${grib_set} \
-s dataDate=$(printf "%04d" $((leg_start_date_yyyy-1)))1215 ${icmclfile}-12 \
                                                       ICMCL${exp_name}INIT

# Create data for all month in the years of the leg
for y in $(seq ${leg_start_date_yyyy} ${leg_end_date_yyyy} )
do
    yy=$(printf "%04d" $y)
    for m in $(seq 1 12)
    do
        mm=$(printf "%02d" $m)
        ${grib_set} -s dataDate=${yy}${mm}15 ${icmclfile}-${mm} ${tempfile}
        cat ${tempfile} >> ICMCL${exp_name}INIT
    done
done

# Create data for january, the year after the leg ends
${grib_set} -s dataDate=$(printf "%04d" $((leg_end_date_yyyy+1)))0115 ${icmclfile}-01 \
                                                                          ${tempfile}
cat ${tempfile} >> ICMCL${exp_name}INIT

# Clean up
rm -f ${tempfile}

# -----------------------------------------------------------------------------
# *** Link the appropriate NEMO restart files of the previous leg
# -----------------------------------------------------------------------------
if $leg_is_restart
then
    ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec )))
    for n in $( seq 0 $((nem_numproc-1)) )
    do
        np=$(printf %04d ${n})
        ln -fs ${exp_name}_${ns}_restart_oce_${np}.nc restart_oce_${np}.nc
        ln -fs ${exp_name}_${ns}_restart_ice_${np}.nc restart_ice_${np}.nc
    done
fi

# -----------------------------------------------------------------------------
# *** Remove some OASIS files of the previous leg
# -----------------------------------------------------------------------------
if $leg_is_restart
then
    rm -f anaisout_*
fi

# -----------------------------------------------------------------------------
# *** Start the run
# -----------------------------------------------------------------------------
export DR_HOOK_IGNORE_SIGNALS='-1'
export OASIS3=yes
export OASIS3DEBUGLEVEL=2

# Use the launch function from the platform configuration file
t1=$(gdate +%s)
rm -f configmpmd
for node in $(seq 1 $oas_numproc)
do
    echo "$oas_exe_file" >> configmpmd
done

for node in $(seq 1 $ifs_numproc)
do
    echo "$ifs_exe_file -v ecmwf -e  $exp_name" >> configmpmd
done

for node in $(seq 1 $nem_numproc)
do
    echo "$nem_exe_file" >> configmpmd
done

t2=$(gdate +%s)
/usr/bin/poe -pgmmodel mpmd -cmdfile configmpmd
tr=$(gdate -d "0 -$t1 sec + $t2 sec" +%T)

# -----------------------------------------------------------------------------
# *** Write the restart control file
# -----------------------------------------------------------------------------
echo "#"                                             | tee -a ${ece_info_file}
echo "# Finished leg at `gdate '+%F %T'` after ${tr} (hh:mm:ss)" \
                                                     | tee -a ${ece_info_file}
echo "leg_number=${leg_number}"                      | tee -a ${ece_info_file}
echo "leg_start_date=\"${leg_start_date}\""          | tee -a ${ece_info_file}
echo "leg_end_date=\"${leg_end_date}\""              | tee -a ${ece_info_file}

# -----------------------------------------------------------------------------
# *** Platform independent finalising of the run
# -----------------------------------------------------------------------------
# Rename run-files so they are not overwritten in the next leg
filesToCopy="NODE.001_01   rcf   $oas_cpl_flds"
for file in ${filesToCopy}
do
    cp ${file} ${file}.$(printf %03d $((leg_number)))
done
rm -f NODE.001_01 

exit 0
