#!/usr/bin/env bash
# ------------------------------------------------------------------------------
# This file submits a job to the SLURM queue for CPU profling.
# ------------------------------------------------------------------------------
#SBATCH --job-name=ectrans_gpu_trace_n1
#SBATCH --partition=dev-g
#SBATCH --exclusive
#SBATCH --mem=0
#SBATCH --account=project_465000454
#SBATCH --nodes=1
#SBATCH --gpus-per-node=8
#SBATCH --ntasks=8
#SBATCH --time=0:01:30

# Purge modules as some default one(s) cause runtime issues.
# module reset
# module purge

# Load required modules.
module load LUMI/23.09 partition/G PrgEnv-cray cce/16.0.1
module load craype/2.7.23
module load cray-mpich cray-libsci cray-fftw cray-python
module load cray-hdf5-parallel cray-netcdf-hdf5parallel
module load rocm/5.2.3

# Load helpers for color printing.
source ../helpers.sh

# Load omnitrace.
module use /project/project_465000527/perf_tools/omnitrace/rocm-5.2/share/modulefiles
module load omnitrace
export OMNITRACE_CONFIG_FILE=$HOME/.omnitrace_rocm-5.2.cfg

# Set required ENV variables.
export BINARY="ectrans-lam-benchmark-gpu-dp-acc"
export RESDIR="res-gpu_trace_n1"
export NTASKS=8

# Set runtime arguments to ENV value or default.
[ -z "$NPROMA" ] && export NPROMA=32
[ -z "$NFLD" ] && export NFLD=100
[ -z "$NLEV" ] && export NLEV=3
[ -z "$NITER" ] && export NITER=10  # TODO: Change back to 4.
[ -z "$NLAT" ] && export NLAT=1024
[ -z "$NLON" ] && export NLON=1536

# Profile the ecTrans dwarf.
./ectrans_trace_lumi-g.sh
