From f488a2c24d3233478780b7a0d44fee6576d6b4dd Mon Sep 17 00:00:00 2001 From: vagudets Date: Tue, 17 Sep 2024 16:27:19 +0200 Subject: [PATCH 01/16] Launch jobs on MN5 (first attempt) --- autosubmit/conf_gpfs/autosubmit.yml | 22 +++++++++++++++ autosubmit/conf_gpfs/expdef.yml | 44 +++++++++++++++++++++++++++++ autosubmit/conf_gpfs/jobs.yml | 28 ++++++++++++++++++ autosubmit/conf_gpfs/platforms.yml | 20 +++++++++++++ autosubmit/conf_gpfs/proj.yml | 4 +++ conf/autosubmit.yml | 7 +++++ launch_SUNSET.sh | 13 +++++++-- 7 files changed, 135 insertions(+), 3 deletions(-) create mode 100644 autosubmit/conf_gpfs/autosubmit.yml create mode 100644 autosubmit/conf_gpfs/expdef.yml create mode 100644 autosubmit/conf_gpfs/jobs.yml create mode 100644 autosubmit/conf_gpfs/platforms.yml create mode 100644 autosubmit/conf_gpfs/proj.yml diff --git a/autosubmit/conf_gpfs/autosubmit.yml b/autosubmit/conf_gpfs/autosubmit.yml new file mode 100644 index 00000000..0fd5d5c6 --- /dev/null +++ b/autosubmit/conf_gpfs/autosubmit.yml @@ -0,0 +1,22 @@ +config: + EXPID: + AUTOSUBMIT_VERSION: 4.0.0b0 + MAXWAITINGJOBS: 16 + # Default maximum number of jobs to be running at the same time at any platform + # Default: 6 + TOTALJOBS: 16 + SAFETYSLEEPTIME: 10 + RETRIALS: 0 +mail: + NOTIFICATIONS: + TO: +communications: + # Communications library used to connect with platforms: paramiko or saga. + # Default: paramiko + API: paramiko +storage: + # Defines the way of storing the progress of the experiment. The available options are: + # A PICKLE file (pkl) or an SQLite database (db). Default: pkl + TYPE: pkl + # Defines if the remote logs will be copied to the local platform. Default: True. + COPY_REMOTE_LOGS: True diff --git a/autosubmit/conf_gpfs/expdef.yml b/autosubmit/conf_gpfs/expdef.yml new file mode 100644 index 00000000..8dc29b27 --- /dev/null +++ b/autosubmit/conf_gpfs/expdef.yml @@ -0,0 +1,44 @@ +DEFAULT: + EXPID: + HPCARCH: nord3v2 +experiment: + DATELIST: + MEMBERS: fc0 + CHUNKSIZEUNIT: month + CHUNKSIZE: 1 + NUMCHUNKS: + CHUNKINI: 1 + CALENDAR: standard +project: + PROJECT_TYPE: local + # Destination folder name for project. type: STRING, default: leave empty, + PROJECT_DESTINATION: auto-s2s +# If PROJECT_TYPE is not git, no need to change +git: + # Repository URL STRING: 'https://github.com/torvalds/linux.git' + PROJECT_ORIGIN: https://earth.bsc.es/gitlab/es/auto-s2s.git + # Select branch or tag, STRING, default: 'master', help: {'master' (default), 'develop', 'v3.1b', ...} + PROJECT_BRANCH: master + # type: STRING, default: leave empty, help: if model branch is a TAG leave empty + PROJECT_COMMIT: '' +svn: + PROJECT_URL: '' + PROJECT_REVISION: '' +# If PROJECT_TYPE is not local, no need to change +local: + # type: STRING, help: /foo/bar/ecearth + PROJECT_PATH: /esarchive/scratch/vagudets/repos/auto-s2s/ +# If PROJECT_TYPE is none, no need to change +project_files: + # Where is PROJECT CONFIGURATION file location relative to project root path + FILE_PROJECT_CONF: '' + # Where is JOBS CONFIGURATION file location relative to project root path + FILE_JOBS_CONF: '' + # Default job scripts type in the project. type: STRING, default: bash, supported: 'bash', 'python' or 'r' + JOB_SCRIPTS_TYPE: '' +rerun: + # Is a rerun or not? [Default: Do set FALSE]. BOOLEAN: TRUE, FALSE + RERUN: FALSE + # If RERUN: TRUE then supply the list of chunks to rerun + # LIST: [ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ] + CHUNKLIST: '' diff --git a/autosubmit/conf_gpfs/jobs.yml b/autosubmit/conf_gpfs/jobs.yml new file mode 100644 index 00000000..04d23ba0 --- /dev/null +++ b/autosubmit/conf_gpfs/jobs.yml @@ -0,0 +1,28 @@ +JOBS: + verification: + FILE: autosubmit/auto-verification.sh + RUNNING: chunk + WALLCLOCK: + NOTIFY_ON: + PLATFORM: nord3v2 + PROCESSORS: + # SPLITS: # n_atomic_recipes, number of atomic recipes + multimodel: + FILE: autosubmit/auto-multimodel.sh + RUNNING: once + WALLCLOCK: + NOTIFY_ON: + PLATFORM: nord3v2 + PROCESSORS: + DEPENDENCIES: + verification: + SPLITS_FROM: + SPLITS: # n_atomic_recipes/n_models = n_multimodels + scorecards: + FILE: autosubmit/auto-scorecards.sh + WALLCLOCK: 00:10 + PLATFORM: nord3v2 + NOTIFY_ON: + PROCESSORS: 1 + DEPENDENCIES: + diff --git a/autosubmit/conf_gpfs/platforms.yml b/autosubmit/conf_gpfs/platforms.yml new file mode 100644 index 00000000..fa0f7559 --- /dev/null +++ b/autosubmit/conf_gpfs/platforms.yml @@ -0,0 +1,20 @@ +## TODO: Change platform +Platforms: + nord3v2: + TYPE: slurm + HOST: nord4.bsc.es + USER: + PROJECT: bsc32 ## TO BE CHANGED + SCRATCH_DIR: /gpfs/scratch/ ## TO BE CHANGED + PROCESSORS_PER_NODE: 16 + SERIAL_QUEUE: debug + QUEUE: bsc_es + mn5: + TYPE: slurm + HOST: glogin1.bsc.es + PROJECT: bsc32 + USER: + QUEUE: gp_bsces + SCRATCH_DIR: /gpfs/scratch/ + ADD_PROJECT_TO_HOST: false + TEMP_DIR: '' diff --git a/autosubmit/conf_gpfs/proj.yml b/autosubmit/conf_gpfs/proj.yml new file mode 100644 index 00000000..679cf63b --- /dev/null +++ b/autosubmit/conf_gpfs/proj.yml @@ -0,0 +1,4 @@ +common: + MODULES: "MODULES" + OUTDIR: + SCRIPT: diff --git a/conf/autosubmit.yml b/conf/autosubmit.yml index 25872a0e..3e19dec4 100644 --- a/conf/autosubmit.yml +++ b/conf/autosubmit.yml @@ -12,3 +12,10 @@ mars: conf_format: yaml experiment_dir: /esarchive/autosubmit/ ## TO BE CHANGED userID: bsc32 ## TO BE CHANGED +gpfs: + platform: mn5 + module_version: autosubmit/4.0.0b-foss-2015a-Python-3.7.3 ## TO BE CHANGED + auto_version: 4.0.0 + conf_format: yaml + experiment_dir: /esarchive/autosubmit/ ## TO BE CHANGED + userID: bsc032 diff --git a/launch_SUNSET.sh b/launch_SUNSET.sh index 6149a963..deb4f112 100644 --- a/launch_SUNSET.sh +++ b/launch_SUNSET.sh @@ -127,6 +127,13 @@ if [[ $run_method == "sbatch" ]]; then logdir=${outdir}/logs/slurm/ mkdir -p $logdir echo "Slurm job logs will be stored in $logdir" + + # Is machine MN5? + if [[ $BSC_MACHINE == "mn5" ]]; then + platform_params="-A bsc32 -q gp_bsces" + else + platform_params="" + fi # Launch one job per atomic recipe cd $codedir @@ -141,7 +148,7 @@ if [[ $run_method == "sbatch" ]]; then outfile=${logdir}/run-${job_name}.out errfile=${logdir}/run-${job_name}.err # Send batch job and capture job ID - job_ID=$(sbatch --parsable --job-name="SUNSET_verification" --output=$outfile --error=$errfile --time=$wallclock --cpus-per-task=$cpus $custom_directives conf/slurm_templates/run_parallel_workflow.sh ${script} ${atomic_recipe}) + job_ID=$(sbatch --parsable --job-name="SUNSET_verification" $platform_params --output=$outfile --error=$errfile --time=$wallclock --cpus-per-task=$cpus $custom_directives conf/slurm_templates/run_parallel_workflow.sh ${script} ${atomic_recipe}) # Add job ID to array verification_job_list+=($job_ID) echo "Submitted batch job $job_ID" @@ -156,7 +163,7 @@ if [[ $run_method == "sbatch" ]]; then outfile=${logdir}/run-multimodel-${job_name}.out errfile=${logdir}/run-multimodel-${job_name}.err # Send batch job and capture job ID - job_ID=$(sbatch --parsable --dependency=afterok:$(IFS=,; echo "${verification_job_list[*]}") --kill-on-invalid-dep=yes --job-name="SUNSET_multimodel" --output=$outfile --error=$errfile --time=$wallclock --cpus-per-task=$cpus $custom_directives conf/slurm_templates/run_parallel_workflow.sh ${script} ${atomic_recipe}) + job_ID=$(sbatch --parsable --dependency=afterok:$(IFS=,; echo "${verification_job_list[*]}") --kill-on-invalid-dep=yes --job-name="SUNSET_multimodel" $platform_params --output=$outfile --error=$errfile --time=$wallclock --cpus-per-task=$cpus $custom_directives conf/slurm_templates/run_parallel_workflow.sh ${script} ${atomic_recipe}) # Add job ID to array multimodel_job_list+=($job_ID) echo "Submitted batch job $job_ID" @@ -171,7 +178,7 @@ if [[ $run_method == "sbatch" ]]; then echo "Submitting scorecards jobs..." outfile=${logdir}/run-scorecards.out errfile=${logdir}/run-scorecards.err - sbatch --dependency=afterok:$(IFS=,; echo "${verification_job_list[*]} ${multimodel_job_list[*]}") --output=$outfile --error=$errfile --time=01:00:00 conf/slurm_templates/run_scorecards.sh ${recipe} ${outdir} + sbatch --dependency=afterok:$(IFS=,; echo "${verification_job_list[*]} ${multimodel_job_list[*]}") $platform_params --output=$outfile --error=$errfile --time=01:00:00 conf/slurm_templates/run_scorecards.sh ${recipe} ${outdir} fi fi -- GitLab From 671c685410bd3cee9ccd11eca9593cb0f4e6e23f Mon Sep 17 00:00:00 2001 From: vagudets Date: Wed, 18 Sep 2024 09:39:03 +0200 Subject: [PATCH 02/16] Fix gpfs/ECMWF-SEAS5.1 reference grid --- conf/archive_seasonal.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/archive_seasonal.yml b/conf/archive_seasonal.yml index 4ab68ab1..660df18b 100644 --- a/conf/archive_seasonal.yml +++ b/conf/archive_seasonal.yml @@ -33,7 +33,7 @@ gpfs: hcst: 25 calendar: "proleptic_gregorian" time_stamp_lag: "0" - reference_grid: "/gpfs/projects/bsc32/esarchive_cache/exp/ecmwf/system51c3s/monthly_mean/tas_f6h/tas_20180501.nc" + reference_grid: "conf/grid_description/griddes_GRIB_system51_m1.txt" land_sea_mask: "/gpfs/projects/bsc32/esarchive_cache/exp/ecmwf/system51c3s/constant/lsm/lsm.nc" CMCC-SPS3.5: name: "CMCC-SPS3.5" -- GitLab From 2ca34b65ee815b8bc574b8a218c136f61790e6dd Mon Sep 17 00:00:00 2001 From: vagudets Date: Wed, 18 Sep 2024 09:44:07 +0200 Subject: [PATCH 03/16] Fix gpfs/ECMWF-SEAS5.1 reference grid --- conf/archive_seasonal.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/archive_seasonal.yml b/conf/archive_seasonal.yml index 660df18b..a9caa6fd 100644 --- a/conf/archive_seasonal.yml +++ b/conf/archive_seasonal.yml @@ -33,7 +33,7 @@ gpfs: hcst: 25 calendar: "proleptic_gregorian" time_stamp_lag: "0" - reference_grid: "conf/grid_description/griddes_GRIB_system51_m1.txt" + reference_grid: "conf/grid_description/griddes_system51c3s.txt" land_sea_mask: "/gpfs/projects/bsc32/esarchive_cache/exp/ecmwf/system51c3s/constant/lsm/lsm.nc" CMCC-SPS3.5: name: "CMCC-SPS3.5" -- GitLab From bec6164684e1baac106c46aed1e75ee37f02a0f2 Mon Sep 17 00:00:00 2001 From: vagudets Date: Thu, 19 Sep 2024 08:46:05 +0200 Subject: [PATCH 04/16] Add script to transfer recipes to gpfs --- autosubmit/auto-transfer_recipes.sh | 18 ++++++++++++++++++ autosubmit/conf_gpfs/jobs.yml | 7 +++++++ autosubmit/conf_gpfs/platforms.yml | 7 +++++++ 3 files changed, 32 insertions(+) create mode 100644 autosubmit/auto-transfer_recipes.sh diff --git a/autosubmit/auto-transfer_recipes.sh b/autosubmit/auto-transfer_recipes.sh new file mode 100644 index 00000000..a4ca993c --- /dev/null +++ b/autosubmit/auto-transfer_recipes.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +############ AUTOSUBMIT INPUTS ############ +proj_dir=%PROJDIR% +outdir=%common.OUTDIR% +script=%common.SCRIPT% +CHUNK=%CHUNK% +############################### + +cd $proj_dir + +username=$(whoami) + +destdir=/gpfs/scratch/bsc32/${username}/tmp/$(basename ${outdir}) + +mkdir -p $destdir + +dtcp ${outdir}/* ${destdir} diff --git a/autosubmit/conf_gpfs/jobs.yml b/autosubmit/conf_gpfs/jobs.yml index 04d23ba0..c5201512 100644 --- a/autosubmit/conf_gpfs/jobs.yml +++ b/autosubmit/conf_gpfs/jobs.yml @@ -1,4 +1,11 @@ JOBS: + recipe_transfer: + FILE: autosubmit/auto-recipe_transfer.sh + RUNNING: once + WALLCLOCK: 00:10 + PLATFORM: transfer + NOTIFY_ON: + PROCESSORS: verification: FILE: autosubmit/auto-verification.sh RUNNING: chunk diff --git a/autosubmit/conf_gpfs/platforms.yml b/autosubmit/conf_gpfs/platforms.yml index fa0f7559..d6c777ba 100644 --- a/autosubmit/conf_gpfs/platforms.yml +++ b/autosubmit/conf_gpfs/platforms.yml @@ -18,3 +18,10 @@ Platforms: SCRATCH_DIR: /gpfs/scratch/ ADD_PROJECT_TO_HOST: false TEMP_DIR: '' + transfer: + TYPE: slurm + HOST: transfer2.bsc.es + PROJECT: bsc32 + SCRATCH_DIR: /gpfs/scratch/ + USER: + -- GitLab From 298a2bdf7919908836aa293b66c7a75d95e6a5bb Mon Sep 17 00:00:00 2001 From: vagudets Date: Thu, 19 Sep 2024 12:29:22 +0200 Subject: [PATCH 05/16] Adapt auto-scripts to new structure! --- autosubmit/auto-transfer_recipes.sh | 13 ++++++------- autosubmit/auto-transfer_results.sh | 9 +++++++++ autosubmit/auto-verification.sh | 3 ++- autosubmit/conf_gpfs/jobs.yml | 14 ++++++++++---- 4 files changed, 27 insertions(+), 12 deletions(-) create mode 100644 autosubmit/auto-transfer_results.sh diff --git a/autosubmit/auto-transfer_recipes.sh b/autosubmit/auto-transfer_recipes.sh index a4ca993c..7f038b4b 100644 --- a/autosubmit/auto-transfer_recipes.sh +++ b/autosubmit/auto-transfer_recipes.sh @@ -3,16 +3,15 @@ ############ AUTOSUBMIT INPUTS ############ proj_dir=%PROJDIR% outdir=%common.OUTDIR% -script=%common.SCRIPT% -CHUNK=%CHUNK% +tmpdir=%common.TMPDIR% ############################### -cd $proj_dir +# cd $proj_dir -username=$(whoami) +# username=$(whoami) -destdir=/gpfs/scratch/bsc32/${username}/tmp/$(basename ${outdir}) +# destdir=/gpfs/scratch/bsc32/${username}/tmp/$(basename ${outdir}) -mkdir -p $destdir +mkdir -p $tmpdir -dtcp ${outdir}/* ${destdir} +dtcp ${outdir}/* ${tmpdir} diff --git a/autosubmit/auto-transfer_results.sh b/autosubmit/auto-transfer_results.sh new file mode 100644 index 00000000..93618af2 --- /dev/null +++ b/autosubmit/auto-transfer_results.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +############ AUTOSUBMIT INPUTS ############ +proj_dir=%PROJDIR% +outdir=%common.OUTDIR% +tmpdir=%common.TMPDIR% +############################### + +dtcp ${tmpdir}/* ${outdir} diff --git a/autosubmit/auto-verification.sh b/autosubmit/auto-verification.sh index e909dbfb..f0c5c043 100644 --- a/autosubmit/auto-verification.sh +++ b/autosubmit/auto-verification.sh @@ -3,6 +3,7 @@ ############ AUTOSUBMIT INPUTS ############ proj_dir=%PROJDIR% outdir=%common.OUTDIR% +tmpdir=%common.TMPDIR% script=%common.SCRIPT% CHUNK=%CHUNK% ############################### @@ -12,7 +13,7 @@ cd $proj_dir source chunk_to_recipe # atomic_recipe_number=$(printf "%02d" $CHUNK) -atomic_recipe=${outdir}/logs/recipes/atomic_recipe_${recipe}.yml +atomic_recipe=${tmpdir}/logs/recipes/atomic_recipe_${recipe}.yml source MODULES diff --git a/autosubmit/conf_gpfs/jobs.yml b/autosubmit/conf_gpfs/jobs.yml index c5201512..973a3a49 100644 --- a/autosubmit/conf_gpfs/jobs.yml +++ b/autosubmit/conf_gpfs/jobs.yml @@ -1,11 +1,11 @@ JOBS: - recipe_transfer: - FILE: autosubmit/auto-recipe_transfer.sh + transfer_recipes: + FILE: autosubmit/auto-transfer_recipes.sh RUNNING: once WALLCLOCK: 00:10 PLATFORM: transfer NOTIFY_ON: - PROCESSORS: + PROCESSORS: 1 verification: FILE: autosubmit/auto-verification.sh RUNNING: chunk @@ -32,4 +32,10 @@ JOBS: NOTIFY_ON: PROCESSORS: 1 DEPENDENCIES: - + transfer_results: + FILE: autosubmit/auto-transfer_results.sh + RUNNING: once + WALLCLOCK: 00:10 + PLATFORM: transfer + NOTIFY_ON: + PROCESSORS: 1 -- GitLab From fd00e12e397fb50573c3412532f65788ce0bea33 Mon Sep 17 00:00:00 2001 From: vagudets Date: Fri, 20 Sep 2024 16:35:28 +0200 Subject: [PATCH 06/16] Adapt auto-jobs to GPFS --- autosubmit/auto-scorecards.sh | 11 ++++++++--- autosubmit/auto-transfer_recipes.sh | 10 +++++++--- autosubmit/auto-transfer_results.sh | 8 +++++++- autosubmit/auto-verification.sh | 8 ++++++-- 4 files changed, 28 insertions(+), 9 deletions(-) diff --git a/autosubmit/auto-scorecards.sh b/autosubmit/auto-scorecards.sh index c30f643f..493a8ad1 100644 --- a/autosubmit/auto-scorecards.sh +++ b/autosubmit/auto-scorecards.sh @@ -4,12 +4,17 @@ proj_dir=%PROJDIR% outdir=%common.OUTDIR% recipe=%common.RECIPE% +tmpdir=%common.TMPDIR% ############################### -cd $proj_dir +cd ${tmpdir}/sunset/$(basename ${outdir})/ -recipe=${outdir}/logs/recipes/${recipe} +srcdir=${tmpdir}/$(basename ${outdir}) +recipe=${srcdir}/logs/recipes/${recipe} + +set +eu source MODULES +set -eu -Rscript modules/Scorecards/execute_scorecards.R ${recipe} ${outdir} +Rscript modules/Scorecards/execute_scorecards.R ${recipe} ${srcdir} diff --git a/autosubmit/auto-transfer_recipes.sh b/autosubmit/auto-transfer_recipes.sh index 7f038b4b..4b332e8b 100644 --- a/autosubmit/auto-transfer_recipes.sh +++ b/autosubmit/auto-transfer_recipes.sh @@ -10,8 +10,12 @@ tmpdir=%common.TMPDIR% # username=$(whoami) -# destdir=/gpfs/scratch/bsc32/${username}/tmp/$(basename ${outdir}) +destdir=${tmpdir}/$(basename ${outdir}) -mkdir -p $tmpdir +# Copy recipes +mkdir -p $destdir +cp -r /gpfs/archive/bsc32/${outdir}/* ${destdir} -dtcp ${outdir}/* ${tmpdir} +# Copy repository with code +mkdir -p ${tmpdir}/sunset/$(basename ${outdir})/ +cp -r /gpfs/archive/bsc32/${proj_dir}/* ${tmpdir}/sunset/$(basename ${outdir})/ diff --git a/autosubmit/auto-transfer_results.sh b/autosubmit/auto-transfer_results.sh index 93618af2..fda0d768 100644 --- a/autosubmit/auto-transfer_results.sh +++ b/autosubmit/auto-transfer_results.sh @@ -6,4 +6,10 @@ outdir=%common.OUTDIR% tmpdir=%common.TMPDIR% ############################### -dtcp ${tmpdir}/* ${outdir} +srcdir=${tmpdir}/$(basename ${outdir}) + +cp -r ${srcdir}/* /gpfs/archive/bsc32/${outdir} + +rm -r $srcdir + +rm -r ${tmpdir}/sunset/$(basename ${outdir})/ diff --git a/autosubmit/auto-verification.sh b/autosubmit/auto-verification.sh index f0c5c043..6eb343bc 100644 --- a/autosubmit/auto-verification.sh +++ b/autosubmit/auto-verification.sh @@ -8,13 +8,17 @@ script=%common.SCRIPT% CHUNK=%CHUNK% ############################### -cd $proj_dir +cd ${tmpdir}/sunset/$(basename ${outdir})/ source chunk_to_recipe # atomic_recipe_number=$(printf "%02d" $CHUNK) -atomic_recipe=${tmpdir}/logs/recipes/atomic_recipe_${recipe}.yml +srcdir=${tmpdir}/$(basename ${outdir}) +atomic_recipe=${srcdir}/logs/recipes/atomic_recipe_${recipe}.yml + +set +eu source MODULES +set -eu Rscript ${script} ${atomic_recipe} -- GitLab From 9a55868c2003ee9c44094d2980b80ef72fdf67df Mon Sep 17 00:00:00 2001 From: vagudets Date: Fri, 20 Sep 2024 16:35:36 +0200 Subject: [PATCH 07/16] autosubmit/conf_gpfs/jobs.yml --- autosubmit/conf_gpfs/platforms.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/autosubmit/conf_gpfs/platforms.yml b/autosubmit/conf_gpfs/platforms.yml index d6c777ba..aca5c79c 100644 --- a/autosubmit/conf_gpfs/platforms.yml +++ b/autosubmit/conf_gpfs/platforms.yml @@ -16,7 +16,6 @@ Platforms: USER: QUEUE: gp_bsces SCRATCH_DIR: /gpfs/scratch/ - ADD_PROJECT_TO_HOST: false TEMP_DIR: '' transfer: TYPE: slurm -- GitLab From b1456284ae6c572d0296f2a2a1481f8e22f18a3f Mon Sep 17 00:00:00 2001 From: vagudets Date: Fri, 20 Sep 2024 16:36:17 +0200 Subject: [PATCH 08/16] Job configuration for GPFS --- autosubmit/conf_gpfs/jobs.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/autosubmit/conf_gpfs/jobs.yml b/autosubmit/conf_gpfs/jobs.yml index 973a3a49..9f9ce281 100644 --- a/autosubmit/conf_gpfs/jobs.yml +++ b/autosubmit/conf_gpfs/jobs.yml @@ -11,7 +11,7 @@ JOBS: RUNNING: chunk WALLCLOCK: NOTIFY_ON: - PLATFORM: nord3v2 + PLATFORM: PROCESSORS: # SPLITS: # n_atomic_recipes, number of atomic recipes multimodel: @@ -19,7 +19,7 @@ JOBS: RUNNING: once WALLCLOCK: NOTIFY_ON: - PLATFORM: nord3v2 + PLATFORM: PROCESSORS: DEPENDENCIES: verification: @@ -28,7 +28,7 @@ JOBS: scorecards: FILE: autosubmit/auto-scorecards.sh WALLCLOCK: 00:10 - PLATFORM: nord3v2 + PLATFORM: NOTIFY_ON: PROCESSORS: 1 DEPENDENCIES: -- GitLab From b3bb312ea7b109fbcc063e004ba14fdd18629c43 Mon Sep 17 00:00:00 2001 From: vagudets Date: Fri, 20 Sep 2024 16:37:00 +0200 Subject: [PATCH 09/16] Add autosubmit conf --- conf/autosubmit.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/conf/autosubmit.yml b/conf/autosubmit.yml index 3e19dec4..99b29f3c 100644 --- a/conf/autosubmit.yml +++ b/conf/autosubmit.yml @@ -5,6 +5,7 @@ esarchive: conf_format: yaml experiment_dir: /esarchive/autosubmit/ userID: bsc032 + tmp_dir: mars: platform: NORD3 ## TO BE CHANGED module_version: autosubmit/4.0.0b-foss-2015a-Python-3.7.3 ## TO BE CHANGED @@ -12,10 +13,12 @@ mars: conf_format: yaml experiment_dir: /esarchive/autosubmit/ ## TO BE CHANGED userID: bsc32 ## TO BE CHANGED + tmp_dir: gpfs: platform: mn5 - module_version: autosubmit/4.0.0b-foss-2015a-Python-3.7.3 ## TO BE CHANGED - auto_version: 4.0.0 + module_version: autosubmit/4.0.98-foss-2015a-Python-3.7.3 + auto_version: 4.0.98 conf_format: yaml experiment_dir: /esarchive/autosubmit/ ## TO BE CHANGED userID: bsc032 + tmp_dir: /gpfs/scratch/bsc32/$user$/tmp/ -- GitLab From d990aa20ccb85c81e90764a6247f468313ad2916 Mon Sep 17 00:00:00 2001 From: vagudets Date: Fri, 20 Sep 2024 16:37:20 +0200 Subject: [PATCH 10/16] Remove browser call --- modules/Scorecards/Scorecards_calculations.R | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/Scorecards/Scorecards_calculations.R b/modules/Scorecards/Scorecards_calculations.R index 093aa426..f9f0f478 100644 --- a/modules/Scorecards/Scorecards_calculations.R +++ b/modules/Scorecards/Scorecards_calculations.R @@ -360,7 +360,6 @@ Scorecards_calculations <- function(recipe, data, skill_metrics, } ## Save metric result in array - browser() aggr_metrics[,,,which(metrics == met)] <- Reorder(data = mean_bias, order = c('var', 'time', 'region')) aggr_significance[,,,which(metrics == met)] <- Reorder(data = sign_mean_bias, -- GitLab From f11f17bd76416a168c55e22ad208abc0a337c3b9 Mon Sep 17 00:00:00 2001 From: vagudets Date: Fri, 20 Sep 2024 16:37:48 +0200 Subject: [PATCH 11/16] Add line for Scorecards to work in MN5 environment --- modules/Scorecards/execute_scorecards.R | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/Scorecards/execute_scorecards.R b/modules/Scorecards/execute_scorecards.R index 2fa27549..763877b6 100644 --- a/modules/Scorecards/execute_scorecards.R +++ b/modules/Scorecards/execute_scorecards.R @@ -1,5 +1,6 @@ source('tools/libs.R') source('modules/Scorecards/Scorecards_plotting.R') +Sys.setenv(OPENSSL_CONF="/dev/null") args = commandArgs(trailingOnly = TRUE) recipe_file <- args[1] -- GitLab From 7557377834c2a03bc0d748b60c4ee8fef7d118fa Mon Sep 17 00:00:00 2001 From: vagudets Date: Fri, 20 Sep 2024 16:38:15 +0200 Subject: [PATCH 12/16] Add temporary directory for filesystems other than esarchive --- tools/divide_recipe.R | 4 ++ tools/prepare_outputs.R | 11 ++++- tools/read_atomic_recipe.R | 5 +++ tools/write_autosubmit_conf.R | 76 +++++++++++++++++++++++------------ 4 files changed, 69 insertions(+), 27 deletions(-) diff --git a/tools/divide_recipe.R b/tools/divide_recipe.R index ceb696f4..c478b20d 100644 --- a/tools/divide_recipe.R +++ b/tools/divide_recipe.R @@ -21,6 +21,10 @@ divide_recipe <- function(recipe) { Run = recipe$Run[c("Loglevel", "output_dir", "Terminal", "code_dir", "logfile", "filesystem")]) + if (!is.null(recipe$Run$tmp_dir)) { + beta_recipe$Run$tmp_dir <- recipe$Run$tmp_dir + } + # duplicate recipe by independent variables: # If a single variable is not given inside a list, rebuild structure if (any(c("name", "freq", "units") %in% names(recipe$Analysis$Variables))) { diff --git a/tools/prepare_outputs.R b/tools/prepare_outputs.R index eab9c4fe..da192448 100644 --- a/tools/prepare_outputs.R +++ b/tools/prepare_outputs.R @@ -32,7 +32,7 @@ prepare_outputs <- function(recipe_file, recipe$recipe_path <- recipe_file recipe$name <- tools::file_path_sans_ext(basename(recipe_file)) - output_dir = recipe$Run$output_dir + output_dir <- recipe$Run$output_dir # Create output folders if (!uniqueID) { folder_name <- paste0(gsub(".yml", "", gsub("/", "_", recipe$name))) @@ -127,9 +127,18 @@ prepare_outputs <- function(recipe_file, } else { recipe <- check_recipe(recipe) } + + # If autosubmit is used, a temporary directory needs to be created if the + # filesystem is not accessible from the autosubmit machine. + tmp_dir <- read_yaml("conf/autosubmit.yml")[[recipe$Run$filesystem]]$tmp_dir + if (recipe$Run$autosubmit && !(is.null(tmp_dir))) { + tmp_dir <- gsub("\\$.+?\\$", recipe$Run$auto_conf$hpc_user, tmp_dir) + recipe$Run$tmp_dir <- file.path(tmp_dir, folder_name) + } # Create a copy of the recipe and remove the logger recipe_copy <- recipe recipe_copy$Run$logger <- NULL + # Copy recipe to output folder write_yaml(recipe_copy, file = file.path(output_dir, folder_name, 'logs', 'recipes', diff --git a/tools/read_atomic_recipe.R b/tools/read_atomic_recipe.R index 3cb11e7e..99f00155 100644 --- a/tools/read_atomic_recipe.R +++ b/tools/read_atomic_recipe.R @@ -26,6 +26,10 @@ read_atomic_recipe <- function(recipe_file) { recipe <- read_yaml(recipe_file, eval.exp = TRUE) recipe$recipe_path <- recipe_file recipe$name <- tools::file_path_sans_ext(basename(recipe_file)) + # Handle temporary directory for autosubmit + if (!is.null(recipe$Run$tmp_dir)) { + recipe$Run$output_dir <- recipe$Run$tmp_dir + } # Create log file for atomic recipe logfile <- file.path(recipe$Run$output_dir, 'logs', paste0(recipe$name, '.log')) @@ -51,6 +55,7 @@ read_atomic_recipe <- function(recipe_file) { appenders = list(file_appender(logfile, append = TRUE, layout = default_log_layout()))) } + recipe$Run$logger <- logger recipe$Run$logfile <- logfile # Restructure recipe to flatten redundant lists diff --git a/tools/write_autosubmit_conf.R b/tools/write_autosubmit_conf.R index 5f398197..a6bc5aa0 100644 --- a/tools/write_autosubmit_conf.R +++ b/tools/write_autosubmit_conf.R @@ -23,6 +23,10 @@ write_autosubmit_conf <- function(recipe, nchunks, # Output directory dest_dir <- paste0(auto_specs$experiment_dir, expid, "/conf/") proj_dir <- paste0(auto_specs$experiment_dir, expid, "/proj/auto-s2s/") + tmp_dir <- auto_specs$tmp_dir + # HPC details + platform <- recipe$Run$auto_conf$platform + hpc_user <- recipe$Run$auto_conf$hpc_user # Create project directory if it does not exist yet so that chunk_to_recipe # and split_to_recipe files can be created if (!dir.exists(proj_dir)) { @@ -58,28 +62,41 @@ write_autosubmit_conf <- function(recipe, nchunks, ## wallclock, notify_on, platform?, processors, # Create bash file to associate chunk number to recipe name chunk_file <- paste0(proj_dir, "chunk_to_recipe") + ## TODO: Specify dir in chunk_to_recipe? .create_bash_file(fileout = chunk_file, dictionary = chunk_to_recipe, variable = "CHUNK") # Define job parameters - conf$JOBS$verification$WALLCLOCK <- recipe$Run$auto_conf$wallclock + notify_on <- "" if (recipe$Run$auto_conf$notify_completed) { - conf$JOBS$verification$NOTIFY_ON <- paste(conf$JOBS$verification$NOTIFY_ON, - "COMPLETED") + notify_on <- "COMPLETED" } if (recipe$Run$auto_conf$notify_failed) { - conf$JOBS$verification$NOTIFY_ON <- paste(conf$JOBS$verification$NOTIFY_ON, - "FAILED") + notify_on <- paste(notify_on, "FAILED") } + # Define last job before file transfer + last_job <- "verification" + # Verification job + conf$JOBS$verification$WALLCLOCK <- recipe$Run$auto_conf$wallclock + conf$JOBS$verification$NOTIFY_ON <- notify_on conf$JOBS$verification$PROCESSORS <- recipe$Run$auto_conf$processors_per_job # ncores? conf$JOBS$verification$CUSTOM_DIRECTIVES <- recipe$Run$auto_conf$custom_directives - # Only include Multimodel job if sections exists in the recipe + conf$JOBS$verification$PLATFORM <- platform + # Recipe transfer job: only include if a tmp_dir is defined + if (!is.null(tmp_dir)) { + conf$JOBS$verification$DEPENDENCIES <- "transfer_recipes" + conf$JOBS$transfer_recipes$NOTIFY_ON <- notify_on + } else { + conf$JOBS$transfer_recipes <- NULL + } + # Multimodel job: Only include if section exists in the recipe # is set to execute = 'True' or 'both' if (!is.null(recipe$Analysis$Datasets$Multimodel) && tolower(recipe$Analysis$Datasets$Multimodel$execute) == "false") { conf$JOBS$multimodel <- NULL conf$JOBS$scorecards$DEPENDENCIES <- "verification" } else { + conf$JOBS$multimodel$PLATFORM <- platform conf$JOBS$scorecards$DEPENDENCIES <- "multimodel" # Create bash file to associate split number to recipe name split_file <- paste0(proj_dir, "split_to_recipe") @@ -97,45 +114,52 @@ write_autosubmit_conf <- function(recipe, nchunks, # 'Splits' parameter should be the number of mulimodel jobs conf$JOBS$multimodel$SPLITS <- length(mm_dependencies) # Define the rest of the parameters - if (recipe$Run$auto_conf$notify_completed) { - conf$JOBS$multimodel$NOTIFY_ON <- paste(conf$JOBS$multimodel$NOTIFY_ON, - "COMPLETED") - } - if (recipe$Run$auto_conf$notify_failed) { - conf$JOBS$multimodel$NOTIFY_ON <- paste(conf$JOBS$multimodel$NOTIFY_ON, - "FAILED") - } - + conf$JOBS$multimodel$NOTIFY_ON <- notify_on conf$JOBS$multimodel$PROCESSORS <- recipe$Run$auto_conf$processors_multimodel conf$JOBS$multimodel$CUSTOM_DIRECTIVES <- recipe$Run$auto_conf$custom_directives_multimodel conf$JOBS$multimodel$WALLCLOCK <- recipe$Run$auto_conf$wallclock_multimodel + last_job <- "multimodel" } - # Only include Scorecards job if section exists in the recipe and + # Scorecards job: include only if section exists in the recipe and # is set to 'execute: True' if (!("Scorecards" %in% names(recipe$Analysis$Workflow)) || (!recipe$Analysis$Workflow$Scorecards$execute)) { conf$JOBS$scorecards <- NULL } else { - if (recipe$Run$auto_conf$notify_completed) { - conf$JOBS$scorecards$NOTIFY_ON <- paste(conf$JOBS$scorecards$NOTIFY_ON, - "COMPLETED") - } - if (recipe$Run$auto_conf$notify_failed) { - conf$JOBS$scorecards$NOTIFY_ON <- paste(conf$JOBS$scorecards$NOTIFY_ON, - "FAILED") - } + conf$JOBS$scorecards$NOTIFY_ON <- notify_on + conf$JOBS$scorecards$PLATFORM <- platform + last_job <- "scorecards" + } + # Results transfer job: only include if a tmp_dir is defined + if (!is.null(tmp_dir)) { + conf$JOBS$transfer_results$DEPENDENCIES <- last_job + conf$JOBS$transfer_results$NOTIFY_ON <- notify_on + } else { + conf$JOBS$transfer_results <- NULL } } else if (conf_type == "platforms") { + ## TODO: Allow user to choose platform # Section 4: platform configuration ## nord3v2 configuration... platform name? user, processors_per_node - conf$Platforms[[auto_specs$platform]]$USER <- - recipe$Run$auto_conf$hpc_user + conf$Platforms[[platform]]$USER <- hpc_user + unused_machines <- names(conf$Platforms)[!names(conf$Platforms) %in% c("transfer", platform)] + conf$Platforms[unused_machines] <- NULL + if (!is.null(tmp_dir)) { + conf$Platforms$transfer$USER <- hpc_user + } else { + conf$Platforms$transfer <- NULL + } } else if (conf_type == "proj") { # Section 5: proj ## modules? Info that goes on script, e.g. output directory conf$common$OUTDIR <- recipe$Run$output_dir conf$common$SCRIPT <- recipe$Run$auto_conf$script conf$common$RECIPE <- paste0(recipe$name, ".yml") + if (!is.null(tmp_dir)) { + conf$common$TMPDIR <- gsub("\\$.+?\\$", hpc_user, tmp_dir) + } else { + conf$common$TMPDIR <- recipe$Run$output_dir + } } # Write config file inside autosubmit dir write.config(conf, paste0(dest_dir, dest_file), -- GitLab From 3ded6135effe65674bf197640d452ea07b2cb7e9 Mon Sep 17 00:00:00 2001 From: vagudets Date: Fri, 20 Sep 2024 16:38:34 +0200 Subject: [PATCH 13/16] Example of recipe for MN5 --- recipes/examples/scorecards_on_MN5.yml | 100 +++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 recipes/examples/scorecards_on_MN5.yml diff --git a/recipes/examples/scorecards_on_MN5.yml b/recipes/examples/scorecards_on_MN5.yml new file mode 100644 index 00000000..32decb69 --- /dev/null +++ b/recipes/examples/scorecards_on_MN5.yml @@ -0,0 +1,100 @@ +Description: + Author: An-Chi Ho + Info: Compute Skills and Plot Scorecards with Autosubmit + +Analysis: + Horizon: seasonal + Variables: + - {name: tas, freq: monthly_mean} + Datasets: + System: # multiple systems for single model, split if Multimodel = F + - {name: ECMWF-SEAS5.1} + Multimodel: + execute: False # single option + Reference: + - {name: ERA5} + Time: + sdate: # list, split + - '0101' + - '0201' + - '0301' + - '0401' + - '0501' + - '0601' + - '0701' + - '0801' + - '0901' + - '1001' + - '1101' + - '1201' + fcst_year: + hcst_start: '1993' # single option + hcst_end: '2003' # single option + ftime_min: 1 # single option + ftime_max: 6 # single option + Region: # multiple lists, split? Add region name if length(Region) > 1 + - {name: "global", latmin: -90, latmax: 90, lonmin: 0, lonmax: 359.9} + Regrid: + method: bilinear + type: to_system + Workflow: + Anomalies: + compute: yes + cross_validation: no + save: 'none' + Calibration: + method: raw + save: 'none' + Skill: + metric: mean_bias EnsCorr rps rpss crps crpss EnsSprErr # list, don't split + cross_validation: yes + save: 'all' + Statistics: + metric: cov std n_eff spread + save: 'all' + Probabilities: + percentiles: [[1/3, 2/3]] + save: 'none' + Scorecards: + execute: yes # yes/no + regions: + Extra-tropical NH: {lon.min: 0, lon.max: 360, lat.min: 30, lat.max: 90} + Tropics: {lon.min: 0, lon.max: 360, lat.min: -30, lat.max: 30} + Extra-tropical SH : {lon.min: 0, lon.max: 360, lat.min: -30, lat.max: -90} + start_months: 'all' + metric: mean_bias enscorr rpss crpss enssprerr + metric_aggregation: 'score' + inf_to_na: TRUE # Optional, bool: set inf values in data to NA, default is FALSE table_label: NULL + fileout_label: NULL + col1_width: NULL + col2_width: NULL + calculate_diff: FALSE + ncores: 8 + remove_NAs: no # bool, don't split + Output_format: Scorecards # string, don't split + +################################################################################ +## Run CONFIGURATION +################################################################################ +Run: + Loglevel: INFO + Terminal: yes + filesystem: gpfs + output_dir: /esarchive/scratch/vagudets/auto-s2s-outputs/ + code_dir: /home/Earth/vagudets/git/auto-s2s/ + autosubmit: yes + # fill only if using autosubmit + auto_conf: + script: use_cases/ex1_2_autosubmit_scorecards/ex1_2-script.R # replace with the path to your script + expid: a7rn # replace with your EXPID + hpc_user: bsc032762 # replace with your hpc username + wallclock: 03:00 # hh:mm + processors_per_job: 16 + platform: mn5 + custom_directives: # ['#SBATCH --exclusive'] + email_notifications: yes # enable/disable email notifications. Change it if you want to. + email_address: victoria.agudetse@bsc.es # replace with your email address + notify_completed: yes # notify me by email when a job finishes + notify_failed: no # notify me by email when a job fails + + -- GitLab From c24b04ed3dc6cc1705c2112cd0cf0d6567c0ff7b Mon Sep 17 00:00:00 2001 From: vagudets Date: Wed, 25 Sep 2024 12:30:26 +0200 Subject: [PATCH 14/16] Add condition to cd to proper directory depending on filesystem --- autosubmit/auto-multimodel.sh | 17 +++++++++++++---- autosubmit/auto-scorecards.sh | 8 ++++++-- autosubmit/auto-verification.sh | 9 ++++++--- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/autosubmit/auto-multimodel.sh b/autosubmit/auto-multimodel.sh index a9912666..69d58560 100644 --- a/autosubmit/auto-multimodel.sh +++ b/autosubmit/auto-multimodel.sh @@ -1,18 +1,27 @@ #!/bin/bash ############ AUTOSUBMIT INPUTS ############ -proj_dir=%PROJDIR% +projdir=%PROJDIR% outdir=%common.OUTDIR% script=%common.SCRIPT% +tmpdir=%common.TMPDIR% SPLIT=%SPLIT% ############################### -cd $proj_dir +if [ -d $projdir ]; then + cd $projdir +else + cd ${tmpdir}/sunset/$(basename ${outdir})/ +fi source split_to_recipe -# atomic_recipe_number=$(printf "%02d" $CHUNK) -atomic_recipe=${outdir}/logs/recipes/multimodel/atomic_recipe_sys-Multimodel${recipe}.yml +srcdir=${tmpdir}/$(basename ${outdir}) + +atomic_recipe=${srcdir}/logs/recipes/multimodel/atomic_recipe_sys-Multimodel${recipe}.yml + +set +eu source MODULES +set -eu Rscript ${script} ${atomic_recipe} diff --git a/autosubmit/auto-scorecards.sh b/autosubmit/auto-scorecards.sh index 493a8ad1..6f924d88 100644 --- a/autosubmit/auto-scorecards.sh +++ b/autosubmit/auto-scorecards.sh @@ -1,13 +1,17 @@ #!/bin/bash ############ AUTOSUBMIT INPUTS ############ -proj_dir=%PROJDIR% +projdir=%PROJDIR% outdir=%common.OUTDIR% recipe=%common.RECIPE% tmpdir=%common.TMPDIR% ############################### -cd ${tmpdir}/sunset/$(basename ${outdir})/ +if [ -d $projdir ]; then + cd $projdir +else + cd ${tmpdir}/sunset/$(basename ${outdir})/ +fi srcdir=${tmpdir}/$(basename ${outdir}) diff --git a/autosubmit/auto-verification.sh b/autosubmit/auto-verification.sh index 6eb343bc..5454a094 100644 --- a/autosubmit/auto-verification.sh +++ b/autosubmit/auto-verification.sh @@ -1,18 +1,21 @@ #!/bin/bash ############ AUTOSUBMIT INPUTS ############ -proj_dir=%PROJDIR% +projdir=%PROJDIR% outdir=%common.OUTDIR% tmpdir=%common.TMPDIR% script=%common.SCRIPT% CHUNK=%CHUNK% ############################### -cd ${tmpdir}/sunset/$(basename ${outdir})/ +if [ -d $projdir ]; then + cd $projdir +else + cd ${tmpdir}/sunset/$(basename ${outdir})/ +fi source chunk_to_recipe -# atomic_recipe_number=$(printf "%02d" $CHUNK) srcdir=${tmpdir}/$(basename ${outdir}) atomic_recipe=${srcdir}/logs/recipes/atomic_recipe_${recipe}.yml -- GitLab From 9d376bf33f89cac44163f7f12a52bfad2fda6fa8 Mon Sep 17 00:00:00 2001 From: vagudets Date: Wed, 25 Sep 2024 13:45:53 +0200 Subject: [PATCH 15/16] Fix auto-jobs scripts for running on esarchive --- autosubmit/auto-multimodel.sh | 6 +- autosubmit/auto-scorecards.sh | 4 +- autosubmit/auto-verification.sh | 4 +- recipes/examples/scorecards_on_nord3v2.yml | 100 +++++++++++++++++++++ 4 files changed, 106 insertions(+), 8 deletions(-) create mode 100644 recipes/examples/scorecards_on_nord3v2.yml diff --git a/autosubmit/auto-multimodel.sh b/autosubmit/auto-multimodel.sh index 69d58560..0ca01aeb 100644 --- a/autosubmit/auto-multimodel.sh +++ b/autosubmit/auto-multimodel.sh @@ -10,14 +10,12 @@ SPLIT=%SPLIT% if [ -d $projdir ]; then cd $projdir + srcdir=${outdir} else cd ${tmpdir}/sunset/$(basename ${outdir})/ + srcdir=${tmpdir}/$(basename ${outdir}) fi -source split_to_recipe - -srcdir=${tmpdir}/$(basename ${outdir}) - atomic_recipe=${srcdir}/logs/recipes/multimodel/atomic_recipe_sys-Multimodel${recipe}.yml set +eu diff --git a/autosubmit/auto-scorecards.sh b/autosubmit/auto-scorecards.sh index 6f924d88..7722e3b5 100644 --- a/autosubmit/auto-scorecards.sh +++ b/autosubmit/auto-scorecards.sh @@ -9,12 +9,12 @@ tmpdir=%common.TMPDIR% if [ -d $projdir ]; then cd $projdir + srcdir=${outdir} else cd ${tmpdir}/sunset/$(basename ${outdir})/ + srcdir=${tmpdir}/$(basename ${outdir}) fi -srcdir=${tmpdir}/$(basename ${outdir}) - recipe=${srcdir}/logs/recipes/${recipe} set +eu diff --git a/autosubmit/auto-verification.sh b/autosubmit/auto-verification.sh index 5454a094..d323c451 100644 --- a/autosubmit/auto-verification.sh +++ b/autosubmit/auto-verification.sh @@ -10,14 +10,14 @@ CHUNK=%CHUNK% if [ -d $projdir ]; then cd $projdir + srcdir=${outdir} else cd ${tmpdir}/sunset/$(basename ${outdir})/ + srcdir=${tmpdir}/$(basename ${outdir}) fi source chunk_to_recipe -srcdir=${tmpdir}/$(basename ${outdir}) - atomic_recipe=${srcdir}/logs/recipes/atomic_recipe_${recipe}.yml set +eu diff --git a/recipes/examples/scorecards_on_nord3v2.yml b/recipes/examples/scorecards_on_nord3v2.yml new file mode 100644 index 00000000..06333966 --- /dev/null +++ b/recipes/examples/scorecards_on_nord3v2.yml @@ -0,0 +1,100 @@ +Description: + Author: An-Chi Ho + Info: Compute Skills and Plot Scorecards with Autosubmit + +Analysis: + Horizon: seasonal + Variables: + - {name: tas, freq: monthly_mean} + Datasets: + System: # multiple systems for single model, split if Multimodel = F + - {name: ECMWF-SEAS5.1} + Multimodel: + execute: False # single option + Reference: + - {name: ERA5} + Time: + sdate: # list, split + - '0101' + - '0201' + - '0301' + - '0401' + - '0501' + - '0601' + - '0701' + - '0801' + - '0901' + - '1001' + - '1101' + - '1201' + fcst_year: + hcst_start: '1993' # single option + hcst_end: '2003' # single option + ftime_min: 1 # single option + ftime_max: 6 # single option + Region: # multiple lists, split? Add region name if length(Region) > 1 + - {name: "global", latmin: -90, latmax: 90, lonmin: 0, lonmax: 359.9} + Regrid: + method: bilinear + type: to_system + Workflow: + Anomalies: + compute: yes + cross_validation: no + save: 'none' + Calibration: + method: raw + save: 'none' + Skill: + metric: mean_bias EnsCorr rps rpss crps crpss EnsSprErr # list, don't split + cross_validation: yes + save: 'all' + Statistics: + metric: cov std n_eff spread + save: 'all' + Probabilities: + percentiles: [[1/3, 2/3]] + save: 'none' + Scorecards: + execute: yes # yes/no + regions: + Extra-tropical NH: {lon.min: 0, lon.max: 360, lat.min: 30, lat.max: 90} + Tropics: {lon.min: 0, lon.max: 360, lat.min: -30, lat.max: 30} + Extra-tropical SH : {lon.min: 0, lon.max: 360, lat.min: -30, lat.max: -90} + start_months: 'all' + metric: mean_bias enscorr rpss crpss enssprerr + metric_aggregation: 'score' + inf_to_na: TRUE # Optional, bool: set inf values in data to NA, default is FALSE table_label: NULL + fileout_label: NULL + col1_width: NULL + col2_width: NULL + calculate_diff: FALSE + ncores: 8 + remove_NAs: no # bool, don't split + Output_format: Scorecards # string, don't split + +################################################################################ +## Run CONFIGURATION +################################################################################ +Run: + Loglevel: INFO + Terminal: yes + filesystem: esarchive + output_dir: /esarchive/scratch/vagudets/auto-s2s-outputs/ + code_dir: /home/Earth/vagudets/git/auto-s2s/ + autosubmit: yes + # fill only if using autosubmit + auto_conf: + script: use_cases/ex1_2_autosubmit_scorecards/ex1_2-script.R # replace with the path to your script + expid: a6wq # replace with your EXPID + hpc_user: bsc032762 # replace with your hpc username + wallclock: 03:00 # hh:mm + processors_per_job: 16 + platform: nord3v2 + custom_directives: ['#SBATCH --exclusive'] + email_notifications: yes # enable/disable email notifications. Change it if you want to. + email_address: victoria.agudetse@bsc.es # replace with your email address + notify_completed: yes # notify me by email when a job finishes + notify_failed: no # notify me by email when a job fails + + -- GitLab From e3ce9b8118d3248416134f66dec6d8d8eb420514 Mon Sep 17 00:00:00 2001 From: vagudets Date: Thu, 26 Sep 2024 10:47:17 +0200 Subject: [PATCH 16/16] Add AMD configuration --- MODULES | 10 ++++++++++ autosubmit/conf_gpfs/platforms.yml | 9 ++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/MODULES b/MODULES index 7ed13edc..e88620be 100644 --- a/MODULES +++ b/MODULES @@ -22,6 +22,16 @@ elif [[ $BSC_MACHINE == "mn5" ]]; then source /gpfs/projects/bsc32/software/suselinux/11/software/Miniconda3/4.7.10/etc/profile.d/conda.sh conda activate /gpfs/projects/bsc32/repository/apps/conda_envs/SUNSET-env_2.0.0 +elif [[ $BSC_MACHINE == "amd" ]]; then + + module purge + module load CDO/1.9.10-foss-2019b + module load R/4.1.2-foss-2019b + module load GEOS/3.7.2-foss-2019b-Python-3.7.4 + module load GDAL/3.5.0-foss-2019b-Python-3.7.4 + module load PROJ/9.0.0-GCCcore-8.3.0 + module load Phantomjs/2.1.1 + elif [[ $HOSTNAME == "bsceshub02.bsc.es" ]]; then module purge diff --git a/autosubmit/conf_gpfs/platforms.yml b/autosubmit/conf_gpfs/platforms.yml index aca5c79c..03f4b940 100644 --- a/autosubmit/conf_gpfs/platforms.yml +++ b/autosubmit/conf_gpfs/platforms.yml @@ -17,10 +17,17 @@ Platforms: QUEUE: gp_bsces SCRATCH_DIR: /gpfs/scratch/ TEMP_DIR: '' + amd: + TYPE: slurm + HOST: amdlogin1.bsc.es + PROJECT: bsc32 + USER: + SCRATCH_DIR: /gpfs/scratch/ + TEMP_DIR: '' transfer: TYPE: slurm HOST: transfer2.bsc.es PROJECT: bsc32 SCRATCH_DIR: /gpfs/scratch/ USER: - + -- GitLab