From c09a0e22972b8f7b3e85864f607052d465610a59 Mon Sep 17 00:00:00 2001 From: vagudets Date: Wed, 14 May 2025 14:50:56 +0200 Subject: [PATCH 1/2] Always request one node with Autosubmit + Update use case --- autosubmit/conf_esarchive/autosubmit.yml | 6 +++--- autosubmit/conf_esarchive/expdef.yml | 2 +- autosubmit/conf_esarchive/jobs.yml | 12 +++++------- autosubmit/conf_gpfs/autosubmit.yml | 6 +++--- autosubmit/conf_gpfs/jobs.yml | 6 ++---- .../ex1_2_autosubmit_scorecards/ex1_2-handson.md | 6 +++--- .../ex1_2_autosubmit_scorecards/ex1_2-recipe.yml | 4 ++-- 7 files changed, 19 insertions(+), 23 deletions(-) diff --git a/autosubmit/conf_esarchive/autosubmit.yml b/autosubmit/conf_esarchive/autosubmit.yml index 0fd5d5c6..04fb9210 100644 --- a/autosubmit/conf_esarchive/autosubmit.yml +++ b/autosubmit/conf_esarchive/autosubmit.yml @@ -1,10 +1,10 @@ config: EXPID: - AUTOSUBMIT_VERSION: 4.0.0b0 - MAXWAITINGJOBS: 16 + AUTOSUBMIT_VERSION: 4.0.98 + MAXWAITINGJOBS: 100 # Default maximum number of jobs to be running at the same time at any platform # Default: 6 - TOTALJOBS: 16 + TOTALJOBS: 100 SAFETYSLEEPTIME: 10 RETRIALS: 0 mail: diff --git a/autosubmit/conf_esarchive/expdef.yml b/autosubmit/conf_esarchive/expdef.yml index 8dc29b27..bd579342 100644 --- a/autosubmit/conf_esarchive/expdef.yml +++ b/autosubmit/conf_esarchive/expdef.yml @@ -1,6 +1,6 @@ DEFAULT: EXPID: - HPCARCH: nord3v2 + HPCARCH: nord4 experiment: DATELIST: MEMBERS: fc0 diff --git a/autosubmit/conf_esarchive/jobs.yml b/autosubmit/conf_esarchive/jobs.yml index 8196dcc1..f856b441 100644 --- a/autosubmit/conf_esarchive/jobs.yml +++ b/autosubmit/conf_esarchive/jobs.yml @@ -4,20 +4,18 @@ JOBS: RUNNING: chunk WALLCLOCK: NOTIFY_ON: - PLATFORM: nord3v2 + PLATFORM: nord4 PROCESSORS: - ## TODO: Uncomment (see #162) - # NODES: 1 + NODES: 1 # SPLITS: # n_atomic_recipes, number of atomic recipes multimodel: FILE: autosubmit/auto-multimodel.sh RUNNING: once WALLCLOCK: NOTIFY_ON: - PLATFORM: nord3v2 + PLATFORM: nord4 PROCESSORS: - ## TODO: Uncomment - # NODES: 1 + NODES: 1 DEPENDENCIES: verification: SPLITS_FROM: @@ -25,7 +23,7 @@ JOBS: scorecards: FILE: autosubmit/auto-scorecards.sh WALLCLOCK: 00:10 - PLATFORM: nord3v2 + PLATFORM: nord4 NOTIFY_ON: PROCESSORS: 1 DEPENDENCIES: diff --git a/autosubmit/conf_gpfs/autosubmit.yml b/autosubmit/conf_gpfs/autosubmit.yml index 0fd5d5c6..04fb9210 100644 --- a/autosubmit/conf_gpfs/autosubmit.yml +++ b/autosubmit/conf_gpfs/autosubmit.yml @@ -1,10 +1,10 @@ config: EXPID: - AUTOSUBMIT_VERSION: 4.0.0b0 - MAXWAITINGJOBS: 16 + AUTOSUBMIT_VERSION: 4.0.98 + MAXWAITINGJOBS: 100 # Default maximum number of jobs to be running at the same time at any platform # Default: 6 - TOTALJOBS: 16 + TOTALJOBS: 100 SAFETYSLEEPTIME: 10 RETRIALS: 0 mail: diff --git a/autosubmit/conf_gpfs/jobs.yml b/autosubmit/conf_gpfs/jobs.yml index e6806ca7..36591feb 100644 --- a/autosubmit/conf_gpfs/jobs.yml +++ b/autosubmit/conf_gpfs/jobs.yml @@ -13,8 +13,7 @@ JOBS: NOTIFY_ON: PLATFORM: PROCESSORS: - ## TODO: Uncomment - # NODES: 1 + NODES: 1 # SPLITS: # n_atomic_recipes, number of atomic recipes multimodel: FILE: autosubmit/auto-multimodel.sh @@ -24,8 +23,7 @@ JOBS: PLATFORM: PROCESSORS: DEPENDENCIES: - ## TODO: Uncomment - # NODES: 1 + NODES: 1 verification: SPLITS_FROM: SPLITS: # n_atomic_recipes/n_models = n_multimodels diff --git a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-handson.md b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-handson.md index 9f3b05f9..59a8dcf6 100644 --- a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-handson.md +++ b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-handson.md @@ -34,7 +34,7 @@ Since we're going to use Autosubmit to dispatch jobs, we need to have an Autosub On the workstation or the Autosubmit machine, you can create an experiment by the following commands. **AUTOSUBMIT EXPERIMENTS ARE REUSABLE!** You do not need to create a new one every time. ```shell -module load autosubmit/4.0.98b-foss-2015a-Python-3.7.3 +module load autosubmit/4.0.98-foss-2015a-Python-3.7.3 autosubmit expid -H nord4 -d "SUNSET use case 1_2" ``` You will see the messages like below: @@ -141,7 +141,7 @@ INFO [2023-11-29 00:37:41] Check output directory /esarchive/scratch/aho/auto-s INFO [2023-11-29 00:37:41] ##### AUTOSUBMIT CONFIGURATION WRITTEN FOR a6pc ##### INFO [2023-11-29 00:37:41] You can check your experiment configuration at: /esarchive/autosubmit/a6pc/conf/ INFO [2023-11-29 00:37:41] Please SSH into bscesautosubmit01 or bscesautosubmit02 and run the following commands: -INFO [2023-11-29 00:37:41] module load autosubmit/4.0.0b-foss-2015a-Python-3.7.3 +INFO [2023-11-29 00:37:41] module load autosubmit/4.0.98-foss-2015a-Python-3.7.3 INFO [2023-11-29 00:37:41] autosubmit create a6pc INFO [2023-11-29 00:37:41] autosubmit refresh a6pc INFO [2023-11-29 00:37:41] nohup autosubmit run a6pc & disown @@ -151,7 +151,7 @@ You can see some useful information, like the the path to atomic recipes, the Au ```shell ssh bscesautosubmit01.bsc.es (enter Autosubmit machine) -module load autosubmit/4.0.0b-foss-2015a-Python-3.7.3 +module load autosubmit/4.0.98-foss-2015a-Python-3.7.3 autosubmit create a6pc autosubmit refresh a6pc nohup autosubmit run a6pc & disown diff --git a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml index a4e4b367..adf53ffd 100644 --- a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml +++ b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml @@ -69,7 +69,7 @@ Analysis: col1_width: NULL col2_width: NULL calculate_diff: FALSE - ncores: 8 + ncores: 16 remove_NAs: no # bool, don't split Output_format: Scorecards # string, don't split @@ -91,7 +91,7 @@ Run: wallclock: 03:00 # hh:mm processors_per_job: 16 platform: nord4 - custom_directives: ['#SBATCH --exclusive'] + custom_directives: # ['#SBATCH --exclusive'] email_notifications: yes # enable/disable email notifications. Change it if you want to. email_address: victoria.agudetse@bsc.es # replace with your email address notify_completed: yes # notify me by email when a job finishes -- GitLab From 508795737e337a689797d038f5facbdb4a6de195 Mon Sep 17 00:00:00 2001 From: vagudets Date: Fri, 16 May 2025 10:35:52 +0200 Subject: [PATCH 2/2] Fix chunk_to_recipe file path --- tools/write_autosubmit_conf.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/write_autosubmit_conf.R b/tools/write_autosubmit_conf.R index dd943f15..1b490d1c 100644 --- a/tools/write_autosubmit_conf.R +++ b/tools/write_autosubmit_conf.R @@ -61,7 +61,7 @@ write_autosubmit_conf <- function(recipe, nchunks, # Section 3: jobs ## wallclock, notify_on, platform?, processors, # Create bash file to associate chunk number to recipe name - chunk_file <- paste0(proj_dir, "chunk_to_recipe") + chunk_file <- paste0(proj_dir, "/", "chunk_to_recipe") ## TODO: Specify dir in chunk_to_recipe? .create_bash_file(fileout = chunk_file, dictionary = chunk_to_recipe, -- GitLab