From c04b617ee322ebb32e223c86506cc4e097ac63bc Mon Sep 17 00:00:00 2001 From: vagudets Date: Wed, 8 Jan 2025 12:57:58 +0100 Subject: [PATCH 1/3] Update README --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2d97af18..ed32aeee 100644 --- a/README.md +++ b/README.md @@ -55,11 +55,11 @@ SUNSET has two different modes: - A 'debug mode' which you can run in an interactive R session. See an example here: [Use case 1.1](https://earth.bsc.es/gitlab/es/sunset/-/blob/master/use_cases/ex1_1_single_analysis_terminal/ex1_1-handson.md) -- A 'job launch' mode which you can use to run one or more jobs on an HPC platform. In this way, you can run the same analysis for multiple models, initialization dates, variables or regions at the same time, from one recipe. See an example here: [Use case 1.3](https://earth.bsc.es/gitlab/es/sunset/-/blob/master/use_cases/ex1_1_single_analysis_terminal/ex1_1-handson.md) +- A 'job launch' mode which you can use to run one or more jobs on an HPC platform. In this way, you can run the same analysis for multiple models, initialization dates, variables or regions at the same time, from one recipe. See an example here: [Use case 1.3](https://earth.bsc.es/gitlab/es/sunset/-/blob/master/use_cases/ex1_3_nino_indices_comparison/ex1_3-handson.md) You can find a summary of the differences between the two modes in this presentation: [Steps to use SUNSET](https://docs.google.com/presentation/d/1FdW25b4JZLxXDkVRVIOz-P_2A1WXLWDXhxZvisrVdqU/edit#slide=id.g308472768fe_0_124) -Resources +Documentation and Use Cases --------- You can access the documentation through the wiki: @@ -69,6 +69,9 @@ Use cases and hands-on tutorials are available in the repository for you to foll [SUNSET Use Cases](https://earth.bsc.es/gitlab/es/sunset/-/tree/master/use_cases/) +Other resources +--------- + You may also find useful information in the slides from past user meetings: [R-tools Climate Forecast Analysis Training session 2023](https://earth.bsc.es/wiki/lib/exe/fetch.php?media=tools:day2_04_handson_r_tools.pdf) -- GitLab From a79c7834b1331d8c6bf1bafb6633e2e6d73e6de9 Mon Sep 17 00:00:00 2001 From: vagudets Date: Wed, 8 Jan 2025 13:12:07 +0100 Subject: [PATCH 2/3] Request 1 node for verification jobs --- autosubmit/conf_esarchive/jobs.yml | 2 ++ autosubmit/conf_gpfs/jobs.yml | 2 ++ launch_SUNSET.sh | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/autosubmit/conf_esarchive/jobs.yml b/autosubmit/conf_esarchive/jobs.yml index 04d23ba0..0479f2a8 100644 --- a/autosubmit/conf_esarchive/jobs.yml +++ b/autosubmit/conf_esarchive/jobs.yml @@ -6,6 +6,7 @@ JOBS: NOTIFY_ON: PLATFORM: nord3v2 PROCESSORS: + NODES: 1 # SPLITS: # n_atomic_recipes, number of atomic recipes multimodel: FILE: autosubmit/auto-multimodel.sh @@ -14,6 +15,7 @@ JOBS: NOTIFY_ON: PLATFORM: nord3v2 PROCESSORS: + NODES: 1 DEPENDENCIES: verification: SPLITS_FROM: diff --git a/autosubmit/conf_gpfs/jobs.yml b/autosubmit/conf_gpfs/jobs.yml index 9f9ce281..36591feb 100644 --- a/autosubmit/conf_gpfs/jobs.yml +++ b/autosubmit/conf_gpfs/jobs.yml @@ -13,6 +13,7 @@ JOBS: NOTIFY_ON: PLATFORM: PROCESSORS: + NODES: 1 # SPLITS: # n_atomic_recipes, number of atomic recipes multimodel: FILE: autosubmit/auto-multimodel.sh @@ -22,6 +23,7 @@ JOBS: PLATFORM: PROCESSORS: DEPENDENCIES: + NODES: 1 verification: SPLITS_FROM: SPLITS: # n_atomic_recipes/n_models = n_multimodels diff --git a/launch_SUNSET.sh b/launch_SUNSET.sh index 74c16123..b95898b5 100644 --- a/launch_SUNSET.sh +++ b/launch_SUNSET.sh @@ -151,7 +151,7 @@ if [[ $run_method == "sbatch" ]]; then outfile=${logdir}/run-${job_name}.out errfile=${logdir}/run-${job_name}.err # Send batch job and capture job ID - job_ID=$(sbatch --parsable --job-name="SUNSET_verification" $platform_params --output=$outfile --error=$errfile --time=$wallclock --cpus-per-task=$cpus $custom_directives conf/slurm_templates/run_parallel_workflow.sh ${script} ${atomic_recipe}) + job_ID=$(sbatch --parsable --job-name="SUNSET_verification" $platform_params --nodes=1 --output=$outfile --error=$errfile --time=$wallclock --cpus-per-task=$cpus $custom_directives conf/slurm_templates/run_parallel_workflow.sh ${script} ${atomic_recipe}) # Add job ID to array verification_job_list+=($job_ID) echo "Submitted batch job $job_ID" @@ -166,7 +166,7 @@ if [[ $run_method == "sbatch" ]]; then outfile=${logdir}/run-multimodel-${job_name}.out errfile=${logdir}/run-multimodel-${job_name}.err # Send batch job and capture job ID - job_ID=$(sbatch --parsable --dependency=afterok:$(IFS=,; echo "${verification_job_list[*]}") --kill-on-invalid-dep=yes --job-name="SUNSET_multimodel" $platform_params --output=$outfile --error=$errfile --time=$wallclock --cpus-per-task=$cpus $custom_directives conf/slurm_templates/run_parallel_workflow.sh ${script} ${atomic_recipe}) + job_ID=$(sbatch --parsable --dependency=afterok:$(IFS=,; echo "${verification_job_list[*]}") --kill-on-invalid-dep=yes --job-name="SUNSET_multimodel" $platform_params --nodes=1 --output=$outfile --error=$errfile --time=$wallclock --cpus-per-task=$cpus $custom_directives conf/slurm_templates/run_parallel_workflow.sh ${script} ${atomic_recipe}) # Add job ID to array multimodel_job_list+=($job_ID) echo "Submitted batch job $job_ID" -- GitLab From 6c7757cbfbc57f2706e7dfc3eddef48482a8d966 Mon Sep 17 00:00:00 2001 From: vagudets Date: Wed, 8 Jan 2025 15:05:07 +0100 Subject: [PATCH 3/3] Comment 'NODES: 1' line in AS job config files --- autosubmit/conf_esarchive/jobs.yml | 6 ++++-- autosubmit/conf_gpfs/jobs.yml | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/autosubmit/conf_esarchive/jobs.yml b/autosubmit/conf_esarchive/jobs.yml index 0479f2a8..8196dcc1 100644 --- a/autosubmit/conf_esarchive/jobs.yml +++ b/autosubmit/conf_esarchive/jobs.yml @@ -6,7 +6,8 @@ JOBS: NOTIFY_ON: PLATFORM: nord3v2 PROCESSORS: - NODES: 1 + ## TODO: Uncomment (see #162) + # NODES: 1 # SPLITS: # n_atomic_recipes, number of atomic recipes multimodel: FILE: autosubmit/auto-multimodel.sh @@ -15,7 +16,8 @@ JOBS: NOTIFY_ON: PLATFORM: nord3v2 PROCESSORS: - NODES: 1 + ## TODO: Uncomment + # NODES: 1 DEPENDENCIES: verification: SPLITS_FROM: diff --git a/autosubmit/conf_gpfs/jobs.yml b/autosubmit/conf_gpfs/jobs.yml index 36591feb..e6806ca7 100644 --- a/autosubmit/conf_gpfs/jobs.yml +++ b/autosubmit/conf_gpfs/jobs.yml @@ -13,7 +13,8 @@ JOBS: NOTIFY_ON: PLATFORM: PROCESSORS: - NODES: 1 + ## TODO: Uncomment + # NODES: 1 # SPLITS: # n_atomic_recipes, number of atomic recipes multimodel: FILE: autosubmit/auto-multimodel.sh @@ -23,7 +24,8 @@ JOBS: PLATFORM: PROCESSORS: DEPENDENCIES: - NODES: 1 + ## TODO: Uncomment + # NODES: 1 verification: SPLITS_FROM: SPLITS: # n_atomic_recipes/n_models = n_multimodels -- GitLab