diff --git a/MODULES b/MODULES index e88620be04ae598f6c9310e66751ae7967880d21..8939971c9e8a1c1aa974934eaf9891a99e2be984 100644 --- a/MODULES +++ b/MODULES @@ -3,6 +3,8 @@ # WARNING: CDO HAS TO BE ON VERSION 1.9.4 # (If not, conflicts with weekly means computation could appear) +singu_prefix="" + if [[ $BSC_MACHINE == "nord3v2" ]]; then module use /gpfs/projects/bsc32/software/suselinux/11/modules/all @@ -22,6 +24,25 @@ elif [[ $BSC_MACHINE == "mn5" ]]; then source /gpfs/projects/bsc32/software/suselinux/11/software/Miniconda3/4.7.10/etc/profile.d/conda.sh conda activate /gpfs/projects/bsc32/repository/apps/conda_envs/SUNSET-env_2.0.0 +elif [[ $BSC_MACHINE == "nord4" ]]; then + + module load nord3-singu + module load bsc/current + module use /gpfs/projects/bsc32/software/suselinux/11/modules/all + unset PYTHONSTARTUP + + module load CDO/1.9.8-foss-2019b + module load R/4.1.2-foss-2019b + module load OpenMPI/4.0.5-GCC-8.3.0-nord3-v2 + module load GEOS/3.7.2-foss-2019b-Python-3.7.4 + module load GDAL/3.5.0-foss-2019b-Python-3.7.4 + module load PROJ/9.0.0-GCCcore-8.3.0 + module load Phantomjs/2.1.1 + + singu_prefix='nord3_singu_es' + # source /gpfs/projects/bsc32/software/suselinux/11/software/Miniconda3/4.7.10/etc/profile.d/conda.sh + # conda activate /gpfs/projects/bsc32/repository/apps/conda_envs/SUNSET-env_2.0.0 + elif [[ $BSC_MACHINE == "amd" ]]; then module purge diff --git a/autosubmit/auto-scorecards.sh b/autosubmit/auto-scorecards.sh index c1140526fe3bd603d463e9a33b91581cfba20d86..d6ec8a2eb8a8a22bd84f39cb8cf5f4fc1568e6d6 100644 --- a/autosubmit/auto-scorecards.sh +++ b/autosubmit/auto-scorecards.sh @@ -22,4 +22,4 @@ source MODULES set -eu export OPENSSL_CONF=/dev/null -Rscript modules/Scorecards/execute_scorecards.R ${recipe} ${srcdir} +${singu_prefix} Rscript modules/Scorecards/execute_scorecards.R ${recipe} ${srcdir} diff --git a/autosubmit/auto-verification.sh b/autosubmit/auto-verification.sh index d323c45103c593933eccfa18758a58d9c28e5e8c..25184537ac59c592f4dd348829998049a6d390b5 100644 --- a/autosubmit/auto-verification.sh +++ b/autosubmit/auto-verification.sh @@ -24,4 +24,4 @@ set +eu source MODULES set -eu -Rscript ${script} ${atomic_recipe} +${singu_prefix} Rscript ${script} ${atomic_recipe} diff --git a/autosubmit/conf_esarchive/platforms.yml b/autosubmit/conf_esarchive/platforms.yml index 78056d62973552432a9e7b55194b8c5e0ecac09a..e97609ff33a9a3576a1a1312172edc8ef36c94f2 100644 --- a/autosubmit/conf_esarchive/platforms.yml +++ b/autosubmit/conf_esarchive/platforms.yml @@ -9,3 +9,13 @@ Platforms: PROCESSORS_PER_NODE: 16 SERIAL_QUEUE: debug QUEUE: bsc_es + nord4: + TYPE: slurm + HOST: n4login1.bsc.es + PROJECT: bsc32 + ADD_PROJECT_TO_HOST: "false" + USER: + SCRATCH_DIR: /gpfs/scratch/ + QUEUE: bsc_es + PROCESSORS_PER_NODE: 48 + diff --git a/autosubmit/conf_gpfs/expdef.yml b/autosubmit/conf_gpfs/expdef.yml index 8dc29b27843729afa89be242a0d0de96bad1b3ec..bd579342640cb71924d8db524e99a2b671c5a707 100644 --- a/autosubmit/conf_gpfs/expdef.yml +++ b/autosubmit/conf_gpfs/expdef.yml @@ -1,6 +1,6 @@ DEFAULT: EXPID: - HPCARCH: nord3v2 + HPCARCH: nord4 experiment: DATELIST: MEMBERS: fc0 diff --git a/autosubmit/conf_gpfs/platforms.yml b/autosubmit/conf_gpfs/platforms.yml index 03f4b9401d687792ca8b0554c7dd94cd95865294..c1e8a48fb795df8cd61046197a11f8d56d86f5a8 100644 --- a/autosubmit/conf_gpfs/platforms.yml +++ b/autosubmit/conf_gpfs/platforms.yml @@ -9,6 +9,16 @@ Platforms: PROCESSORS_PER_NODE: 16 SERIAL_QUEUE: debug QUEUE: bsc_es + nord4: + TYPE: slurm + HOST: n4login1.bsc.es + PROJECT: bsc32 + # ADD_PROJECT_TO_HOST: "false" + USER: + SCRATCH_DIR: /gpfs/scratch/ + QUEUE: bsc_es + PROCESSORS_PER_NODE: 48 + TEMP_DIR: '' mn5: TYPE: slurm HOST: glogin1.bsc.es diff --git a/conf/autosubmit.yml b/conf/autosubmit.yml index eb37b8a0e48a77f9c9398a0028c25cac7ecaa19e..ad0b4a49ce34c85e3787185ddf09eab560987f41 100644 --- a/conf/autosubmit.yml +++ b/conf/autosubmit.yml @@ -1,5 +1,5 @@ esarchive: - platform: nord3v2 + platform: nord4 module_version: autosubmit/4.0.98-foss-2015a-Python-3.7.3 auto_version: 4.0.98 conf_format: yaml diff --git a/conf/slurm_templates/run_parallel_workflow.sh b/conf/slurm_templates/run_parallel_workflow.sh index e9ef6964e74919c809931a206d25bc731384a86b..f18d91442066dd0fdac1c51e25f7ed6f9ba1b2e8 100644 --- a/conf/slurm_templates/run_parallel_workflow.sh +++ b/conf/slurm_templates/run_parallel_workflow.sh @@ -13,4 +13,5 @@ source MODULES # conda activate condaCerise # export LD_LIBRARY_PATH=/perm/cyce/conda/envs/condaCerise/lib -Rscript ${script} ${atomic_recipe} +${singu_prefix} Rscript ${script} ${atomic_recipe} + diff --git a/conf/slurm_templates/run_scorecards.sh b/conf/slurm_templates/run_scorecards.sh index 5350b09bf7349f7f4641ef242d118fbda9ef4ebc..4d02cd2495ba44021fd1c343f839c6cdd4657c25 100644 --- a/conf/slurm_templates/run_scorecards.sh +++ b/conf/slurm_templates/run_scorecards.sh @@ -19,5 +19,5 @@ source MODULES # Export variable for phantomjs use export OPENSSL_CONF=/dev/null # Execute scorecards -Rscript modules/Scorecards/execute_scorecards.R ${recipe} ${outdir} +${singu_prefix} Rscript modules/Scorecards/execute_scorecards.R ${recipe} ${outdir} diff --git a/launch_SUNSET.sh b/launch_SUNSET.sh index b95898b549d55a954628345499c0c45b294d3b0d..193f2ebf689d08e96010b6f28cfe46767e1368f9 100644 --- a/launch_SUNSET.sh +++ b/launch_SUNSET.sh @@ -108,7 +108,7 @@ source MODULES # conda activate condaCerise # export LD_LIBRARY_PATH=/perm/cyce/conda/envs/condaCerise/lib -Rscript split.R ${recipe} $disable_unique_ID --tmpfile $tmpfile +${singu_prefix} Rscript split.R ${recipe} $disable_unique_ID --tmpfile $tmpfile # Run with Autosubmit or directly with Slurm's sbatch? run_method=$( head -1 $tmpfile | tail -1 ) @@ -119,9 +119,9 @@ if [[ $run_method == "sbatch" ]]; then # Retrieve output directory outdir=$( head -3 $tmpfile | tail -1 ) # Multimodel TRUE/FALSE - multimodel=$( head -4 $tmpfile | tail -1) + multimodel=$( head -4 $tmpfile | tail -1 ) # Scorecards TRUE/FALSE - scorecards=$( head -5 $tmpfile | tail -1) + scorecards=$( head -5 $tmpfile | tail -1 ) # Create directory for slurm output logdir=${outdir}/logs/slurm/ @@ -131,9 +131,11 @@ if [[ $run_method == "sbatch" ]]; then # Get corrected recipe recipe=${outdir}/logs/recipes/$(basename $recipe) - # Is machine MN5? + # Does machine require special queue/partition/group params? if [[ $BSC_MACHINE == "mn5" ]]; then platform_params="-A bsc32 -q gp_bsces" + elif [[ $BSC_MACHINE == "nord4" ]]; then + platform_params="-A bsc32 -q bsc_es" else platform_params="" fi diff --git a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-handson.md b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-handson.md index 8ad96b15837fd0627c93c0c442c9b48b4951ebcc..9f3b05f974d2151be657e95973a50f8895c92d73 100644 --- a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-handson.md +++ b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-handson.md @@ -31,16 +31,16 @@ You should see a git folder "sunset" under the current directory. Now you have a Since we're going to use Autosubmit to dispatch jobs, we need to have an Autosubmit experiment. Note that SUNSET uses Autosubmit >= 4.0.0. -On the workstation or the Autosubmit machine, you can create an experiment by the following commands. +On the workstation or the Autosubmit machine, you can create an experiment by the following commands. **AUTOSUBMIT EXPERIMENTS ARE REUSABLE!** You do not need to create a new one every time. ```shell -module load autosubmit/4.0.0b-foss-2015a-Python-3.7.3 -autosubmit expid -H nord3v2 -d "SUNSET use case 1_2" +module load autosubmit/4.0.98b-foss-2015a-Python-3.7.3 +autosubmit expid -H nord4 -d "SUNSET use case 1_2" ``` You will see the messages like below: ```shell -Autosubmit is running with 4.0.0b +Autosubmit is running with 4.0.98b The new experiment "a6pc" has been registered. Generating folder structure... Experiment folder: /esarchive/autosubmit/a6pc @@ -58,7 +58,7 @@ You should at least edit some items in the "Run" section: - `code_dir`: The directory where your SUNSET code is stored (i.e., the git folder) - `auto_conf$script`: The path to the script ex1_2-recipe.yml - `auto_conf$expid`: The experiment "xxxx" you just created -- `auto_conf$hpc_user`: You user ID on Nord3, which should be bsc032xxx +- `auto_conf$hpc_user`: You user ID on Nord4, which should be bsc032xxx - `auto_conf$email_address`: Your email. You can also adjust other email notification parts up to your preference. In the recipe, we ask for anomaly calculation after loading the data, calculate the skill scores and save the result for scorecards. In the Scorecard section, three regions are requested. @@ -123,7 +123,7 @@ We will start the jobs with the launcher. The SUNSET Launcher is a bash script n The bash script needs two inputs: (1) [recipe](#2-modifying-the-recipe) (2) [R script](#3-the-user-defined-script). - On your workstation or Nord3 under the SUNSET code directory, run: + On your workstation or Nord4 under the SUNSET code directory, run: ```shell bash launch_SUNSET.sh use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml use_cases/ex1_2_autosubmit_scorecards/ex1_2-script.R diff --git a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml index 3c6ed1d73957edf8062904a3cba63066670939f9..a4e4b367b8d3dcd0d4e542db9a6485dbcb348ed2 100644 --- a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml +++ b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml @@ -90,7 +90,7 @@ Run: hpc_user: bsc032762 # replace with your hpc username wallclock: 03:00 # hh:mm processors_per_job: 16 - platform: nord3v2 + platform: nord4 custom_directives: ['#SBATCH --exclusive'] email_notifications: yes # enable/disable email notifications. Change it if you want to. email_address: victoria.agudetse@bsc.es # replace with your email address diff --git a/use_cases/ex1_3_nino_indices_comparison/ex1_3-handson.md b/use_cases/ex1_3_nino_indices_comparison/ex1_3-handson.md index edf6f8f00ac8960cdbd15d777659b101e72d36b9..318a25b9501f77fb583a364fdb7f05417653a6b5 100644 --- a/use_cases/ex1_3_nino_indices_comparison/ex1_3-handson.md +++ b/use_cases/ex1_3_nino_indices_comparison/ex1_3-handson.md @@ -83,13 +83,13 @@ A complete, ready-to-use sample of this example script can be found in `use_case ## 3. Launching the jobs with the SUNSET Launcher -The first step is to connect to the HPC machine through `ssh`. When working without Autosubmit, the SUNSET Launcher should be run directly from the HPC machine where the jobs will run (for example, Nord3v2). There is no need to request an interactive session; the launcher script can be called directly from the login node. You can obtain detailed usage information by running: +The first step is to connect to the HPC machine through `ssh`. When working without Autosubmit, the SUNSET Launcher should be run directly from the HPC machine where the jobs will run (for example, Nord4). There is no need to request an interactive session; the launcher script can be called directly from the login node. You can obtain detailed usage information by running: ```shell bash launch_SUNSET.sh --help ``` -The mandatory arguments are the paths to the recipe and the script. We can also include other optional arguments to be used by SLURM, such as the number of CPUs to request (--cpus), the wallclock time for each job (--wallclock) and other extra directives (--custom_directives). You can refer to the [Nord3v2 user guide](https://www.bsc.es/user-support/nord3v2.php#jobdirectives) and the [SLURM sbatch documentation](https://slurm.schedmd.com/sbatch.html) for more information on the available options for the parameters. +The mandatory arguments are the paths to the recipe and the script. We can also include other optional arguments to be used by SLURM, such as the number of CPUs to request (--cpus), the wallclock time for each job (--wallclock) and other extra directives (--custom_directives). You can refer to the [Nord4 user guide](https://www.bsc.es/supportkc/docs/Nord4/slurm#job-directives) and the [SLURM sbatch documentation](https://slurm.schedmd.com/sbatch.html) for more information on the available options for the parameters. In this case, we are giving each job a wallclock time of 1 hour and requesting exclusive usage of all the cores in one node. The shell command to run SUNSET will look like this: