From 3763d143621e63634973bc9279e0bf025b719c4f Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 2 Sep 2024 09:58:40 +0200 Subject: [PATCH] fixes #1407 --- autosubmit/job/job_packager.py | 2 +- autosubmit/platforms/headers/slurm_header.py | 4 +-- test/unit/files/nodes_pjm.cmd | 23 +++++++++++++++ test/unit/files/nodes_slurm.cmd | 26 +++++++++++++++++ test/unit/test_scheduler_general.py | 30 +++++++++++--------- 5 files changed, 69 insertions(+), 16 deletions(-) create mode 100644 test/unit/files/nodes_pjm.cmd create mode 100644 test/unit/files/nodes_slurm.cmd diff --git a/autosubmit/job/job_packager.py b/autosubmit/job/job_packager.py index f452d40ff..16e78bb87 100644 --- a/autosubmit/job/job_packager.py +++ b/autosubmit/job/job_packager.py @@ -922,7 +922,7 @@ class JobPackagerHorizontal(object): if jobs_processed % 10 == 0 and jobs_processed > 0: Log.info(f"Wrapper package creation is still ongoing. So far {jobs_processed} jobs have been wrapped.") job.update_parameters(wrapper_info[-1], {}) - if str(job.processors).isdigit() and str(job.nodes).isdigit() and int(job.nodes) > 1 and int(job.processors) <= 1: + if str(job.processors).isdigit() and str(job.nodes).isdigit() and int(job.nodes) > 0 and int(job.processors) <= 1: job.processors = 0 if job.total_processors == "": job_total_processors = 0 diff --git a/autosubmit/platforms/headers/slurm_header.py b/autosubmit/platforms/headers/slurm_header.py index 31450c2dd..b87802f08 100644 --- a/autosubmit/platforms/headers/slurm_header.py +++ b/autosubmit/platforms/headers/slurm_header.py @@ -55,7 +55,7 @@ class SlurmHeader(object): job_nodes = 0 else: job_nodes = job.het['NODES'][het] - if len(job.het['PROCESSORS']) == 0 or job.het['PROCESSORS'][het] == '' or job.het['PROCESSORS'][het] == '1' and int(job_nodes) > 1: + if len(job.het['PROCESSORS']) == 0 or job.het['PROCESSORS'][het] == '' or job.het['PROCESSORS'][het] == '1' and int(job_nodes) > 0: return "" else: return "SBATCH -n {0}".format(job.het['PROCESSORS'][het]) @@ -63,7 +63,7 @@ class SlurmHeader(object): job_nodes = 0 else: job_nodes = job.nodes - if job.processors == '' or job.processors == '1' and int(job_nodes) > 1: + if job.processors == '' or job.processors == '1' and int(job_nodes) > 0: return "" else: return "SBATCH -n {0}".format(job.processors) diff --git a/test/unit/files/nodes_pjm.cmd b/test/unit/files/nodes_pjm.cmd new file mode 100644 index 000000000..c305f4dee --- /dev/null +++ b/test/unit/files/nodes_pjm.cmd @@ -0,0 +1,23 @@ +#!/bin/bash + +############################################################################### +# NODES_PJM t000 EXPERIMENT +############################################################################### +# +#PJM -N t000_NODES_PJM +#PJM -L elapse=00:01:00 +#PJM -L rscgrp=dummy +#PJM -g whatever +# + +export OMP_NUM_THREADS=40 +# +#PJM -L node=1 + +#PJM -o /tmp/pytest-of-dbeltran/pytest-2/scheduler_tests1/scratch/whatever/dbeltran/t000/LOG_t000/t000_NODES_PJM.cmd.out.0 +#PJM -e /tmp/pytest-of-dbeltran/pytest-2/scheduler_tests1/scratch/whatever/dbeltran/t000/LOG_t000/t000_NODES_PJM.cmd.err.0 +# +# +############################################################################### +################### +# Autosubmit header \ No newline at end of file diff --git a/test/unit/files/nodes_slurm.cmd b/test/unit/files/nodes_slurm.cmd new file mode 100644 index 000000000..024b2ca91 --- /dev/null +++ b/test/unit/files/nodes_slurm.cmd @@ -0,0 +1,26 @@ +#!/bin/bash + +############################################################################### +# NODES_SLURM t000 EXPERIMENT +############################################################################### +# +#SBATCH --qos=gp_debug +# +# +#SBATCH -A whatever +# +#SBATCH --cpus-per-task=40 +#SBATCH --ntasks-per-node=90 +#SBATCH --nodes=1 +# +# +#SBATCH -t 00:01:00 +#SBATCH -J t000_NODES_SLURM +#SBATCH --output=/tmp/pytest-of-dbeltran/pytest-2/scheduler_tests1/scratch/whatever/dbeltran/t000/LOG_t000/t000_NODES_SLURM.cmd.out.0 +#SBATCH --error=/tmp/pytest-of-dbeltran/pytest-2/scheduler_tests1/scratch/whatever/dbeltran/t000/LOG_t000/t000_NODES_SLURM.cmd.err.0 + +# +# +############################################################################### +################### +# Autosubmit header \ No newline at end of file diff --git a/test/unit/test_scheduler_general.py b/test/unit/test_scheduler_general.py index 42d6e1b9a..39215d6d8 100644 --- a/test/unit/test_scheduler_general.py +++ b/test/unit/test_scheduler_general.py @@ -109,7 +109,7 @@ PLATFORMS: with jobs_path.open('w') as f: f.write(f""" JOBS: - job: + nodes: SCRIPT: | echo "Hello World" For: @@ -118,8 +118,7 @@ JOBS: NAME: [pjm, slurm, ecaccess, ps] RUNNING: once wallclock: 00:01 - PROCESSORS: 5 - nodes: 2 + nodes: 1 threads: 40 tasks: 90 base: @@ -170,10 +169,6 @@ EXPERIMENT: CALENDAR: standard """) - - - - expid_dir = Path(f"{scheduler_tmpdir.strpath}/scratch/whatever/{scheduler_tmpdir.owner}/t000") dummy_dir = Path(f"{scheduler_tmpdir.strpath}/scratch/whatever/{scheduler_tmpdir.owner}/t000/dummy_dir") real_data = Path(f"{scheduler_tmpdir.strpath}/scratch/whatever/{scheduler_tmpdir.owner}/t000/real_data") @@ -195,10 +190,12 @@ def generate_cmds(prepare_scheduler): return prepare_scheduler @pytest.mark.parametrize("scheduler, job_type", [ - ('pjm', 'SINGLE'), - ('slurm', 'SINGLE'), - ('ecaccess', 'SINGLE'), - ('ps', 'SINGLE'), + ('pjm', 'DEFAULT'), + ('slurm', 'DEFAULT'), + ('ecaccess', 'DEFAULT'), + ('ps', 'DEFAULT'), + ('pjm', 'NODES'), + ('slurm', 'NODES'), ('slurm', 'horizontal'), ('slurm', 'vertical'), ('slurm', 'horizontal_vertical'), @@ -218,19 +215,26 @@ def test_scheduler_job_types(scheduler, job_type, generate_cmds): scheduler = scheduler.upper() job_type = job_type.upper() expected_data = {} - if job_type == "SINGLE": + if job_type == "DEFAULT": for base_f in _get_script_files_path().glob('base_*.cmd'): if scheduler in base_f.stem.split('_')[1].upper(): expected_data = Path(base_f).read_text() break + elif job_type == "NODES": + for nodes_f in _get_script_files_path().glob('nodes_*.cmd'): + if scheduler in nodes_f.stem.split('_')[1].upper(): + expected_data = Path(nodes_f).read_text() + break else: expected_data = (Path(_get_script_files_path()) / Path(f"base_{job_type.lower()}_{scheduler.lower()}.cmd")).read_text() if not expected_data: assert False, f"Could not find the expected data for {scheduler} and {job_type}" # Get the actual default parameters for the scheduler - if job_type == "SINGLE": + if job_type == "DEFAULT": actual = Path(f"{generate_cmds.strpath}/t000/tmp/t000_BASE_{scheduler}.cmd").read_text() + elif job_type == "NODES": + actual = Path(f"{generate_cmds.strpath}/t000/tmp/t000_NODES_{scheduler}.cmd").read_text() else: for asthread in Path(f"{generate_cmds.strpath}/t000/tmp").glob(f"*ASThread_WRAP_{job_type}_[0-9]*.cmd"): actual = asthread.read_text() -- GitLab