From af0574a5c9e19dfe577169496b3ddece1ffa2ebd Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 28 Jun 2024 09:21:24 +0200 Subject: [PATCH 1/6] fixed, test pending --- autosubmit/platforms/headers/slurm_header.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autosubmit/platforms/headers/slurm_header.py b/autosubmit/platforms/headers/slurm_header.py index a99b3997f..4a35ed90c 100644 --- a/autosubmit/platforms/headers/slurm_header.py +++ b/autosubmit/platforms/headers/slurm_header.py @@ -127,10 +127,10 @@ class SlurmHeader(object): :rtype: str """ if het > -1 and len(job.het['EXCLUSIVE']) > 0: - if job.het['EXCLUSIVE'][het] != '': + if job.het['EXCLUSIVE'][het] != '' and str(job.parameters['EXCLUSIVE']).lower() == 'true': return "SBATCH --exclusive" else: - if job.parameters['EXCLUSIVE'] != '': + if job.parameters['EXCLUSIVE'] != '' and str(job.parameters['EXCLUSIVE']).lower() == 'true': return "SBATCH --exclusive" return "" -- GitLab From 61bd9b41f97760428901e2941db05617f2cfe72b Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 2 Jul 2024 10:49:48 +0200 Subject: [PATCH 2/6] wip --- autosubmit/autosubmit.py | 1 + autosubmit/job/job_list.py | 2 +- test/unit/files/slurm_cmd.sh | 0 test/unit/test_scheduler_general.py | 166 ++++++++++++++++++++++++++++ test/unit/utils/common.py | 5 + 5 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 test/unit/files/slurm_cmd.sh create mode 100644 test/unit/test_scheduler_general.py diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 627c2bb31..8d3516c37 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -1473,6 +1473,7 @@ class Autosubmit: :rtype: bool """ try: + Log.info(f"Inspecting experiment {expid}") Autosubmit._check_ownership(expid, raise_error=True) exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid) tmp_path = os.path.join(exp_path, BasicConfig.LOCAL_TMP_DIR) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 883a6bd5a..d83a3df66 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -1876,7 +1876,7 @@ class JobList(object): """ unsubmitted = [job for job in self._job_list if (platform is None or job.platform.name == platform.name) and ( - job.status != Status.SUBMITTED and job.status != Status.QUEUING and job.status == Status.RUNNING and job.status == Status.COMPLETED)] + job.status != Status.SUBMITTED and job.status != Status.QUEUING and job.status != Status.RUNNING)] if wrapper: return [job for job in unsubmitted if job.packed is False] diff --git a/test/unit/files/slurm_cmd.sh b/test/unit/files/slurm_cmd.sh new file mode 100644 index 000000000..e69de29bb diff --git a/test/unit/test_scheduler_general.py b/test/unit/test_scheduler_general.py new file mode 100644 index 000000000..486fedcb2 --- /dev/null +++ b/test/unit/test_scheduler_general.py @@ -0,0 +1,166 @@ +import pytest +from pathlib import Path +from autosubmit.autosubmit import Autosubmit +from autosubmit.migrate.migrate import Migrate +from autosubmitconfigparser.config.configcommon import AutosubmitConfig +from autosubmitconfigparser.config.yamlparser import YAMLParserFactory +from autosubmitconfigparser.config.basicconfig import BasicConfig +import os + +import pwd +from log.log import AutosubmitCritical + +from test.unit.utils.common import create_database, generate_expid, create_expid + +# Maybe this should be a regression test +class TestScheduler: + + @pytest.fixture(scope='class') + def scheduler_tmpdir(self, tmpdir_factory): + folder = tmpdir_factory.mktemp(f'scheduler_tests') + os.mkdir(folder.join('scratch')) + os.mkdir(folder.join('scheduler_tmp_dir')) + file_stat = os.stat(f"{folder.strpath}") + file_owner_id = file_stat.st_uid + file_owner = pwd.getpwuid(file_owner_id).pw_name + folder.owner = file_owner + + # Write an autosubmitrc file in the temporary directory + autosubmitrc = folder.join('autosubmitrc') + autosubmitrc.write(f''' +[database] +path = {folder} +filename = tests.db + +[local] +path = {folder} + +[globallogs] +path = {folder} + +[structures] +path = {folder} + +[historicdb] +path = {folder} + +[historiclog] +path = {folder} + +[defaultstats] +path = {folder} + +''') + os.environ['AUTOSUBMIT_CONFIGURATION'] = str(folder.join('autosubmitrc')) + create_database(str(folder.join('autosubmitrc'))) + assert "tests.db" in [Path(f).name for f in folder.listdir()] + generate_expid(str(folder.join('autosubmitrc')), platform='local') + assert "t000" in [Path(f).name for f in folder.listdir()] + return folder + + @pytest.fixture(scope='class') + def prepare_scheduler(self, scheduler_tmpdir): + # touch as_misc + platforms_path = Path(f"{scheduler_tmpdir.strpath}/t000/conf/platforms_t000.yml") + jobs_path = Path(f"{scheduler_tmpdir.strpath}/t000/conf/jobs_t000.yml") + # Add each platform to test + with platforms_path.open('w') as f: + f.write(f""" +PLATFORMS: + pytest-pjm: + type: pjm + host: 127.0.0.1 + user: {scheduler_tmpdir.owner} + project: whatever + scratch_dir: {scheduler_tmpdir}/scratch + MAX_WALLCLOCK: 48:00 + TEMP_DIR: '' + MAX_PROCESSORS: 99999 + queue: dummy + pytest-slurm: + type: slurm + host: 127.0.0.1 + user: {scheduler_tmpdir.owner} + project: whatever + scratch_dir: {scheduler_tmpdir}/scratch + QUEUE: gp_debug + ADD_PROJECT_TO_HOST: false + MAX_WALLCLOCK: 48:00 + TEMP_DIR: '' + MAX_PROCESSORS: 99999 + pytest-ecaccess: + type: ecaccess + version: slurm + host: 127.0.0.1 + QUEUE: nf + EC_QUEUE: hpc + user: {scheduler_tmpdir.owner} + project: whatever + scratch_dir: {scheduler_tmpdir}/scratch + pytest-ps: + type: ps + host: 127.0.0.1 + user: {scheduler_tmpdir.owner} + project: whatever + scratch_dir: {scheduler_tmpdir}/scratch + """) + # add a job of each platform type + with jobs_path.open('w') as f: + f.write(f""" +JOBS: + job: + SCRIPT: | + echo "Hello World" + For: + PLATFORM: [ pytest-pjm , pytest-slurm, pytest-ecaccess, pytest-ps] + QUEUE: [dummy, gp_debug, nf, hpc] + NAME: [pjm, slurm, ecaccess, ps] + RUNNING: once + wallclock: 00:01 + PROCESSORS: 5 + nodes: 2 + threads: 40 + tasks: 90 + base: + SCRIPT: | + echo "Hello World" + For: + PLATFORM: [ pytest-pjm , pytest-slurm, pytest-ecaccess, pytest-ps] + QUEUE: [dummy, gp_debug, nf, hpc] + NAME: [pjm, slurm, ecaccess, ps] + RUNNING: once + wallclock: 00:01 + """) + + + + expid_dir = Path(f"{scheduler_tmpdir.strpath}/scratch/whatever/{scheduler_tmpdir.owner}/t000") + dummy_dir = Path(f"{scheduler_tmpdir.strpath}/scratch/whatever/{scheduler_tmpdir.owner}/t000/dummy_dir") + real_data = Path(f"{scheduler_tmpdir.strpath}/scratch/whatever/{scheduler_tmpdir.owner}/t000/real_data") + # write some dummy data inside scratch dir + os.makedirs(expid_dir, exist_ok=True) + os.makedirs(dummy_dir, exist_ok=True) + os.makedirs(real_data, exist_ok=True) + + with open(dummy_dir.joinpath('dummy_file'), 'w') as f: + f.write('dummy data') + # create some dummy absolute symlinks in expid_dir to test migrate function + os.symlink(dummy_dir.joinpath('dummy_file'), real_data.joinpath('dummy_symlink')) + return scheduler_tmpdir + + @pytest.fixture + def generate_cmds(self, prepare_scheduler): + create_expid(os.environ["AUTOSUBMIT_CONFIGURATION"], 't000') + Autosubmit.inspect(expid='t000',check_wrapper=False,force=True, lst=None, filter_chunks=None, filter_status=None, filter_section=None) + return prepare_scheduler + + def test_check_slurm(self, generate_cmds, scheduler_tmpdir): + # get t000_JOB_SLURM.cmd + pjm_cmd = Path(f"{scheduler_tmpdir.strpath}/t000/tmp/t000_JOB_SLURM.cmd") + pjm_basic_cmd = Path(f"{scheduler_tmpdir.strpath}/t000/tmp/t000_BASIC_SLURM.cmd") + with pjm_cmd.open() as f: + pjm_cmd_content = f.read() + assert "pjsub" in pjm_cmd_content + + + diff --git a/test/unit/utils/common.py b/test/unit/utils/common.py index 5f767bb7d..8c9067411 100644 --- a/test/unit/utils/common.py +++ b/test/unit/utils/common.py @@ -12,3 +12,8 @@ def generate_expid(envirom, platform="local"): Autosubmit.create(expid, True,False, force=True) return expid +def create_expid(envirom, expid): + os.environ['AUTOSUBMIT_CONFIGURATION'] = envirom + Autosubmit.create(expid, True,False, force=True) + + -- GitLab From 8259c29af228d7f6f0dffbf05d379dbb96721727 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 3 Jul 2024 15:51:58 +0200 Subject: [PATCH 3/6] added default pytests parameter --- test/unit/files/base_ecaccess.cmd | 27 +++++++++++++++ test/unit/files/base_pjm.cmd | 23 +++++++++++++ test/unit/files/base_ps.cmd | 7 ++++ test/unit/files/base_slurm.cmd | 26 ++++++++++++++ test/unit/files/slurm_cmd.sh | 0 test/unit/test_scheduler_general.py | 53 ++++++++++++++++++++++++----- 6 files changed, 127 insertions(+), 9 deletions(-) create mode 100644 test/unit/files/base_ecaccess.cmd create mode 100644 test/unit/files/base_pjm.cmd create mode 100644 test/unit/files/base_ps.cmd create mode 100644 test/unit/files/base_slurm.cmd delete mode 100644 test/unit/files/slurm_cmd.sh diff --git a/test/unit/files/base_ecaccess.cmd b/test/unit/files/base_ecaccess.cmd new file mode 100644 index 000000000..b3fca6828 --- /dev/null +++ b/test/unit/files/base_ecaccess.cmd @@ -0,0 +1,27 @@ +#!/bin/bash + +############################################################################### +# BASE_ECACCESS t000 EXPERIMENT +############################################################################### +# +#SBATCH --qos=nf +# +# +#SBATCH -A whatever +# +#SBATCH --cpus-per-task=1 +# +# +#SBATCH -n 1 +# +#SBATCH -t 00:01:00 +#SBATCH -J t000_BASE_ECACCESS +#SBATCH --output=/tmp/pytest-of-dbeltran/pytest-15/scheduler_tests0/scratch/whatever/dbeltran/t000/LOG_t000/t000_BASE_ECACCESS.cmd.out.0 +#SBATCH --error=/tmp/pytest-of-dbeltran/pytest-15/scheduler_tests0/scratch/whatever/dbeltran/t000/LOG_t000/t000_BASE_ECACCESS.cmd.err.0 + +# +# +############################################################################### +################### +# Autosubmit header +################### \ No newline at end of file diff --git a/test/unit/files/base_pjm.cmd b/test/unit/files/base_pjm.cmd new file mode 100644 index 000000000..b6b52eb79 --- /dev/null +++ b/test/unit/files/base_pjm.cmd @@ -0,0 +1,23 @@ +#!/bin/bash + +############################################################################### +# BASE_PJM t000 EXPERIMENT +############################################################################### +# +#PJM -N t000_BASE_PJM +#PJM -L elapse=00:01:00 +#PJM -L rscgrp=dummy +#PJM -g whatever +# + +export OMP_NUM_THREADS=1 +# +# + +#PJM -o /tmp/pytest-of-dbeltran/pytest-0/scheduler_tests0/scratch/whatever/dbeltran/t000/LOG_t000/t000_BASE_PJM.cmd.out.0 +#PJM -e /tmp/pytest-of-dbeltran/pytest-0/scheduler_tests0/scratch/whatever/dbeltran/t000/LOG_t000/t000_BASE_PJM.cmd.err.0 +# +# +############################################################################### +################### +# Autosubmit header \ No newline at end of file diff --git a/test/unit/files/base_ps.cmd b/test/unit/files/base_ps.cmd new file mode 100644 index 000000000..ae2f9c0ae --- /dev/null +++ b/test/unit/files/base_ps.cmd @@ -0,0 +1,7 @@ +#!/bin/bash + +############################################################################### +# BASE_PS t000 EXPERIMENT +############################################################################### +################### +# Autosubmit header \ No newline at end of file diff --git a/test/unit/files/base_slurm.cmd b/test/unit/files/base_slurm.cmd new file mode 100644 index 000000000..6116a60dd --- /dev/null +++ b/test/unit/files/base_slurm.cmd @@ -0,0 +1,26 @@ +#!/bin/bash + +############################################################################### +# BASE_SLURM t000 EXPERIMENT +############################################################################### +# +#SBATCH --qos=gp_debug +# +# +#SBATCH -A whatever +# +#SBATCH --cpus-per-task=1 +# +# +#SBATCH -n 1 +# +#SBATCH -t 00:01:00 +#SBATCH -J t000_BASE_SLURM +#SBATCH --output=/tmp/pytest-of-dbeltran/pytest-0/scheduler_tests0/scratch/whatever/dbeltran/t000/LOG_t000/t000_BASE_SLURM.cmd.out.0 +#SBATCH --error=/tmp/pytest-of-dbeltran/pytest-0/scheduler_tests0/scratch/whatever/dbeltran/t000/LOG_t000/t000_BASE_SLURM.cmd.err.0 + +# +# +############################################################################### +################### +# Autosubmit header diff --git a/test/unit/files/slurm_cmd.sh b/test/unit/files/slurm_cmd.sh deleted file mode 100644 index e69de29bb..000000000 diff --git a/test/unit/test_scheduler_general.py b/test/unit/test_scheduler_general.py index 486fedcb2..3f40701f2 100644 --- a/test/unit/test_scheduler_general.py +++ b/test/unit/test_scheduler_general.py @@ -6,12 +6,17 @@ from autosubmitconfigparser.config.configcommon import AutosubmitConfig from autosubmitconfigparser.config.yamlparser import YAMLParserFactory from autosubmitconfigparser.config.basicconfig import BasicConfig import os - +import sys import pwd from log.log import AutosubmitCritical from test.unit.utils.common import create_database, generate_expid, create_expid +def get_script_files_path(): + current_folder = Path(__file__).resolve().parent + print(current_folder) + files_folder = os.path.join(current_folder, 'files') + return files_folder # Maybe this should be a regression test class TestScheduler: @@ -148,19 +153,49 @@ JOBS: os.symlink(dummy_dir.joinpath('dummy_file'), real_data.joinpath('dummy_symlink')) return scheduler_tmpdir - @pytest.fixture + @pytest.fixture(scope='class') def generate_cmds(self, prepare_scheduler): create_expid(os.environ["AUTOSUBMIT_CONFIGURATION"], 't000') Autosubmit.inspect(expid='t000',check_wrapper=False,force=True, lst=None, filter_chunks=None, filter_status=None, filter_section=None) return prepare_scheduler - def test_check_slurm(self, generate_cmds, scheduler_tmpdir): - # get t000_JOB_SLURM.cmd - pjm_cmd = Path(f"{scheduler_tmpdir.strpath}/t000/tmp/t000_JOB_SLURM.cmd") - pjm_basic_cmd = Path(f"{scheduler_tmpdir.strpath}/t000/tmp/t000_BASIC_SLURM.cmd") - with pjm_cmd.open() as f: - pjm_cmd_content = f.read() - assert "pjsub" in pjm_cmd_content + def test_default_parameters(self, generate_cmds, scheduler_tmpdir): + """ + Test that the default parameters are correctly set in the scheduler files. It is a comparasion line to line, so the new templates must match the same line order as the old ones. Additional default parameters must be filled in the files/base_{scheduler}.yml as well as any change in the order + :param generate_cmds: + :param scheduler_tmpdir: + :return: + """ + # Get all expected default parameters from files/base_{scheduler}.yml + schedulers_to_test = ['pjm', 'slurm', 'ecaccess', 'ps'] + # Load the base file for each scheduler + files_folder = get_script_files_path() + default_data = {} + for base_f in Path(files_folder).glob('base_*.cmd'): + name = base_f.stem.split('_')[1].upper() + default_data[name] = Path(base_f).read_text() + for scheduler in schedulers_to_test: + scheduler = scheduler.upper() + # Get the expected default parameters for the scheduler + expected = default_data[scheduler] + # Get the actual default parameters for the scheduler + actual = Path(f"{scheduler_tmpdir.strpath}/t000/tmp/t000_BASE_{scheduler}.cmd").read_text() + # Remove all after # Autosubmit header + # ################### + # count number of lines in expected + expected_lines = expected.split('\n') + actual = actual.split('\n')[:len(expected_lines)] + actual = '\n'.join(actual) + # Compare line to line + for i, (line1, line2) in enumerate(zip(expected.split('\n'), actual.split('\n'))): + if "PJM -o" in line1 or "PJM -e" in line1 or "#SBATCH --output" in line1 or "#SBATCH --error" in line1: # output error will be different + continue + elif "##" in line1 or "##" in line2: # comment line + continue + elif "header" in line1 or "header" in line2: # header line + continue + else: + assert line1 == line2 -- GitLab From d032ccc0d60ceff46e107f3a2df641c136939c56 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 8 Jul 2024 15:50:30 +0200 Subject: [PATCH 4/6] Adressing feedback (1) --- autosubmit/platforms/headers/slurm_header.py | 4 ++-- test/unit/test_migrate.py | 4 ++-- test/unit/test_scheduler_general.py | 15 ++++----------- test/unit/utils/common.py | 11 +++++------ 4 files changed, 13 insertions(+), 21 deletions(-) diff --git a/autosubmit/platforms/headers/slurm_header.py b/autosubmit/platforms/headers/slurm_header.py index 4a35ed90c..31450c2dd 100644 --- a/autosubmit/platforms/headers/slurm_header.py +++ b/autosubmit/platforms/headers/slurm_header.py @@ -127,10 +127,10 @@ class SlurmHeader(object): :rtype: str """ if het > -1 and len(job.het['EXCLUSIVE']) > 0: - if job.het['EXCLUSIVE'][het] != '' and str(job.parameters['EXCLUSIVE']).lower() == 'true': + if str(job.parameters['EXCLUSIVE']).lower() == 'true': return "SBATCH --exclusive" else: - if job.parameters['EXCLUSIVE'] != '' and str(job.parameters['EXCLUSIVE']).lower() == 'true': + if str(job.parameters['EXCLUSIVE']).lower() == 'true': return "SBATCH --exclusive" return "" diff --git a/test/unit/test_migrate.py b/test/unit/test_migrate.py index 63a656e5d..0dd87f2e8 100644 --- a/test/unit/test_migrate.py +++ b/test/unit/test_migrate.py @@ -9,7 +9,7 @@ import os import pwd from log.log import AutosubmitCritical -from test.unit.utils.common import create_database, generate_expid +from test.unit.utils.common import create_database, init_expid class TestMigrate: @@ -53,7 +53,7 @@ path = {folder} os.environ['AUTOSUBMIT_CONFIGURATION'] = str(folder.join('autosubmitrc')) create_database(str(folder.join('autosubmitrc'))) assert "tests.db" in [Path(f).name for f in folder.listdir()] - generate_expid(str(folder.join('autosubmitrc')), platform='pytest-local') + init_expid(str(folder.join('autosubmitrc')), platform='pytest-local',create=False) assert "t000" in [Path(f).name for f in folder.listdir()] return folder diff --git a/test/unit/test_scheduler_general.py b/test/unit/test_scheduler_general.py index 3f40701f2..d766b55b2 100644 --- a/test/unit/test_scheduler_general.py +++ b/test/unit/test_scheduler_general.py @@ -1,20 +1,13 @@ import pytest from pathlib import Path from autosubmit.autosubmit import Autosubmit -from autosubmit.migrate.migrate import Migrate -from autosubmitconfigparser.config.configcommon import AutosubmitConfig -from autosubmitconfigparser.config.yamlparser import YAMLParserFactory -from autosubmitconfigparser.config.basicconfig import BasicConfig import os -import sys import pwd -from log.log import AutosubmitCritical -from test.unit.utils.common import create_database, generate_expid, create_expid +from test.unit.utils.common import create_database, init_expid def get_script_files_path(): current_folder = Path(__file__).resolve().parent - print(current_folder) files_folder = os.path.join(current_folder, 'files') return files_folder # Maybe this should be a regression test @@ -59,7 +52,7 @@ path = {folder} os.environ['AUTOSUBMIT_CONFIGURATION'] = str(folder.join('autosubmitrc')) create_database(str(folder.join('autosubmitrc'))) assert "tests.db" in [Path(f).name for f in folder.listdir()] - generate_expid(str(folder.join('autosubmitrc')), platform='local') + init_expid(str(folder.join('autosubmitrc')), platform='local', create=False) assert "t000" in [Path(f).name for f in folder.listdir()] return folder @@ -155,7 +148,7 @@ JOBS: @pytest.fixture(scope='class') def generate_cmds(self, prepare_scheduler): - create_expid(os.environ["AUTOSUBMIT_CONFIGURATION"], 't000') + init_expid(os.environ["AUTOSUBMIT_CONFIGURATION"], platform='local', expid='t000', create=True) Autosubmit.inspect(expid='t000',check_wrapper=False,force=True, lst=None, filter_chunks=None, filter_status=None, filter_section=None) return prepare_scheduler @@ -179,7 +172,7 @@ JOBS: # Get the expected default parameters for the scheduler expected = default_data[scheduler] # Get the actual default parameters for the scheduler - actual = Path(f"{scheduler_tmpdir.strpath}/t000/tmp/t000_BASE_{scheduler}.cmd").read_text() + actual = Path(f"{generate_cmds.strpath}/t000/tmp/t000_BASE_{scheduler}.cmd").read_text() # Remove all after # Autosubmit header # ################### # count number of lines in expected diff --git a/test/unit/utils/common.py b/test/unit/utils/common.py index 8c9067411..50f613c34 100644 --- a/test/unit/utils/common.py +++ b/test/unit/utils/common.py @@ -6,14 +6,13 @@ def create_database(envirom): BasicConfig.read() Autosubmit.install() -def generate_expid(envirom, platform="local"): +def init_expid(envirom, platform="local", expid=None, create=True): os.environ['AUTOSUBMIT_CONFIGURATION'] = envirom - expid = Autosubmit.expid("pytest", hpc=platform, copy_id='', dummy=True, minimal_configuration=False, git_repo="", git_branch="", git_as_conf="", operational=False, testcase = True, use_local_minimal=False) - Autosubmit.create(expid, True,False, force=True) + if not expid: + expid = Autosubmit.expid("pytest", hpc=platform, copy_id='', dummy=True, minimal_configuration=False, git_repo="", git_branch="", git_as_conf="", operational=False, testcase = True, use_local_minimal=False) + if create: + Autosubmit.create(expid, True,False, force=True) return expid -def create_expid(envirom, expid): - os.environ['AUTOSUBMIT_CONFIGURATION'] = envirom - Autosubmit.create(expid, True,False, force=True) -- GitLab From 3c019364af618544d6fd7e8eea4a4a343ece2084 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 9 Jul 2024 15:28:14 +0200 Subject: [PATCH 5/6] Test is no longer inside a class --- test/unit/test_scheduler_general.py | 178 ++++++++++++++-------------- 1 file changed, 88 insertions(+), 90 deletions(-) diff --git a/test/unit/test_scheduler_general.py b/test/unit/test_scheduler_general.py index d766b55b2..2a2b9244a 100644 --- a/test/unit/test_scheduler_general.py +++ b/test/unit/test_scheduler_general.py @@ -11,21 +11,20 @@ def get_script_files_path(): files_folder = os.path.join(current_folder, 'files') return files_folder # Maybe this should be a regression test -class TestScheduler: - - @pytest.fixture(scope='class') - def scheduler_tmpdir(self, tmpdir_factory): - folder = tmpdir_factory.mktemp(f'scheduler_tests') - os.mkdir(folder.join('scratch')) - os.mkdir(folder.join('scheduler_tmp_dir')) - file_stat = os.stat(f"{folder.strpath}") - file_owner_id = file_stat.st_uid - file_owner = pwd.getpwuid(file_owner_id).pw_name - folder.owner = file_owner - - # Write an autosubmitrc file in the temporary directory - autosubmitrc = folder.join('autosubmitrc') - autosubmitrc.write(f''' + +@pytest.fixture +def scheduler_tmpdir(tmpdir_factory): + folder = tmpdir_factory.mktemp(f'scheduler_tests') + os.mkdir(folder.join('scratch')) + os.mkdir(folder.join('scheduler_tmp_dir')) + file_stat = os.stat(f"{folder.strpath}") + file_owner_id = file_stat.st_uid + file_owner = pwd.getpwuid(file_owner_id).pw_name + folder.owner = file_owner + + # Write an autosubmitrc file in the temporary directory + autosubmitrc = folder.join('autosubmitrc') + autosubmitrc.write(f''' [database] path = {folder} filename = tests.db @@ -49,21 +48,21 @@ path = {folder} path = {folder} ''') - os.environ['AUTOSUBMIT_CONFIGURATION'] = str(folder.join('autosubmitrc')) - create_database(str(folder.join('autosubmitrc'))) - assert "tests.db" in [Path(f).name for f in folder.listdir()] - init_expid(str(folder.join('autosubmitrc')), platform='local', create=False) - assert "t000" in [Path(f).name for f in folder.listdir()] - return folder - - @pytest.fixture(scope='class') - def prepare_scheduler(self, scheduler_tmpdir): - # touch as_misc - platforms_path = Path(f"{scheduler_tmpdir.strpath}/t000/conf/platforms_t000.yml") - jobs_path = Path(f"{scheduler_tmpdir.strpath}/t000/conf/jobs_t000.yml") - # Add each platform to test - with platforms_path.open('w') as f: - f.write(f""" + os.environ['AUTOSUBMIT_CONFIGURATION'] = str(folder.join('autosubmitrc')) + create_database(str(folder.join('autosubmitrc'))) + assert "tests.db" in [Path(f).name for f in folder.listdir()] + init_expid(str(folder.join('autosubmitrc')), platform='local', create=False) + assert "t000" in [Path(f).name for f in folder.listdir()] + return folder + +@pytest.fixture +def prepare_scheduler(scheduler_tmpdir): + # touch as_misc + platforms_path = Path(f"{scheduler_tmpdir.strpath}/t000/conf/platforms_t000.yml") + jobs_path = Path(f"{scheduler_tmpdir.strpath}/t000/conf/jobs_t000.yml") + # Add each platform to test + with platforms_path.open('w') as f: + f.write(f""" PLATFORMS: pytest-pjm: type: pjm @@ -102,9 +101,9 @@ PLATFORMS: project: whatever scratch_dir: {scheduler_tmpdir}/scratch """) - # add a job of each platform type - with jobs_path.open('w') as f: - f.write(f""" + # add a job of each platform type + with jobs_path.open('w') as f: + f.write(f""" JOBS: job: SCRIPT: | @@ -132,63 +131,62 @@ JOBS: - expid_dir = Path(f"{scheduler_tmpdir.strpath}/scratch/whatever/{scheduler_tmpdir.owner}/t000") - dummy_dir = Path(f"{scheduler_tmpdir.strpath}/scratch/whatever/{scheduler_tmpdir.owner}/t000/dummy_dir") - real_data = Path(f"{scheduler_tmpdir.strpath}/scratch/whatever/{scheduler_tmpdir.owner}/t000/real_data") - # write some dummy data inside scratch dir - os.makedirs(expid_dir, exist_ok=True) - os.makedirs(dummy_dir, exist_ok=True) - os.makedirs(real_data, exist_ok=True) - - with open(dummy_dir.joinpath('dummy_file'), 'w') as f: - f.write('dummy data') - # create some dummy absolute symlinks in expid_dir to test migrate function - os.symlink(dummy_dir.joinpath('dummy_file'), real_data.joinpath('dummy_symlink')) - return scheduler_tmpdir - - @pytest.fixture(scope='class') - def generate_cmds(self, prepare_scheduler): - init_expid(os.environ["AUTOSUBMIT_CONFIGURATION"], platform='local', expid='t000', create=True) - Autosubmit.inspect(expid='t000',check_wrapper=False,force=True, lst=None, filter_chunks=None, filter_status=None, filter_section=None) - return prepare_scheduler - - def test_default_parameters(self, generate_cmds, scheduler_tmpdir): - """ - Test that the default parameters are correctly set in the scheduler files. It is a comparasion line to line, so the new templates must match the same line order as the old ones. Additional default parameters must be filled in the files/base_{scheduler}.yml as well as any change in the order - :param generate_cmds: - :param scheduler_tmpdir: - :return: - """ - # Get all expected default parameters from files/base_{scheduler}.yml - schedulers_to_test = ['pjm', 'slurm', 'ecaccess', 'ps'] - # Load the base file for each scheduler - files_folder = get_script_files_path() - default_data = {} - for base_f in Path(files_folder).glob('base_*.cmd'): - name = base_f.stem.split('_')[1].upper() - default_data[name] = Path(base_f).read_text() - for scheduler in schedulers_to_test: - scheduler = scheduler.upper() - # Get the expected default parameters for the scheduler - expected = default_data[scheduler] - # Get the actual default parameters for the scheduler - actual = Path(f"{generate_cmds.strpath}/t000/tmp/t000_BASE_{scheduler}.cmd").read_text() - # Remove all after # Autosubmit header - # ################### - # count number of lines in expected - expected_lines = expected.split('\n') - actual = actual.split('\n')[:len(expected_lines)] - actual = '\n'.join(actual) - # Compare line to line - for i, (line1, line2) in enumerate(zip(expected.split('\n'), actual.split('\n'))): - if "PJM -o" in line1 or "PJM -e" in line1 or "#SBATCH --output" in line1 or "#SBATCH --error" in line1: # output error will be different - continue - elif "##" in line1 or "##" in line2: # comment line - continue - elif "header" in line1 or "header" in line2: # header line - continue - else: - assert line1 == line2 + expid_dir = Path(f"{scheduler_tmpdir.strpath}/scratch/whatever/{scheduler_tmpdir.owner}/t000") + dummy_dir = Path(f"{scheduler_tmpdir.strpath}/scratch/whatever/{scheduler_tmpdir.owner}/t000/dummy_dir") + real_data = Path(f"{scheduler_tmpdir.strpath}/scratch/whatever/{scheduler_tmpdir.owner}/t000/real_data") + # write some dummy data inside scratch dir + os.makedirs(expid_dir, exist_ok=True) + os.makedirs(dummy_dir, exist_ok=True) + os.makedirs(real_data, exist_ok=True) + + with open(dummy_dir.joinpath('dummy_file'), 'w') as f: + f.write('dummy data') + # create some dummy absolute symlinks in expid_dir to test migrate function + os.symlink(dummy_dir.joinpath('dummy_file'), real_data.joinpath('dummy_symlink')) + return scheduler_tmpdir + +@pytest.fixture +def generate_cmds(prepare_scheduler): + init_expid(os.environ["AUTOSUBMIT_CONFIGURATION"], platform='local', expid='t000', create=True) + Autosubmit.inspect(expid='t000',check_wrapper=False,force=True, lst=None, filter_chunks=None, filter_status=None, filter_section=None) + return prepare_scheduler + +def test_default_parameters(generate_cmds): + """ + Test that the default parameters are correctly set in the scheduler files. It is a comparasion line to line, so the new templates must match the same line order as the old ones. Additional default parameters must be filled in the files/base_{scheduler}.yml as well as any change in the order + :param generate_cmds: fixture that generates the templates. + :return: + """ + # Get all expected default parameters from files/base_{scheduler}.yml + schedulers_to_test = ['pjm', 'slurm', 'ecaccess', 'ps'] + # Load the base file for each scheduler + files_folder = get_script_files_path() + default_data = {} + for base_f in Path(files_folder).glob('base_*.cmd'): + name = base_f.stem.split('_')[1].upper() + default_data[name] = Path(base_f).read_text() + for scheduler in schedulers_to_test: + scheduler = scheduler.upper() + # Get the expected default parameters for the scheduler + expected = default_data[scheduler] + # Get the actual default parameters for the scheduler + actual = Path(f"{generate_cmds.strpath}/t000/tmp/t000_BASE_{scheduler}.cmd").read_text() + # Remove all after # Autosubmit header + # ################### + # count number of lines in expected + expected_lines = expected.split('\n') + actual = actual.split('\n')[:len(expected_lines)] + actual = '\n'.join(actual) + # Compare line to line + for i, (line1, line2) in enumerate(zip(expected.split('\n'), actual.split('\n'))): + if "PJM -o" in line1 or "PJM -e" in line1 or "#SBATCH --output" in line1 or "#SBATCH --error" in line1: # output error will be different + continue + elif "##" in line1 or "##" in line2: # comment line + continue + elif "header" in line1 or "header" in line2: # header line + continue + else: + assert line1 == line2 -- GitLab From fd8786ed544996e2bf482f9e76e979d6036be6ef Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 10 Jul 2024 14:57:00 +0200 Subject: [PATCH 6/6] Bruno feeback --- test/unit/test_scheduler_general.py | 69 ++++++++++++++--------------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/test/unit/test_scheduler_general.py b/test/unit/test_scheduler_general.py index 2a2b9244a..46e62ef79 100644 --- a/test/unit/test_scheduler_general.py +++ b/test/unit/test_scheduler_general.py @@ -6,10 +6,9 @@ import pwd from test.unit.utils.common import create_database, init_expid -def get_script_files_path(): - current_folder = Path(__file__).resolve().parent - files_folder = os.path.join(current_folder, 'files') - return files_folder +def _get_script_files_path() -> Path: + return Path(__file__).resolve().parent / 'files' + # Maybe this should be a regression test @pytest.fixture @@ -142,7 +141,7 @@ JOBS: with open(dummy_dir.joinpath('dummy_file'), 'w') as f: f.write('dummy data') # create some dummy absolute symlinks in expid_dir to test migrate function - os.symlink(dummy_dir.joinpath('dummy_file'), real_data.joinpath('dummy_symlink')) + (real_data / 'dummy_symlink').symlink_to(dummy_dir / 'dummy_file') return scheduler_tmpdir @pytest.fixture @@ -151,42 +150,42 @@ def generate_cmds(prepare_scheduler): Autosubmit.inspect(expid='t000',check_wrapper=False,force=True, lst=None, filter_chunks=None, filter_status=None, filter_section=None) return prepare_scheduler -def test_default_parameters(generate_cmds): +@pytest.mark.parametrize("scheduler", ['pjm', 'slurm', 'ecaccess', 'ps']) +def test_default_parameters(scheduler: str, generate_cmds): """ Test that the default parameters are correctly set in the scheduler files. It is a comparasion line to line, so the new templates must match the same line order as the old ones. Additional default parameters must be filled in the files/base_{scheduler}.yml as well as any change in the order :param generate_cmds: fixture that generates the templates. :return: """ - # Get all expected default parameters from files/base_{scheduler}.yml - schedulers_to_test = ['pjm', 'slurm', 'ecaccess', 'ps'] + # Load the base file for each scheduler - files_folder = get_script_files_path() - default_data = {} - for base_f in Path(files_folder).glob('base_*.cmd'): - name = base_f.stem.split('_')[1].upper() - default_data[name] = Path(base_f).read_text() - for scheduler in schedulers_to_test: - scheduler = scheduler.upper() - # Get the expected default parameters for the scheduler - expected = default_data[scheduler] - # Get the actual default parameters for the scheduler - actual = Path(f"{generate_cmds.strpath}/t000/tmp/t000_BASE_{scheduler}.cmd").read_text() - # Remove all after # Autosubmit header - # ################### - # count number of lines in expected - expected_lines = expected.split('\n') - actual = actual.split('\n')[:len(expected_lines)] - actual = '\n'.join(actual) - # Compare line to line - for i, (line1, line2) in enumerate(zip(expected.split('\n'), actual.split('\n'))): - if "PJM -o" in line1 or "PJM -e" in line1 or "#SBATCH --output" in line1 or "#SBATCH --error" in line1: # output error will be different - continue - elif "##" in line1 or "##" in line2: # comment line - continue - elif "header" in line1 or "header" in line2: # header line - continue - else: - assert line1 == line2 + scheduler = scheduler.upper() + expected_data = {} + for base_f in _get_script_files_path().glob('base_*.cmd'): + if scheduler in base_f.stem.split('_')[1].upper(): + expected_data = Path(base_f).read_text() + break + if not expected_data: + raise NotImplemented + + # Get the actual default parameters for the scheduler + actual = Path(f"{generate_cmds.strpath}/t000/tmp/t000_BASE_{scheduler}.cmd").read_text() + # Remove all after # Autosubmit header + # ################### + # count number of lines in expected + expected_lines = expected_data.split('\n') + actual = actual.split('\n')[:len(expected_lines)] + actual = '\n'.join(actual) + # Compare line to line + for i, (line1, line2) in enumerate(zip(expected_data.split('\n'), actual.split('\n'))): + if "PJM -o" in line1 or "PJM -e" in line1 or "#SBATCH --output" in line1 or "#SBATCH --error" in line1: # output error will be different + continue + elif "##" in line1 or "##" in line2: # comment line + continue + elif "header" in line1 or "header" in line2: # header line + continue + else: + assert line1 == line2 -- GitLab