From e2a8b3db6d7d302213114aec141de9565a8ebf45 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 19 Nov 2024 13:28:19 +0100 Subject: [PATCH 1/6] Fixes !1458, pending pytest --- autosubmit/job/job.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index ba4d70407..c4318217d 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1352,14 +1352,30 @@ class Job(object): self.status = default_status else: return default_status + + def update_current_parameters(self, as_conf: AutosubmitConfig, parameters: dict) -> dict: + """ + Update the %CURRENT_*% parameters with the current platform and jobs. + + :param as_conf: The Autosubmit configuration object. + :type as_conf: AutosubmitConfig + :param parameters: The dictionary to update with current parameters. + :type parameters: dict + :return: The updated parameter's dictionary. + :rtype: dict + """ + if self.platform_name.upper() != "LOCAL": + for key, value in as_conf.platforms_data[self.platform_name].items(): + parameters[f"CURRENT_{key.upper()}"] = value + for key, value in as_conf.jobs_data[self.section].items(): + parameters[f"CURRENT_{key.upper()}"] = value + return parameters def update_platform_parameters(self,as_conf,parameters,job_platform): if not job_platform: submitter = job_utils._get_submitter(as_conf) submitter.load_platforms(as_conf) job_platform = submitter.platforms[self.platform_name] self.platform = job_platform - for key,value in as_conf.platforms_data.get(job_platform.name,{}).items(): - parameters["CURRENT_"+key.upper()] = value parameters['CURRENT_ARCH'] = job_platform.name parameters['CURRENT_HOST'] = job_platform.host parameters['CURRENT_USER'] = job_platform.user @@ -1921,8 +1937,6 @@ class Job(object): parameters['PROJECT_TYPE'] = as_conf.get_project_type() parameters['X11'] = self.x11 self.wchunkinc = as_conf.get_wchunkinc(self.section) - for key,value in as_conf.jobs_data[self.section].items(): - parameters["CURRENT_"+key.upper()] = value return parameters @@ -1986,7 +2000,8 @@ class Job(object): parameters['PROJDIR'] = as_conf.get_project_dir() # Set parameters dictionary # Set final value - parameters = self.update_job_parameters(as_conf,parameters) + parameters = self.update_current_parameters(as_conf, parameters) + parameters = self.update_job_parameters(as_conf, parameters) parameters = self.update_platform_parameters(as_conf, parameters, self._platform) parameters = self.update_platform_associated_parameters(as_conf, parameters, self._platform, parameters['CHUNK']) parameters = self.update_wrapper_parameters(as_conf, parameters) -- GitLab From 93f0bf492f6793582e897cfa972ef67ae3d25ee5 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 27 Nov 2024 17:20:55 +0100 Subject: [PATCH 2/6] Fixes #1467 --- autosubmit/job/job.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index c4318217d..8bcffb0eb 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -2005,9 +2005,8 @@ class Job(object): parameters = self.update_platform_parameters(as_conf, parameters, self._platform) parameters = self.update_platform_associated_parameters(as_conf, parameters, self._platform, parameters['CHUNK']) parameters = self.update_wrapper_parameters(as_conf, parameters) - parameters = as_conf.normalize_parameters_keys(parameters,default_parameters) + parameters = as_conf.deep_read_loops(parameters) parameters = as_conf.substitute_dynamic_variables(parameters,80) - parameters = as_conf.normalize_parameters_keys(parameters,default_parameters) self.update_job_variables_final_values(parameters) # For some reason, there is return but the assignee is also necessary self.parameters = parameters @@ -2152,6 +2151,7 @@ class Job(object): if lang is None: lang = 'UTF-8' parameters = self.parameters + parameters = self.update_parameters(as_conf, parameters) template_content,additional_templates = self.update_content(as_conf) #enumerate and get value #TODO regresion test -- GitLab From 61e5733a7297c9a396dd4ebd81272fa312bf67c2 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 27 Nov 2024 17:25:20 +0100 Subject: [PATCH 3/6] delete unnecesary addition --- autosubmit/job/job.py | 1 - 1 file changed, 1 deletion(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 8bcffb0eb..1ebe28261 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -2151,7 +2151,6 @@ class Job(object): if lang is None: lang = 'UTF-8' parameters = self.parameters - parameters = self.update_parameters(as_conf, parameters) template_content,additional_templates = self.update_content(as_conf) #enumerate and get value #TODO regresion test -- GitLab From 3086145bed95e5681ca8c7c6351781daf5e8c1ad Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 28 Nov 2024 09:30:27 +0100 Subject: [PATCH 4/6] Added pytest --- autosubmit/job/job.py | 2 +- test/unit/conftest.py | 3 ++- test/unit/test_job.py | 8 ++---- test/unit/test_job_pytest.py | 52 ++++++++++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 8 deletions(-) create mode 100644 test/unit/test_job_pytest.py diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 1ebe28261..d1967c377 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1364,7 +1364,7 @@ class Job(object): :return: The updated parameter's dictionary. :rtype: dict """ - if self.platform_name.upper() != "LOCAL": + if self.platform_name != "LOCAL": for key, value in as_conf.platforms_data[self.platform_name].items(): parameters[f"CURRENT_{key.upper()}"] = value for key, value in as_conf.jobs_data[self.section].items(): diff --git a/test/unit/conftest.py b/test/unit/conftest.py index b83345b03..f2ab6ab27 100644 --- a/test/unit/conftest.py +++ b/test/unit/conftest.py @@ -168,6 +168,7 @@ def autosubmit_config( for arg, value in kwargs.items(): setattr(config, arg, value) + config.current_loaded_files = [conf_dir / 'dummy-so-it-doesnt-force-reload.yml'] return config def finalizer() -> None: @@ -177,4 +178,4 @@ def autosubmit_config( request.addfinalizer(finalizer) - return _create_autosubmit_config \ No newline at end of file + return _create_autosubmit_config diff --git a/test/unit/test_job.py b/test/unit/test_job.py index 02133f1af..9980e7f68 100644 --- a/test/unit/test_job.py +++ b/test/unit/test_job.py @@ -1088,6 +1088,7 @@ CONFIG: 'WALLCLOCK': wallclock, 'CUSTOM_DIRECTIVES': custom_directives, 'SCRATCH_FREE_SPACE': 0, + 'PLATFORM': 'dummy_platform', } self.as_conf.jobs_data[section] = options @@ -1108,7 +1109,7 @@ CONFIG: self.as_conf.normalize_parameters_keys = MagicMock() self.as_conf.normalize_parameters_keys.return_value = default self.job._platform = dummy_platform - self.as_conf.platforms_data = { "dummy_platform":{ "whatever":"dummy_value", "whatever2":"dummy_value2"} } + self.as_conf.platforms_data = { "DUMMY_PLATFORM":{ "whatever":"dummy_value", "whatever2":"dummy_value2"} } parameters = {} # Act @@ -1440,8 +1441,3 @@ class FakeBasicConfig: DEFAULT_PLATFORMS_CONF = '' DEFAULT_JOBS_CONF = '' STRUCTURES_DIR = '/dummy/structures/dir' - - - - - diff --git a/test/unit/test_job_pytest.py b/test/unit/test_job_pytest.py new file mode 100644 index 000000000..421429fbc --- /dev/null +++ b/test/unit/test_job_pytest.py @@ -0,0 +1,52 @@ +import pytest + +from autosubmit.job.job import Job +from autosubmit.platforms.psplatform import PsPlatform + + +@pytest.mark.parametrize('experiment_data, expected_data', [( + { + 'JOBS': { + 'RANDOM-SECTION': { + 'FILE': "test.sh", + 'PLATFORM': 'DUMMY_PLATFORM', + 'TEST': "%other%" + }, + }, + 'PLATFORMS': { + 'dummy_platform': { + 'type': 'ps', + 'whatever': 'dummy_value', + 'whatever2': 'dummy_value2', + 'CUSTOM_DIRECTIVES': ['$SBATCH directive1', '$SBATCH directive2'], + }, + }, + 'OTHER': "%CURRENT_WHATEVER%/%CURRENT_WHATEVER2%", + 'ROOTDIR': 'dummy_rootdir', + 'LOCAL_TMP_DIR': 'dummy_tmpdir', + 'LOCAL_ROOT_DIR': 'dummy_rootdir', + }, + { + 'CURRENT_FILE': "test.sh", + 'CURRENT_PLATFORM': 'DUMMY_PLATFORM', + 'CURRENT_WHATEVER': 'dummy_value', + 'CURRENT_WHATEVER2': 'dummy_value2', + 'CURRENT_TEST': 'dummy_value/dummy_value2', + + } +)]) +def test_update_parameters_current_variables(autosubmit_config, experiment_data, expected_data): + as_conf = autosubmit_config("test-expid", experiment_data) + as_conf.experiment_data = as_conf.deep_normalize(as_conf.experiment_data) + as_conf.experiment_data = as_conf.normalize_variables(as_conf.experiment_data, must_exists=True) + as_conf.experiment_data = as_conf.deep_read_loops(as_conf.experiment_data) + as_conf.experiment_data = as_conf.substitute_dynamic_variables(as_conf.experiment_data) + as_conf.experiment_data = as_conf.parse_data_loops(as_conf.experiment_data) + # Create some jobs + job = Job('A', '1', 0, 1) + platform = PsPlatform(expid='a000', name='DUMMY_PLATFORM', config=as_conf.experiment_data) + job.section = 'RANDOM-SECTION' + job.platform = platform + job.update_parameters(as_conf, {}) + for key, value in expected_data.items(): + assert job.parameters[key] == value -- GitLab From 9d9714ce533ff3f45763bdd87a6939786da64a7d Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 28 Nov 2024 15:38:09 +0100 Subject: [PATCH 5/6] added comment --- autosubmit/job/job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index d1967c377..627015989 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1364,7 +1364,7 @@ class Job(object): :return: The updated parameter's dictionary. :rtype: dict """ - if self.platform_name != "LOCAL": + if self.platform_name != "LOCAL": # Local platform doesn't have any configuration in the yaml file for key, value in as_conf.platforms_data[self.platform_name].items(): parameters[f"CURRENT_{key.upper()}"] = value for key, value in as_conf.jobs_data[self.section].items(): -- GitLab From 0c936dbe7349b92bc8bc0b2414cf682cee230aee Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 28 Nov 2024 15:39:04 +0100 Subject: [PATCH 6/6] changed the function --- autosubmit/job/job.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 627015989..914ef275b 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1364,9 +1364,8 @@ class Job(object): :return: The updated parameter's dictionary. :rtype: dict """ - if self.platform_name != "LOCAL": # Local platform doesn't have any configuration in the yaml file - for key, value in as_conf.platforms_data[self.platform_name].items(): - parameters[f"CURRENT_{key.upper()}"] = value + for key, value in as_conf.platforms_data.get(self.platform_name, {}).items(): + parameters[f"CURRENT_{key.upper()}"] = value for key, value in as_conf.jobs_data[self.section].items(): parameters[f"CURRENT_{key.upper()}"] = value return parameters -- GitLab