From 158da280a6fc7fe9df1c6f561d491e1252eb2499 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 12 Nov 2021 18:12:35 +0100 Subject: [PATCH 1/7] Added a way to use %_% variables defined in proj.conf inside the same proj.conf --- autosubmit/job/job.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index e799bf013..6ab7caade 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1037,6 +1037,30 @@ class Job(object): "PARAMETER export: Variable: {0} doesn't exist".format(e.message)) parameters['EXPORT'] = self.export + #PROJECT VARIABLES: + if parameters['PROJECT_TYPE'] != "none" and as_conf._proj_parser is not None: + # Load project parameters + proj_param = as_conf.load_project_parameters() + variables_to_modify = [] + deep_level = 1 + max_deep_level = 10 + for key, value in proj_param.items(): + if type(proj_param[key]) is str: + look_non_sub_variables = re.findall('%(? 0: + variables_to_modify.append(key) + while len(variables_to_modify) > 0 and deep_level < max_deep_level: + for key, value in parameters.items(): + for var in variables_to_modify: + proj_param[var] = re.sub('%(? 0: + variables_to_modify.append(key) + parameters.update(proj_param) self.parameters = parameters return parameters -- GitLab From 1aa91afefb7af746f5d5282f2d6e896e5894a60c Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 15 Nov 2021 11:16:28 +0100 Subject: [PATCH 2/7] Added a way to use %_% variables defined in proj.conf inside the same proj.conf --- autosubmit/autosubmit.py | 4 ++-- autosubmit/job/job.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 11b00bbee..7d2caccb7 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -714,9 +714,9 @@ class Autosubmit: #Enforce LANG=C try: try: - locale.setlocale(locale.LC_ALL,'C.utf8') + locale.setlocale(locale.LC_ALL,'C.UTF-8') except: - locale.setlocale(locale.LC_ALL, 'C.UTF-8') + locale.setlocale(locale.LC_ALL, 'C.utf8') except: Log.info("Locale C.utf8 is not found, using '{0}' as fallback".format("C")) locale.setlocale(locale.LC_ALL, 'C') diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 6ab7caade..c6059ad9d 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1038,7 +1038,7 @@ class Job(object): parameters['EXPORT'] = self.export #PROJECT VARIABLES: - if parameters['PROJECT_TYPE'] != "none" and as_conf._proj_parser is not None: + if as_conf._proj_parser is not None and parameters['PROJECT_TYPE'] != "none": # Load project parameters proj_param = as_conf.load_project_parameters() variables_to_modify = [] -- GitLab From bc5994e2bc03e2ced4567a1946e433c883384fe6 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 15 Nov 2021 11:34:29 +0100 Subject: [PATCH 3/7] Added a way to use %_% variables defined in proj.conf inside the same proj.conf --- test/unit/test_job.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/test_job.py b/test/unit/test_job.py index 5248377d4..d3f58f03d 100644 --- a/test/unit/test_job.py +++ b/test/unit/test_job.py @@ -266,6 +266,7 @@ class TestJob(TestCase): dummy_platform.serial_platform = dummy_serial_platform dummy_platform.custom_directives = '["whatever"]' self.job._platform = dummy_platform + self.job.parameters['PROJECT_TYPE'] = "none" # Act parameters = self.job.update_parameters(as_conf, dict()) # Assert -- GitLab From bef10584e329633d3be4de74196605c8ff527d38 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 15 Nov 2021 11:44:44 +0100 Subject: [PATCH 4/7] Added a way to use %_% variables defined in proj.conf inside the same proj.conf --- autosubmit/job/job.py | 1 + test/unit/test_job.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index c6059ad9d..f75c7b5cc 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1038,6 +1038,7 @@ class Job(object): parameters['EXPORT'] = self.export #PROJECT VARIABLES: + parameters['PROJECT_TYPE'] = as_conf.get_project_type().lower() if as_conf._proj_parser is not None and parameters['PROJECT_TYPE'] != "none": # Load project parameters proj_param = as_conf.load_project_parameters() diff --git a/test/unit/test_job.py b/test/unit/test_job.py index d3f58f03d..9fdb54146 100644 --- a/test/unit/test_job.py +++ b/test/unit/test_job.py @@ -259,14 +259,14 @@ class TestJob(TestCase): as_conf.get_wallclock = Mock(return_value='00:30') as_conf.get_member_list = Mock(return_value=[]) as_conf.get_custom_directives = Mock(return_value='["whatever"]') - + self.job.parameters['PROJECT_TYPE'] = "none" dummy_serial_platform = Mock() dummy_serial_platform.name = 'serial' dummy_platform = Mock() dummy_platform.serial_platform = dummy_serial_platform dummy_platform.custom_directives = '["whatever"]' self.job._platform = dummy_platform - self.job.parameters['PROJECT_TYPE'] = "none" + # Act parameters = self.job.update_parameters(as_conf, dict()) # Assert -- GitLab From 2e0f5646db0ac8fa67a4ea4f8c592e9be2d2ee8b Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 15 Nov 2021 12:31:03 +0100 Subject: [PATCH 5/7] Added a way to use %_% variables defined in proj.conf inside the same proj.conf --- test/unit/test_job.py | 2 ++ test/unit/test_job_package.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/test/unit/test_job.py b/test/unit/test_job.py index 9fdb54146..f942baebb 100644 --- a/test/unit/test_job.py +++ b/test/unit/test_job.py @@ -259,12 +259,14 @@ class TestJob(TestCase): as_conf.get_wallclock = Mock(return_value='00:30') as_conf.get_member_list = Mock(return_value=[]) as_conf.get_custom_directives = Mock(return_value='["whatever"]') + as_conf.load_project_parameters = Mock(return_value='[]') self.job.parameters['PROJECT_TYPE'] = "none" dummy_serial_platform = Mock() dummy_serial_platform.name = 'serial' dummy_platform = Mock() dummy_platform.serial_platform = dummy_serial_platform dummy_platform.custom_directives = '["whatever"]' + self.job._platform = dummy_platform # Act diff --git a/test/unit/test_job_package.py b/test/unit/test_job_package.py index a695c95bb..2f1ffd52c 100644 --- a/test/unit/test_job_package.py +++ b/test/unit/test_job_package.py @@ -45,6 +45,8 @@ class TestJobPackage(TestCase): def test_job_package_submission(self): # arrange + for job in self.jobs: + job._get_paramiko_template = Mock("false","empty") self.job_package._create_scripts = Mock() self.job_package._send_files = Mock() self.job_package._do_submission = Mock() -- GitLab From 3bdee6dc140099fb9d12707c3a03dd4ec16392ce Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 15 Nov 2021 12:46:55 +0100 Subject: [PATCH 6/7] Fix pipeline and inner_retrials when non_defined --- autosubmit/job/job_packages.py | 1 + test/unit/test_job.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/autosubmit/job/job_packages.py b/autosubmit/job/job_packages.py index c34338bad..a9d25b562 100644 --- a/autosubmit/job/job_packages.py +++ b/autosubmit/job/job_packages.py @@ -362,6 +362,7 @@ class JobPackageThread(JobPackageBase): self._jobs_resources = jobs_resources self._wrapper_factory = self.platform.wrapper self.current_wrapper_section = wrapper_section + self.inner_retrials = 0 if configuration is not None: self.inner_retrials = configuration.get_retrials() self.export = configuration.get_wrapper_export(self.current_wrapper_section) diff --git a/test/unit/test_job.py b/test/unit/test_job.py index f942baebb..d1d3e91b5 100644 --- a/test/unit/test_job.py +++ b/test/unit/test_job.py @@ -173,7 +173,9 @@ class TestJob(TestCase): with patch.object(builtins, "open", open_mock): # act self.job.create_script(config) - + with patch.object(builtins, "write", write_mock): + # act + self.job.create_script(config) # assert update_content_mock.assert_called_with(config) open_mock.assert_called_with(os.path.join(self.job._tmp_path, self.job.name + '.cmd'), 'w') @@ -259,7 +261,7 @@ class TestJob(TestCase): as_conf.get_wallclock = Mock(return_value='00:30') as_conf.get_member_list = Mock(return_value=[]) as_conf.get_custom_directives = Mock(return_value='["whatever"]') - as_conf.load_project_parameters = Mock(return_value='[]') + as_conf.load_project_parameters = Mock(return_value=[]) self.job.parameters['PROJECT_TYPE'] = "none" dummy_serial_platform = Mock() dummy_serial_platform.name = 'serial' -- GitLab From 1a9f4c920a18fda6a06d3a9a82131a31fc7a3478 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 15 Nov 2021 13:03:18 +0100 Subject: [PATCH 7/7] Fix pipeline and inner_retrials when non_defined --- autosubmit/autosubmit.py | 3 ++- autosubmit/job/job.py | 3 +-- test/unit/test_job.py | 5 +---- test/unit/test_job_package.py | 3 +++ 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 7d2caccb7..999d5896d 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -1872,6 +1872,7 @@ class Autosubmit: e.message) # Restore platforms and try again, to avoid endless loop with failed configuration, a hard limit is set. reconnected = False + send_mail = True while not reconnected and main_loop_retrials > 0: main_loop_retrials = main_loop_retrials - 1 sleep(30) @@ -1885,7 +1886,7 @@ class Autosubmit: )] # noinspection PyTypeChecker platforms_to_test.add(job.platform) - Autosubmit.restore_platforms(platforms_to_test,mail_notify=True,as_conf=as_conf,expid=expid) + Autosubmit.restore_platforms(platforms_to_test,mail_notify=send_mail,as_conf=as_conf,expid=expid) reconnected = True except AutosubmitCritical: # Message prompt by restore_platforms. diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index f75c7b5cc..0615e5d97 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1187,8 +1187,7 @@ class Job(object): template_content = template_content.replace("%%", "%") script_name = '{0}.cmd'.format(self.name) self.script_name = '{0}.cmd'.format(self.name) - open(os.path.join(self._tmp_path, script_name), - 'w').write(template_content) + open(os.path.join(self._tmp_path, script_name),'w').write(template_content) os.chmod(os.path.join(self._tmp_path, script_name), 0o755) return script_name diff --git a/test/unit/test_job.py b/test/unit/test_job.py index d1d3e91b5..bbf369aa4 100644 --- a/test/unit/test_job.py +++ b/test/unit/test_job.py @@ -173,9 +173,6 @@ class TestJob(TestCase): with patch.object(builtins, "open", open_mock): # act self.job.create_script(config) - with patch.object(builtins, "write", write_mock): - # act - self.job.create_script(config) # assert update_content_mock.assert_called_with(config) open_mock.assert_called_with(os.path.join(self.job._tmp_path, self.job.name + '.cmd'), 'w') @@ -261,7 +258,7 @@ class TestJob(TestCase): as_conf.get_wallclock = Mock(return_value='00:30') as_conf.get_member_list = Mock(return_value=[]) as_conf.get_custom_directives = Mock(return_value='["whatever"]') - as_conf.load_project_parameters = Mock(return_value=[]) + as_conf.load_project_parameters = Mock(return_value=dict()) self.job.parameters['PROJECT_TYPE'] = "none" dummy_serial_platform = Mock() dummy_serial_platform.name = 'serial' diff --git a/test/unit/test_job_package.py b/test/unit/test_job_package.py index 2f1ffd52c..147ffec52 100644 --- a/test/unit/test_job_package.py +++ b/test/unit/test_job_package.py @@ -45,8 +45,11 @@ class TestJobPackage(TestCase): def test_job_package_submission(self): # arrange + write_mock = Mock().write = Mock() + self._tmp_path = Mock() for job in self.jobs: job._get_paramiko_template = Mock("false","empty") + self.job_package._create_scripts = Mock() self.job_package._send_files = Mock() self.job_package._do_submission = Mock() -- GitLab