From 90c8f0307898b83627f3fa0d0ab5fde9d9e776cc Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 27 Jul 2023 08:47:05 +0200 Subject: [PATCH 01/10] fixed test --- autosubmit/job/job_packager.py | 27 +++++++++++++-------------- autosubmit/job/job_packages.py | 21 ++++++++++++--------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/autosubmit/job/job_packager.py b/autosubmit/job/job_packager.py index ea3e0b9f1..57d20f275 100644 --- a/autosubmit/job/job_packager.py +++ b/autosubmit/job/job_packager.py @@ -17,17 +17,17 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . -from log.log import Log, AutosubmitCritical, AutosubmitError -from autosubmit.job.job_common import Status, Type +import operator from bscearth.utils.date import sum_str_hours -from autosubmit.job.job_packages import JobPackageSimple, JobPackageVertical, JobPackageHorizontal, \ - JobPackageSimpleWrapped, JobPackageHorizontalVertical, JobPackageVerticalHorizontal, JobPackageBase -from operator import attrgetter from math import ceil -import operator -from collections import defaultdict +from operator import attrgetter from typing import List +from autosubmit.job.job_common import Status, Type +from autosubmit.job.job_packages import JobPackageSimple, JobPackageVertical, JobPackageHorizontal, \ + JobPackageSimpleWrapped, JobPackageHorizontalVertical, JobPackageVerticalHorizontal, JobPackageBase +from log.log import Log, AutosubmitCritical + class JobPackager(object): """ @@ -437,10 +437,10 @@ class JobPackager(object): wrapper_limits["min_v"], wrapper_limits["min"], len(active_jobs)), 6013) else: - message = "Wrapper couldn't be formed under {0} POLICY due minimum limit not being reached: [wrappeable:{4} < defined_min:{5}] [wrappeable_h:{1} < defined_min_h:{2}]|[wrappeable_v:{3} < defined_min_v:{4}] ".format( + message = "Wrapper couldn't be formed under {0} POLICY due minimum limit not being reached: [wrappeable:{5} <= defined_min:{6}] [wrappeable_h:{1} <= defined_min_h:{2}]|[wrappeable_v:{3} <= defined_min_v:{4}] ".format( self.wrapper_policy[self.current_wrapper_section], min_h, - wrapper_limits["min_h"], min_v, wrapper_limits["min_v"], - wrapper_limits["min"], len(active_jobs)) + wrapper_limits["min_h"], min_v, wrapper_limits["min_v"], len(p.jobs), + wrapper_limits["min"]) if hard_deadlock: message += "\nCheck your configuration: The next wrappeable job can't be wrapped until some of inner jobs of current packages finishes which is imposible" if min_v > 1: @@ -481,13 +481,12 @@ class JobPackager(object): if len(active_jobs) > 0: if show_log: Log.printlog( - "Wrapper policy is set to MIXED and there are not enough jobs to form a wrapper.[wrappeable:{4} < defined_min:{5}] [wrappeable_h:{0} < defined_min_h:{1}]|[wrappeable_v:{2} < defined_min_v:{3}] waiting until the wrapper can be formed.".format( + "Wrapper policy is set to MIXED and there are not enough jobs to form a wrapper.[wrappeable:{4} <= defined_min:{5}] [wrappeable_h:{0} <= defined_min_h:{1}]|[wrappeable_v:{2} <= defined_min_v:{3}] waiting until the wrapper can be formed.".format( min_h, wrapper_limits["min_h"], min_v, wrapper_limits["min_v"],wrapper_limits["min"],len(active_jobs)), 6013) else: - message = "Wrapper couldn't be formed under {0} POLICY due minimum limit not being reached: [wrappeable:{4} < defined_min:{5}] [wrappeable_h:{1} < defined_min_h:{2}]|[wrappeable_v:{3} < defined_min_v:{4}] ".format( - self.wrapper_policy[self.current_wrapper_section], min_h, - wrapper_limits["min_h"], min_v, wrapper_limits["min_v"],wrapper_limits["min"],len(active_jobs)) + message = "Wrapper couldn't be formed under {0} POLICY due minimum limit not being reached: [wrappeable:{5} <= defined_min:{6}] [wrappeable_h:{1} <= defined_min_h:{2}]|[wrappeable_v:{3} <= defined_min_v:{4}] ".format( + self.wrapper_policy[self.current_wrapper_section], min_h,wrapper_limits["min_h"],min_v, wrapper_limits["min_v"], len(p.jobs),wrapper_limits["min"]) if hard_deadlock: message += "\nCheck your configuration: The next wrappeable job can't be wrapped until some of inner jobs of current packages finishes which is imposible" if min_v > 1: diff --git a/autosubmit/job/job_packages.py b/autosubmit/job/job_packages.py index 8e3c3409a..6174ffc69 100644 --- a/autosubmit/job/job_packages.py +++ b/autosubmit/job/job_packages.py @@ -113,9 +113,9 @@ class JobPackageBase(object): Log.warning("On submission script has some empty variables") else: Log.result("Script {0} OK", job.name) - lock.acquire() - job.update_parameters(configuration, parameters) - lock.release() + # lock.acquire() + # job.update_parameters(configuration, parameters) + # lock.release() # looking for directives on jobs self._custom_directives = self._custom_directives | set(job.custom_directives) @threaded @@ -149,13 +149,16 @@ class JobPackageBase(object): try: # get one job of each section jobs by section if only_generate: - sections = configuration.get_wrapper_jobs(self.current_wrapper_section) - if "&" in sections: - sections.split("&") - elif " " in sections: - sections.split(" ") + if hasattr(configuration, 'current_wrapper_section'): + sections = configuration.get_wrapper_jobs(self.current_wrapper_section) + if "&" in sections: + sections.split("&") + elif " " in sections: + sections.split(" ") + else: + sections = [sections] else: - sections = [sections] + sections = [self.jobs[0].section] for section in sections: if str(configuration._jobs_parser.get_option(section, "CHECK", 'True')).lower() == str(Job.CHECK_ON_SUBMISSION).lower(): exit = True -- GitLab From f5d69ef0e8617653995fea25f5763cdc9a68a48f Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 27 Jul 2023 09:03:28 +0200 Subject: [PATCH 02/10] not sure why fails only in remote pipeline --- test/unit/test_job_package.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/unit/test_job_package.py b/test/unit/test_job_package.py index a929d6c43..ca5455f2b 100644 --- a/test/unit/test_job_package.py +++ b/test/unit/test_job_package.py @@ -49,13 +49,11 @@ class TestJobPackage(TestCase): for job in self.job_package.jobs: job._tmp_path = Mock() job.name = "fake-name" - job._get_paramiko_template = Mock("false","empty") + job._get_paramiko_template = Mock("false", "empty") job.file = "fake-file" job.update_parameters = MagicMock(return_value="fake-params") job.parameters = "fake-params" - - self.job_package._create_scripts = Mock() self.job_package._send_files = Mock() self.job_package._do_submission = Mock() @@ -63,7 +61,6 @@ class TestJobPackage(TestCase): configuration.get_project_type = Mock(return_value='fake-type') configuration.get_project_dir = Mock(return_value='fake-dir') configuration.get_project_name = Mock(return_value='fake-name') - # act self.job_package.submit(configuration, 'fake-params') # assert -- GitLab From 7d35a596274cd6da0b5f5885d08f448b41d67af6 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 27 Jul 2023 09:04:38 +0200 Subject: [PATCH 03/10] not sure why fails only in remote pipeline --- test/unit/test_job_package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_job_package.py b/test/unit/test_job_package.py index ca5455f2b..5f1185d37 100644 --- a/test/unit/test_job_package.py +++ b/test/unit/test_job_package.py @@ -64,7 +64,7 @@ class TestJobPackage(TestCase): # act self.job_package.submit(configuration, 'fake-params') # assert - for job in self.jobs: + for job in self.job_package.jobs: job.update_parameters.assert_called_once_with(configuration, 'fake-params') self.job_package._create_scripts.is_called_once_with() self.job_package._send_files.is_called_once_with() -- GitLab From 1fccd5321230cac3feab539361b589b903c7d36c Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 27 Jul 2023 09:11:24 +0200 Subject: [PATCH 04/10] not sure why fails only in remote pipeline --- test/unit/test_job_package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_job_package.py b/test/unit/test_job_package.py index 5f1185d37..648218ef2 100644 --- a/test/unit/test_job_package.py +++ b/test/unit/test_job_package.py @@ -52,7 +52,7 @@ class TestJobPackage(TestCase): job._get_paramiko_template = Mock("false", "empty") job.file = "fake-file" job.update_parameters = MagicMock(return_value="fake-params") - job.parameters = "fake-params" + job.parameters = ["fake-params"] self.job_package._create_scripts = Mock() self.job_package._send_files = Mock() -- GitLab From 26b4aa088b3bf6c4794d33485cbb0e598c389a5a Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 27 Jul 2023 09:17:50 +0200 Subject: [PATCH 05/10] not sure why fails only in remote pipeline --- test/unit/test_job_package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_job_package.py b/test/unit/test_job_package.py index 648218ef2..5c37749c3 100644 --- a/test/unit/test_job_package.py +++ b/test/unit/test_job_package.py @@ -52,7 +52,7 @@ class TestJobPackage(TestCase): job._get_paramiko_template = Mock("false", "empty") job.file = "fake-file" job.update_parameters = MagicMock(return_value="fake-params") - job.parameters = ["fake-params"] + job.parameters = {"fake-params":"fake-value"} self.job_package._create_scripts = Mock() self.job_package._send_files = Mock() -- GitLab From 53eb8df390fb64f8de0bf69b25a9ad3fdf24e76e Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 27 Jul 2023 09:23:59 +0200 Subject: [PATCH 06/10] fixes https://earth.bsc.es/gitlab/es/autosubmit/-/issues/1073 --- autosubmit/job/job_packages.py | 8 ++++++-- test/unit/test_job_package.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/autosubmit/job/job_packages.py b/autosubmit/job/job_packages.py index 6174ffc69..c6c8dbc6d 100644 --- a/autosubmit/job/job_packages.py +++ b/autosubmit/job/job_packages.py @@ -186,8 +186,12 @@ class JobPackageBase(object): except BaseException as e: original = e if not exit: - raise AutosubmitCritical( - "Error on {1}, template [{0}] still does not exists in running time(check=on_submission actived)\n{2} ".format(self.jobs[0].file,self.jobs[0].name,e), 7014) + if not os.path.exists(os.path.join(configuration.get_project_dir(), self.jobs[0].file)) and str(configuration.get_project_type()).lower() != "none": + raise AutosubmitCritical( + "Error on {1}, template [{0}] still does not exists in running time(check=on_submission actived)\n{2} ".format(self.jobs[0].file,self.jobs[0].name,e), 7014) + else: + raise AutosubmitCritical("Error in Custom_directives of {0} \n{1}, Check if it format is correct and there are no extra commas".format(self.jobs[0].section,e),7014) + else: raise AutosubmitCritical(original,7014) Log.debug("Creating Scripts") diff --git a/test/unit/test_job_package.py b/test/unit/test_job_package.py index 5c37749c3..dcc781581 100644 --- a/test/unit/test_job_package.py +++ b/test/unit/test_job_package.py @@ -47,7 +47,7 @@ class TestJobPackage(TestCase): write_mock = Mock().write = Mock() os_mock.return_value = True for job in self.job_package.jobs: - job._tmp_path = Mock() + job._tmp_path = "fake-path" job.name = "fake-name" job._get_paramiko_template = Mock("false", "empty") job.file = "fake-file" -- GitLab From 50739d789524909bf499e459bd429dd21f4341bb Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 27 Jul 2023 09:36:02 +0200 Subject: [PATCH 07/10] pipeline fails in remote only --- test/unit/test_job_package.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/unit/test_job_package.py b/test/unit/test_job_package.py index dcc781581..8c5983f1b 100644 --- a/test/unit/test_job_package.py +++ b/test/unit/test_job_package.py @@ -42,8 +42,10 @@ class TestJobPackage(TestCase): self.assertEquals(self.platform, self.job_package.platform) @patch('os.path.exists') - def test_job_package_submission(self, os_mock): + @patch('__builtin__.open') + def test_job_package_submission(self, os_mock, builtins_mock): # arrange + builtins_mock.return_value = "fake-content" write_mock = Mock().write = Mock() os_mock.return_value = True for job in self.job_package.jobs: @@ -52,7 +54,7 @@ class TestJobPackage(TestCase): job._get_paramiko_template = Mock("false", "empty") job.file = "fake-file" job.update_parameters = MagicMock(return_value="fake-params") - job.parameters = {"fake-params":"fake-value"} + job.parameters = {"fake-params": "fake-value"} self.job_package._create_scripts = Mock() self.job_package._send_files = Mock() -- GitLab From e6e15105af295b6ed0c8cc4409cd77217e09ec40 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 27 Jul 2023 09:52:56 +0200 Subject: [PATCH 08/10] fixed --- autosubmit/job/job_packages.py | 26 +++++++++++++------------- test/unit/test_job_package.py | 5 ++--- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/autosubmit/job/job_packages.py b/autosubmit/job/job_packages.py index c6c8dbc6d..dc7985a89 100644 --- a/autosubmit/job/job_packages.py +++ b/autosubmit/job/job_packages.py @@ -37,7 +37,6 @@ from autosubmit.job.job import Job from bscearth.utils.date import sum_str_hours from threading import Thread, Lock from typing import List -import multiprocessing import tarfile import datetime import re @@ -101,18 +100,18 @@ class JobPackageBase(object): if str(job.check).lower() == str(Job.CHECK_ON_SUBMISSION).lower(): if only_generate: break - if not os.path.exists(os.path.join(configuration.get_project_dir(), job.file)): - lock.acquire() - if str(configuration.get_project_type()).lower() != "none": - raise AutosubmitCritical( - "Template [ {0} ] using CHECK=On_submission has some empty variable {0}".format( - job.name), 7014) - lock.release() - if not job.check_script(configuration, parameters, show_logs=job.check_warnings): - Log.warning("Script {0} check failed", job.name) - Log.warning("On submission script has some empty variables") - else: - Log.result("Script {0} OK", job.name) + if not os.path.exists(os.path.join(configuration.get_project_dir(), job.file)): + lock.acquire() + if str(configuration.get_project_type()).lower() != "none": + raise AutosubmitCritical( + "Template [ {0} ] using CHECK=On_submission has some empty variable {0}".format( + job.name), 7014) + lock.release() + if not job.check_script(configuration, parameters, show_logs=job.check_warnings): + Log.warning("Script {0} check failed", job.name) + Log.warning("On submission script has some empty variables") + else: + Log.result("Script {0} OK", job.name) # lock.acquire() # job.update_parameters(configuration, parameters) # lock.release() @@ -137,6 +136,7 @@ class JobPackageBase(object): """ exit=False thread_number = multiprocessing.cpu_count() + #thread_number = 1 if len(self.jobs) > 2500: thread_number = thread_number * 2 elif len(self.jobs) > 5000: diff --git a/test/unit/test_job_package.py b/test/unit/test_job_package.py index 8c5983f1b..fa89f2e55 100644 --- a/test/unit/test_job_package.py +++ b/test/unit/test_job_package.py @@ -43,10 +43,9 @@ class TestJobPackage(TestCase): @patch('os.path.exists') @patch('__builtin__.open') - def test_job_package_submission(self, os_mock, builtins_mock): + def test_job_package_submission(self, os_mock, open_mock): # arrange - builtins_mock.return_value = "fake-content" - write_mock = Mock().write = Mock() + open_mock.return_value = MagicMock() os_mock.return_value = True for job in self.job_package.jobs: job._tmp_path = "fake-path" -- GitLab From bddd6a5656e38b654e640a96e56c2f38ec470f5c Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 27 Jul 2023 09:54:01 +0200 Subject: [PATCH 09/10] fixed --- autosubmit/job/job_packages.py | 1 + 1 file changed, 1 insertion(+) diff --git a/autosubmit/job/job_packages.py b/autosubmit/job/job_packages.py index dc7985a89..b98c1fc72 100644 --- a/autosubmit/job/job_packages.py +++ b/autosubmit/job/job_packages.py @@ -24,6 +24,7 @@ except ImportError: # noinspection PyCompatibility from ConfigParser import SafeConfigParser +import multiprocessing import os import random import time -- GitLab From 606e9a6cf00a8f2768547853e7325d2559a3e0d7 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 17 Jul 2024 17:43:46 +0200 Subject: [PATCH 10/10] Fix after conflict --- autosubmit/job/job_packages.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/autosubmit/job/job_packages.py b/autosubmit/job/job_packages.py index abfd63930..a1464f13e 100644 --- a/autosubmit/job/job_packages.py +++ b/autosubmit/job/job_packages.py @@ -113,9 +113,9 @@ class JobPackageBase(object): Log.warning("On submission script has some empty variables") else: Log.result("Script {0} OK", job.name) - # lock.acquire() - # job.update_parameters(configuration, parameters) - # lock.release() + lock.acquire() + job.update_parameters(configuration, parameters) + lock.release() # looking for directives on jobs self._custom_directives = self._custom_directives | set(job.custom_directives) @threaded @@ -150,7 +150,7 @@ class JobPackageBase(object): try: # get one job of each section jobs by section if only_generate: - if hasattr(configuration, 'current_wrapper_section'): + if hasattr(self, 'current_wrapper_section'): sections = configuration.get_wrapper_jobs(self.current_wrapper_section) if "&" in sections: sections.split("&") -- GitLab