From 54dfb95b55ae4c871cfc5abc6935b5d4b3e953f2 Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Wed, 8 Jul 2020 12:00:49 +0200 Subject: [PATCH 1/5] Fixed #545 Added a new flag that will force the structure to be updated on every autosubmit create. --- autosubmit/autosubmit.py | 18 ++++++++++-------- autosubmit/job/job_list.py | 14 ++++++++------ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 8711afcce..2c847a389 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -1131,7 +1131,7 @@ class Autosubmit: :rtype: \n """ job_list._job_list = jobs_filtered - job_list.update_list(as_conf,False) + job_list.update_list(as_conf, False) # Current choice is Paramiko Submitter submitter = Autosubmit._get_submitter(as_conf) @@ -1156,11 +1156,10 @@ class Autosubmit: # Loading parameters again Autosubmit._load_parameters(as_conf, job_list, submitter.platforms) while job_list.get_active(): - Autosubmit.submit_ready_jobs(as_conf, job_list, platforms_to_test, packages_persistence, True, only_wrappers, hold=False) + Autosubmit.submit_ready_jobs( + as_conf, job_list, platforms_to_test, packages_persistence, True, only_wrappers, hold=False) job_list.update_list(as_conf, False) - - @staticmethod def run_experiment(expid, notransitive=False, update_version=False): """ @@ -1484,7 +1483,8 @@ class Autosubmit: Log.debug("\nJobs prepared for {1}: {0}", len( job_list.get_prepared(platform)), platform.name) - packages_to_submit = JobPackager(as_conf, platform, job_list, hold=hold).build_packages() + packages_to_submit = JobPackager( + as_conf, platform, job_list, hold=hold).build_packages() if not inspect: platform.open_submit_script() @@ -1501,7 +1501,8 @@ class Autosubmit: package._wallclock, package._num_processors, package.platform, as_conf, hold) job_list.job_package_map[package.jobs[0].id] = wrapper_job - packages_persistence.save(package.name, package.jobs, package._expid, inspect) + packages_persistence.save( + package.name, package.jobs, package._expid, inspect) for innerJob in package._jobs: # Setting status to COMPLETED so it does not get stuck in the loop that calls this function innerJob.status = Status.COMPLETED @@ -3070,7 +3071,7 @@ class Autosubmit: job_list.generate(date_list, member_list, num_chunks, chunk_ini, parameters, date_format, as_conf.get_retrials(), as_conf.get_default_job_type(), - as_conf.get_wrapper_type(), as_conf.get_wrapper_jobs(), notransitive=notransitive) + as_conf.get_wrapper_type(), as_conf.get_wrapper_jobs(), notransitive=notransitive, update_structure=True) if rerun == "true": chunk_list = Autosubmit._create_json( @@ -3115,7 +3116,8 @@ class Autosubmit: for job in jobs_wr: job.children = job.children - referenced_jobs_to_remove job.parents = job.parents - referenced_jobs_to_remove - Autosubmit.generate_scripts_andor_wrappers(as_conf, job_list_wrappers, jobs_wr,packages_persistence, True) + Autosubmit.generate_scripts_andor_wrappers( + as_conf, job_list_wrappers, jobs_wr, packages_persistence, True) packages = packages_persistence.load(True) else: diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 962ac9daa..a03ced461 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -99,7 +99,7 @@ class JobList: self._graph = value def generate(self, date_list, member_list, num_chunks, chunk_ini, parameters, date_format, default_retrials, - default_job_type, wrapper_type=None, wrapper_jobs=None, new=True, notransitive=False): + default_job_type, wrapper_type=None, wrapper_jobs=None, new=True, notransitive=False, update_structure=False): """ Creates all jobs needed for the current workflow @@ -152,7 +152,8 @@ class JobList: chunk_list, dic_jobs, jobs_parser, self.graph) Log.info("Removing redundant dependencies...") - self.update_genealogy(new, notransitive) + self.update_genealogy( + new, notransitive, update_structure=update_structure) for job in self._job_list: job.parameters = parameters @@ -1218,7 +1219,7 @@ class JobList: return save - def update_genealogy(self, new=True, notransitive=False): + def update_genealogy(self, new=True, notransitive=False, update_structure=False): """ When we have created the job list, every type of job is created. Update genealogy remove jobs that have no templates @@ -1236,14 +1237,14 @@ class JobList: if not notransitive: # Transitive reduction required current_structure = None - if os.path.exists(os.path.join(self._config.STRUCTURES_DIR,"structure_"+self.expid+".db")): + if os.path.exists(os.path.join(self._config.STRUCTURES_DIR, "structure_"+self.expid+".db")): try: current_structure = DbStructure.get_structure( self.expid, self._config.STRUCTURES_DIR) except Exception as exp: pass structure_valid = False - if ((current_structure) and (len(self._job_list) == len(current_structure.keys()))): + if ((current_structure) and (len(self._job_list) == len(current_structure.keys())) and update_structure == False): structure_valid = True # print(current_structure.keys()) # Structure exists and is valid, use it as a source of dependencies @@ -1273,9 +1274,10 @@ class JobList: job.children.remove(child) child.parents.remove(job) try: - DbStructure.save_strucexiture( + DbStructure.save_structure( self.graph, self.expid, self._config.STRUCTURES_DIR) except Exception as exp: + Log.warning(exp) pass for job in self._job_list: -- GitLab From 4cae2af46ab4175d00e9eb6b25338e4901a08de4 Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Wed, 8 Jul 2020 12:07:10 +0200 Subject: [PATCH 2/5] Fixed #545 Added a necessary test modification. --- test/unit/test_job_list.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/test/unit/test_job_list.py b/test/unit/test_job_list.py index 2b87cc650..4f5a330d2 100644 --- a/test/unit/test_job_list.py +++ b/test/unit/test_job_list.py @@ -176,7 +176,8 @@ class TestJobList(TestCase): sorted_by_name = self.job_list.sort_by_name() for i in xrange(len(sorted_by_name) - 1): - self.assertTrue(sorted_by_name[i].name <= sorted_by_name[i + 1].name) + self.assertTrue( + sorted_by_name[i].name <= sorted_by_name[i + 1].name) def test_sort_by_id_returns_the_list_of_jobs_well_sorted(self): sorted_by_id = self.job_list.sort_by_id() @@ -188,13 +189,15 @@ class TestJobList(TestCase): sorted_by_type = self.job_list.sort_by_type() for i in xrange(len(sorted_by_type) - 1): - self.assertTrue(sorted_by_type[i].type <= sorted_by_type[i + 1].type) + self.assertTrue( + sorted_by_type[i].type <= sorted_by_type[i + 1].type) def test_sort_by_status_returns_the_list_of_jobs_well_sorted(self): sorted_by_status = self.job_list.sort_by_status() for i in xrange(len(sorted_by_status) - 1): - self.assertTrue(sorted_by_status[i].status <= sorted_by_status[i + 1].status) + self.assertTrue( + sorted_by_status[i].status <= sorted_by_status[i + 1].status) def test_that_create_method_makes_the_correct_calls(self): parser_mock = Mock() @@ -203,7 +206,8 @@ class TestJobList(TestCase): factory = ConfigParserFactory() factory.create_parser = Mock(return_value=parser_mock) - job_list = JobList(self.experiment_id, FakeBasicConfig, factory, JobListPersistenceDb('.', '.')) + job_list = JobList(self.experiment_id, FakeBasicConfig, + factory, JobListPersistenceDb('.', '.')) job_list._create_jobs = Mock() job_list._add_dependencies = Mock() job_list.update_genealogy = Mock() @@ -218,7 +222,8 @@ class TestJobList(TestCase): graph_mock = Mock() job_list.graph = graph_mock # act - job_list.generate(date_list, member_list, num_chunks, 1, parameters, 'H', 9999, Type.BASH, 'None') + job_list.generate(date_list, member_list, num_chunks, + 1, parameters, 'H', 9999, Type.BASH, 'None') # assert self.assertEquals(job_list.parameters, parameters) @@ -232,7 +237,7 @@ class TestJobList(TestCase): self.assertEquals(0, cj_args[2]) job_list._add_dependencies.assert_called_once_with(date_list, member_list, chunk_list, cj_args[0], parser_mock, graph_mock) - job_list.update_genealogy.assert_called_once_with(True, False) + job_list.update_genealogy.assert_called_once_with(True, False, False) for job in job_list._job_list: self.assertEquals(parameters, job.parameters) @@ -248,8 +253,10 @@ class TestJobList(TestCase): JobList._create_jobs(dic_mock, parser_mock, 0, Type.BASH) # arrange - dic_mock.read_section.assert_any_call('fake-section-1', 0, Type.BASH, dict()) - dic_mock.read_section.assert_any_call('fake-section-2', 1, Type.BASH, dict()) + dic_mock.read_section.assert_any_call( + 'fake-section-1', 0, Type.BASH, dict()) + dic_mock.read_section.assert_any_call( + 'fake-section-2', 1, Type.BASH, dict()) def _createDummyJobWithStatus(self, status): job_name = str(randrange(999999, 999999999)) -- GitLab From fd0c6b8997365f65701bb80cdadcd56986ba1727 Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Wed, 8 Jul 2020 12:17:52 +0200 Subject: [PATCH 3/5] Pipeline modification --- test/unit/test_job_list.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/test_job_list.py b/test/unit/test_job_list.py index 4f5a330d2..4e4aff5bb 100644 --- a/test/unit/test_job_list.py +++ b/test/unit/test_job_list.py @@ -237,6 +237,7 @@ class TestJobList(TestCase): self.assertEquals(0, cj_args[2]) job_list._add_dependencies.assert_called_once_with(date_list, member_list, chunk_list, cj_args[0], parser_mock, graph_mock) + # Adding flag update structure job_list.update_genealogy.assert_called_once_with(True, False, False) for job in job_list._job_list: self.assertEquals(parameters, job.parameters) -- GitLab From 0b0a20d9f2ff35257c4793b19f016291cb8b9aee Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Wed, 8 Jul 2020 12:23:02 +0200 Subject: [PATCH 4/5] Pipeline modification 2 --- test/unit/test_job_list.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/test_job_list.py b/test/unit/test_job_list.py index 4e4aff5bb..ea4b11124 100644 --- a/test/unit/test_job_list.py +++ b/test/unit/test_job_list.py @@ -223,7 +223,7 @@ class TestJobList(TestCase): job_list.graph = graph_mock # act job_list.generate(date_list, member_list, num_chunks, - 1, parameters, 'H', 9999, Type.BASH, 'None') + 1, parameters, 'H', 9999, Type.BASH, 'None', update_structure=True) # assert self.assertEquals(job_list.parameters, parameters) @@ -238,7 +238,7 @@ class TestJobList(TestCase): job_list._add_dependencies.assert_called_once_with(date_list, member_list, chunk_list, cj_args[0], parser_mock, graph_mock) # Adding flag update structure - job_list.update_genealogy.assert_called_once_with(True, False, False) + job_list.update_genealogy.assert_called_once_with(True, False, True) for job in job_list._job_list: self.assertEquals(parameters, job.parameters) -- GitLab From 8e8a9f66784eedddc8e2ec0cf9b7da1b673e4732 Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Wed, 8 Jul 2020 12:26:24 +0200 Subject: [PATCH 5/5] Pipeline modification 3 --- test/unit/test_job_list.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/unit/test_job_list.py b/test/unit/test_job_list.py index ea4b11124..65d78f28d 100644 --- a/test/unit/test_job_list.py +++ b/test/unit/test_job_list.py @@ -238,7 +238,8 @@ class TestJobList(TestCase): job_list._add_dependencies.assert_called_once_with(date_list, member_list, chunk_list, cj_args[0], parser_mock, graph_mock) # Adding flag update structure - job_list.update_genealogy.assert_called_once_with(True, False, True) + job_list.update_genealogy.assert_called_once_with( + True, False, update_structure=True) for job in job_list._job_list: self.assertEquals(parameters, job.parameters) -- GitLab