diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 22245c23aa12b858e1537e613cb7f00aecfb7c01..bbc536e00aad7e421f4d306a9abd0108f77a9756 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -1190,7 +1190,9 @@ class JobList(object): edge_added = True if parent.section == job.section: self.actual_job_depends_on_special_chunk = True - if edge_added: + + # Only fill this if, per example, jobs.A.Dependencies.A or jobs.A.Dependencies.A-N is set. + if edge_added and job.section in dependencies_keys_without_special_chars: if job.name not in self.depends_on_previous_special_section: self.depends_on_previous_special_section[job.name] = set() if job.section not in self.depends_on_previous_special_section: @@ -1207,6 +1209,25 @@ class JobList(object): return special_dependencies, problematic_dependencies + def get_filters_to_apply(self, job, dependency): + filters_to_apply = self._filter_current_job(job, copy.deepcopy(dependency.relationships)) + filters_to_apply.pop("STATUS", None) + # Don't do perform special filter if only "FROM_STEP" is applied + if "FROM_STEP" in filters_to_apply: + if filters_to_apply.get("CHUNKS_TO","none") == "none" and filters_to_apply.get("MEMBERS_TO","none") == "none" and filters_to_apply.get("DATES_TO","none") == "none" and filters_to_apply.get("SPLITS_TO","none") == "none": + filters_to_apply = {} + filters_to_apply.pop("FROM_STEP", None) + + # If the selected filter is "natural" for all filters_to, trigger the natural dependency calculation + all_natural = True + for f_value in filters_to_apply.values(): + if f_value.lower() != "natural": + all_natural = False + break + if all_natural: + filters_to_apply = {} + return filters_to_apply + def _manage_job_dependencies(self, dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys, dependencies, graph): @@ -1341,9 +1362,8 @@ class JobList(object): dependency) if skip: continue - filters_to_apply = self._filter_current_job(job, copy.deepcopy(dependency.relationships)) - filters_to_apply.pop("STATUS", None) - filters_to_apply.pop("FROM_STEP", None) + filters_to_apply = self.get_filters_to_apply(job, dependency) + if len(filters_to_apply) > 0: dependencies_of_that_section = dic_jobs.as_conf.jobs_data[dependency.section].get("DEPENDENCIES", {}) ## Adds the dependencies to the job, and if not possible, adds the job to the problematic_dependencies diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 2bb91211140c68715fdf0084864c773dcb2dacb9..054728dcb536082fd37458726a39b7311313917b 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -8,6 +8,7 @@ import unittest from copy import deepcopy from datetime import datetime from mock import patch +from mock.mock import MagicMock from autosubmit.job.job_dict import DicJobs from autosubmit.job.job import Job @@ -436,6 +437,84 @@ class TestJobList(unittest.TestCase): job_list.add_special_conditions(job, special_conditions, filters_to_apply, parent2) self.assertEqual(len(job.edge_info.get("RUNNING", "")), 2) + def test_add_special_conditions_chunks_to_once(self): + # Method from job_list + job = Job("child", 1, Status.WAITING, 1) + job.section = "child_one" + job.date = datetime.strptime("20200128", "%Y%m%d") + job.member = "fc0" + job.chunk = 1 + job.split = 1 + job.splits = 1 + job.max_checkpoint_step = 0 + + job_two = Job("child", 1, Status.WAITING, 1) + job_two.section = "child_one" + job_two.date = datetime.strptime("20200128", "%Y%m%d") + job_two.member = "fc0" + job_two.chunk = 2 + job_two.split = 1 + job_two.splits = 1 + job_two.max_checkpoint_step = 0 + + special_conditions = {"STATUS": "RUNNING", "FROM_STEP": "1"} + special_conditions_two = {"STATUS": "RUNNING", "FROM_STEP": "2"} + + parent = Job("parent", 1, Status.RUNNING, 1) + parent.section = "parent_one" + parent.date = datetime.strptime("20200128", "%Y%m%d") + parent.member = None + parent.chunk = None + parent.split = None + parent.splits = None + parent.max_checkpoint_step = 0 + job.status = Status.WAITING + job_two.status = Status.WAITING + + job_list = Mock(wraps=self.JobList) + job_list._job_list = [job, job_two, parent] + + dependency = MagicMock() + dependency.relationships = {'CHUNKS_FROM': {'1': {'FROM_STEP': '1'}, '2': {'FROM_STEP': '2'}, }, 'STATUS': 'RUNNING'} + filters_to_apply = job_list.get_filters_to_apply(job, dependency) + filters_to_apply_two = job_list.get_filters_to_apply(job_two, dependency) + + assert filters_to_apply == {} + assert filters_to_apply_two == {} + + job_list.add_special_conditions(job, special_conditions, filters_to_apply, parent) + job_list.add_special_conditions(job_two, special_conditions_two, filters_to_apply_two, parent) + + dependency = MagicMock() + dependency.relationships = {'CHUNKS_FROM': {'1': {'FROM_STEP': '1', 'CHUNKS_TO':'natural'}, '2': {'FROM_STEP': '2', 'CHUNKS_TO':'natural'}, }, 'STATUS': 'RUNNING'} + filters_to_apply = job_list.get_filters_to_apply(job, dependency) + filters_to_apply_two = job_list.get_filters_to_apply(job_two, dependency) + + assert filters_to_apply == {} + assert filters_to_apply_two == {} + + job_list.add_special_conditions(job, special_conditions, filters_to_apply, parent) + job_list.add_special_conditions(job_two, special_conditions_two, filters_to_apply_two, parent) + + self.assertEqual(job.max_checkpoint_step, 1) + self.assertEqual(job_two.max_checkpoint_step, 2) + + value = job.edge_info.get("RUNNING", "").get("parent", ()) + self.assertEqual((value[0].name, value[1]), (parent.name, "1")) + self.assertEqual(len(job.edge_info.get("RUNNING", "")), 1) + + value_two = job_two.edge_info.get("RUNNING", "").get("parent", ()) + self.assertEqual((value_two[0].name, value_two[1]), (parent.name, "2")) + self.assertEqual(len(job_two.edge_info.get("RUNNING", "")), 1) + + dependency = MagicMock() + dependency.relationships = {'CHUNKS_FROM': {'1': {'FROM_STEP': '1', 'CHUNKS_TO':'natural', 'DATES_TO': "dummy"}, '2': {'FROM_STEP': '2', 'CHUNKS_TO':'natural', 'DATES_TO': "dummy"}, }, 'STATUS': 'RUNNING'} + filters_to_apply = job_list.get_filters_to_apply(job, dependency) + filters_to_apply_two = job_list.get_filters_to_apply(job_two, dependency) + + assert filters_to_apply == {'CHUNKS_TO': 'natural', 'DATES_TO': 'dummy'} + assert filters_to_apply_two == {'CHUNKS_TO': 'natural', 'DATES_TO': 'dummy'} + @patch('autosubmit.job.job_dict.date2str') def test_jobdict_get_jobs_filtered(self, mock_date2str): # Test the get_jobs_filtered function