From b3863f2734ed50144b268e9faf3f542cc25176c6 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 2 Oct 2024 15:52:14 +0200 Subject: [PATCH 1/5] From_step acting as a filter --- autosubmit/job/job_list.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 22245c23a..0c6938a4d 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -1343,7 +1343,14 @@ class JobList(object): continue filters_to_apply = self._filter_current_job(job, copy.deepcopy(dependency.relationships)) filters_to_apply.pop("STATUS", None) + # Don't do perform special filter if only "FROM_STEP" is applied + if "FROM_STEP" in filters_to_apply: + if filters_to_apply["CHUNKS_TO"] == "none" and filters_to_apply["MEMBERS_TO"] == "none" and \ + filters_to_apply["DATES_TO"] == "none": + filters_to_apply = {} + filters_to_apply.pop("FROM_STEP", None) + if len(filters_to_apply) > 0: dependencies_of_that_section = dic_jobs.as_conf.jobs_data[dependency.section].get("DEPENDENCIES", {}) ## Adds the dependencies to the job, and if not possible, adds the job to the problematic_dependencies -- GitLab From 8dc3a3758ec766f5a0e82bd46c0e704771295620 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 2 Oct 2024 16:45:23 +0200 Subject: [PATCH 2/5] Added test ( not pytest ) --- autosubmit/job/job_list.py | 20 ++++++----- test/unit/test_dependencies.py | 61 ++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 9 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 0c6938a4d..1295db285 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -1207,6 +1207,16 @@ class JobList(object): return special_dependencies, problematic_dependencies + def get_filters_to_apply(self, job, dependency): + filters_to_apply = self._filter_current_job(job, copy.deepcopy(dependency.relationships)) + filters_to_apply.pop("STATUS", None) + # Don't do perform special filter if only "FROM_STEP" is applied + if "FROM_STEP" in filters_to_apply: + if filters_to_apply.get("CHUNKS_TO","none") == "none" and filters_to_apply.get("MEMBERS_TO","none") == "none" and filters_to_apply.get("DATES_TO","none") == "none" and filters_to_apply.get("SPLITS_TO","none") == "none": + filters_to_apply = {} + filters_to_apply.pop("FROM_STEP", None) + return filters_to_apply + def _manage_job_dependencies(self, dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys, dependencies, graph): @@ -1341,15 +1351,7 @@ class JobList(object): dependency) if skip: continue - filters_to_apply = self._filter_current_job(job, copy.deepcopy(dependency.relationships)) - filters_to_apply.pop("STATUS", None) - # Don't do perform special filter if only "FROM_STEP" is applied - if "FROM_STEP" in filters_to_apply: - if filters_to_apply["CHUNKS_TO"] == "none" and filters_to_apply["MEMBERS_TO"] == "none" and \ - filters_to_apply["DATES_TO"] == "none": - filters_to_apply = {} - - filters_to_apply.pop("FROM_STEP", None) + filters_to_apply = self.get_filters_to_apply(job, dependency) if len(filters_to_apply) > 0: dependencies_of_that_section = dic_jobs.as_conf.jobs_data[dependency.section].get("DEPENDENCIES", {}) diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 2bb912111..5034aa4a0 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -8,6 +8,7 @@ import unittest from copy import deepcopy from datetime import datetime from mock import patch +from mock.mock import MagicMock from autosubmit.job.job_dict import DicJobs from autosubmit.job.job import Job @@ -436,6 +437,66 @@ class TestJobList(unittest.TestCase): job_list.add_special_conditions(job, special_conditions, filters_to_apply, parent2) self.assertEqual(len(job.edge_info.get("RUNNING", "")), 2) + def test_add_special_conditions_chunks_to_once(self): + # Method from job_list + job = Job("child", 1, Status.WAITING, 1) + job.section = "child_one" + job.date = datetime.strptime("20200128", "%Y%m%d") + job.member = "fc0" + job.chunk = 1 + job.split = 1 + job.splits = 1 + job.max_checkpoint_step = 0 + + job_two = Job("child", 1, Status.WAITING, 1) + job_two.section = "child_one" + job_two.date = datetime.strptime("20200128", "%Y%m%d") + job_two.member = "fc0" + job_two.chunk = 2 + job_two.split = 1 + job_two.splits = 1 + job_two.max_checkpoint_step = 0 + + special_conditions = {"STATUS": "RUNNING", "FROM_STEP": "1"} + special_conditions_two = {"STATUS": "RUNNING", "FROM_STEP": "2"} + + parent = Job("parent", 1, Status.RUNNING, 1) + parent.section = "parent_one" + parent.date = datetime.strptime("20200128", "%Y%m%d") + parent.member = None + parent.chunk = None + parent.split = None + parent.splits = None + parent.max_checkpoint_step = 0 + job.status = Status.WAITING + job_two.status = Status.WAITING + + job_list = Mock(wraps=self.JobList) + job_list._job_list = [job, job_two, parent] + + dependency = MagicMock() + dependency.relationships = {'CHUNKS_FROM': {'1': {'FROM_STEP': '1'}, '2': {'FROM_STEP': '2'}, '3': {'FROM_STEP': '3'}, '4': {'FROM_STEP': '4'}}, 'STATUS': 'RUNNING'} + filters_to_apply = job_list.get_filters_to_apply(job, dependency) + filters_to_apply_two = job_list.get_filters_to_apply(job_two, dependency) + + assert filters_to_apply == {} + assert filters_to_apply_two == {} + + job_list.add_special_conditions(job, special_conditions, filters_to_apply, parent) + job_list.add_special_conditions(job_two, special_conditions_two, filters_to_apply_two, parent) + + self.assertEqual(job.max_checkpoint_step, 1) + self.assertEqual(job_two.max_checkpoint_step, 2) + + value = job.edge_info.get("RUNNING", "").get("parent", ()) + self.assertEqual((value[0].name, value[1]), (parent.name, "1")) + self.assertEqual(len(job.edge_info.get("RUNNING", "")), 1) + + value_two = job_two.edge_info.get("RUNNING", "").get("parent", ()) + self.assertEqual((value_two[0].name, value_two[1]), (parent.name, "2")) + self.assertEqual(len(job_two.edge_info.get("RUNNING", "")), 1) + + @patch('autosubmit.job.job_dict.date2str') def test_jobdict_get_jobs_filtered(self, mock_date2str): # Test the get_jobs_filtered function -- GitLab From 03c0ce226956737667579f9125bc1557d6a07dc6 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 2 Oct 2024 16:47:10 +0200 Subject: [PATCH 3/5] Added test ( not pytest ) --- test/unit/test_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 5034aa4a0..f729bb284 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -475,7 +475,7 @@ class TestJobList(unittest.TestCase): job_list._job_list = [job, job_two, parent] dependency = MagicMock() - dependency.relationships = {'CHUNKS_FROM': {'1': {'FROM_STEP': '1'}, '2': {'FROM_STEP': '2'}, '3': {'FROM_STEP': '3'}, '4': {'FROM_STEP': '4'}}, 'STATUS': 'RUNNING'} + dependency.relationships = {'CHUNKS_FROM': {'1': {'FROM_STEP': '1'}, '2': {'FROM_STEP': '2'}, }, 'STATUS': 'RUNNING'} filters_to_apply = job_list.get_filters_to_apply(job, dependency) filters_to_apply_two = job_list.get_filters_to_apply(job_two, dependency) -- GitLab From 3d0da60f480a6253b39a27f2861de73b1c503e1e Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 3 Oct 2024 11:46:08 +0200 Subject: [PATCH 4/5] Additional fixes --- autosubmit/job/job_list.py | 13 ++++++++++++- test/unit/test_dependencies.py | 11 +++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 1295db285..bbc536e00 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -1190,7 +1190,9 @@ class JobList(object): edge_added = True if parent.section == job.section: self.actual_job_depends_on_special_chunk = True - if edge_added: + + # Only fill this if, per example, jobs.A.Dependencies.A or jobs.A.Dependencies.A-N is set. + if edge_added and job.section in dependencies_keys_without_special_chars: if job.name not in self.depends_on_previous_special_section: self.depends_on_previous_special_section[job.name] = set() if job.section not in self.depends_on_previous_special_section: @@ -1215,6 +1217,15 @@ class JobList(object): if filters_to_apply.get("CHUNKS_TO","none") == "none" and filters_to_apply.get("MEMBERS_TO","none") == "none" and filters_to_apply.get("DATES_TO","none") == "none" and filters_to_apply.get("SPLITS_TO","none") == "none": filters_to_apply = {} filters_to_apply.pop("FROM_STEP", None) + + # If the selected filter is "natural" for all filters_to, trigger the natural dependency calculation + all_natural = True + for f_value in filters_to_apply.values(): + if f_value.lower() != "natural": + all_natural = False + break + if all_natural: + filters_to_apply = {} return filters_to_apply def _manage_job_dependencies(self, dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys, diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index f729bb284..2c9143d2f 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -485,6 +485,17 @@ class TestJobList(unittest.TestCase): job_list.add_special_conditions(job, special_conditions, filters_to_apply, parent) job_list.add_special_conditions(job_two, special_conditions_two, filters_to_apply_two, parent) + dependency = MagicMock() + dependency.relationships = {'CHUNKS_FROM': {'1': {'FROM_STEP': '1', 'CHUNKS_TO':'natural'}, '2': {'FROM_STEP': '2', 'CHUNKS_TO':'natural'}, }, 'STATUS': 'RUNNING'} + filters_to_apply = job_list.get_filters_to_apply(job, dependency) + filters_to_apply_two = job_list.get_filters_to_apply(job_two, dependency) + + assert filters_to_apply == {} + assert filters_to_apply_two == {} + + job_list.add_special_conditions(job, special_conditions, filters_to_apply, parent) + job_list.add_special_conditions(job_two, special_conditions_two, filters_to_apply_two, parent) + self.assertEqual(job.max_checkpoint_step, 1) self.assertEqual(job_two.max_checkpoint_step, 2) -- GitLab From d6f0d798eded1bb7d7beb6fb6ece4480d73ba918 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 3 Oct 2024 12:00:09 +0200 Subject: [PATCH 5/5] Cover the other case --- test/unit/test_dependencies.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 2c9143d2f..054728dcb 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -507,6 +507,13 @@ class TestJobList(unittest.TestCase): self.assertEqual((value_two[0].name, value_two[1]), (parent.name, "2")) self.assertEqual(len(job_two.edge_info.get("RUNNING", "")), 1) + dependency = MagicMock() + dependency.relationships = {'CHUNKS_FROM': {'1': {'FROM_STEP': '1', 'CHUNKS_TO':'natural', 'DATES_TO': "dummy"}, '2': {'FROM_STEP': '2', 'CHUNKS_TO':'natural', 'DATES_TO': "dummy"}, }, 'STATUS': 'RUNNING'} + filters_to_apply = job_list.get_filters_to_apply(job, dependency) + filters_to_apply_two = job_list.get_filters_to_apply(job_two, dependency) + + assert filters_to_apply == {'CHUNKS_TO': 'natural', 'DATES_TO': 'dummy'} + assert filters_to_apply_two == {'CHUNKS_TO': 'natural', 'DATES_TO': 'dummy'} @patch('autosubmit.job.job_dict.date2str') def test_jobdict_get_jobs_filtered(self, mock_date2str): -- GitLab