From 1255b823a2b5eb1eb50b0695a9869ef85cb77630 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 27 Jul 2023 15:36:13 +0200 Subject: [PATCH 1/2] todo --- autosubmit/job/job_list.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index b642fcc2b..a6a71a29a 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -397,7 +397,13 @@ class JobList(object): :return: boolean """ filter_value = filter_value.strip("?") - filter_value = filter_value.strip("*") + if "*" in filter_value: + filter_value,split_info = filter_value.split("*") + split_info = int(split_info.split("\\")[-1]) + else: + split_info = None + # strip substring from char "*" + if "NONE".casefold() in str(parent_value).casefold(): return True to_filter = JobList._parse_filters_to_check(filter_value,associative_list,level_to_check) -- GitLab From 37a9d1b7c7f0561044601769c3c02e8bcd31ca08 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 31 Jul 2023 09:38:00 +0200 Subject: [PATCH 2/2] 1-to-N and n-to-1 implemented --- autosubmit/job/job.py | 4 +- autosubmit/job/job_list.py | 127 +++++++++++++++++++++++---------- test/unit/test_dependencies.py | 5 +- 3 files changed, 97 insertions(+), 39 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 876c70a37..7bf33b5d7 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1638,8 +1638,10 @@ class Job(object): except: pass for key, value in parameters.items(): + # parameters[key] can have '\\' characters that are interpreted as escape characters + # by re.sub. To avoid this, we use re.escape template_content = re.sub( - '%(? child_splits: + lesser = str(child_splits) + greater = str(parent_splits) + lesser_value = "child" + else: + lesser = str(parent_splits) + greater = str(child_splits) + lesser_value = "parent" + to_look_at_lesser = [associative_list[i:i + 1] for i in range(0, int(lesser), 1)] + for lesser_group in range(len(to_look_at_lesser)): + if lesser_value == "parent": + if str(parent_value) in to_look_at_lesser[lesser_group]: + break + else: + if str(child.split) in to_look_at_lesser[lesser_group]: + break + else: + to_look_at_lesser = associative_list + lesser_group = -1 + if "?" in filter_value: + # replace all ? for "" + filter_value = filter_value.replace("?", "") + if "*" in filter_value: + aux_filter = filter_value + filter_value = "" + for filter_ in aux_filter.split(","): + if "*" in filter_: + filter_,split_info = filter_.split("*") + if "\\" in split_info: + split_info = int(split_info.split("\\")[-1]) + else: + split_info = 1 + # split_info: if a value is 1, it means that the filter is 1-to-1, if it is 2, it means that the filter is 1-to-2, etc. + if (split_info == 1 or level_to_check.casefold() != "splits".casefold()) and str(parent_value).casefold() == str(filter_).casefold() : + if child.split == parent_value: + return True + elif split_info > 1 and level_to_check.casefold() == "splits".casefold(): + # 1-to-X filter + to_look_at_greater = [associative_list[i:i + split_info] for i in + range(0, int(greater), split_info)] + if lesser_value == "parent": + if str(child.split) in to_look_at_greater[lesser_group]: + return True + else: + if str(parent_value) in to_look_at_greater[lesser_group]: + return True + else: + filter_value += filter_ + "," + filter_value = filter_value[:-1] to_filter = JobList._parse_filters_to_check(filter_value,associative_list,level_to_check) if to_filter is None: return False @@ -418,7 +474,7 @@ class JobList(object): return True elif "NONE".casefold() == str(to_filter[0]).casefold(): return False - elif len( [ filter_ for filter_ in to_filter if str(parent_value).casefold() == str(filter_).strip("*").strip("?").casefold() ] )>0: + elif len( [ filter_ for filter_ in to_filter if str(parent_value).casefold() == str(filter_).casefold() ] )>0: return True else: return False @@ -666,6 +722,8 @@ class JobList(object): :param filter_type: "DATES_TO", "MEMBERS_TO", "CHUNKS_TO", "SPLITS_TO" :return: unified_filter """ + if len(unified_filter[filter_type]) > 0 and unified_filter[filter_type][-1] != ",": + unified_filter[filter_type] += "," if filter_type == "DATES_TO": value_list = self._date_list level_to_check = "DATES_FROM" @@ -703,11 +761,12 @@ class JobList(object): continue else: for ele in parsed_element: - if str(ele) not in unified_filter[filter_type]: - if len(unified_filter[filter_type]) > 0 and unified_filter[filter_type][-1] == ",": - unified_filter[filter_type] += str(ele) + extra_data - else: - unified_filter[filter_type] += "," + str(ele) + extra_data + "," + if extra_data: + check_whole_string = str(ele)+extra_data+"," + else: + check_whole_string = str(ele)+"," + if str(check_whole_string) not in unified_filter[filter_type]: + unified_filter[filter_type] += check_whole_string return unified_filter @staticmethod @@ -813,8 +872,17 @@ class JobList(object): associative_list["members"] = member_list associative_list["chunks"] = chunk_list - if parent.splits is not None: - associative_list["splits"] = [str(split) for split in range(1, int(parent.splits) + 1)] + if not child.splits: + child_splits = 0 + else: + child_splits = int(child.splits) + if not parent.splits: + parent_splits = 0 + else: + parent_splits = int(parent.splits) + splits = max(child_splits, parent_splits) + if splits > 0: + associative_list["splits"] = [str(split) for split in range(1, int(splits) + 1)] else: associative_list["splits"] = None dates_to = str(filter_.get("DATES_TO", "natural")).lower() @@ -840,25 +908,10 @@ class JobList(object): if "natural" in splits_to: associative_list["splits"] = [parent.split] if parent.split is not None else parent.splits parsed_parent_date = date2str(parent.date) if parent.date is not None else None - # Check for each * char in the filters - # Get all the dates that match * in the filter in a list separated by , - if "*" in dates_to: - dates_to = [ dat for dat in date_list.split(",") if dat is not None and "*" not in dat or ("*" in dat and date2str(child.date,"%Y%m%d") == dat.split("*")[0]) ] - dates_to = ",".join(dates_to) - if "*" in members_to: - members_to = [ mem for mem in member_list.split(",") if mem is not None and "*" not in mem or ("*" in mem and str(child.member) == mem.split("*")[0]) ] - members_to = ",".join(members_to) - if "*" in chunks_to: - chunks_to = [ chu for chu in chunk_list.split(",") if chu is not None and "*" not in chu or ("*" in chu and str(child.chunk) == chu.split("*")[0]) ] - chunks_to = ",".join(chunks_to) - if "*" in splits_to: - splits_to = [ spl for spl in splits_to.split(",") if child.split is None or spl is None or "*" not in spl or ("*" in spl and str(child.split) == spl.split("*")[0]) ] - splits_to = ",".join(splits_to) - # Apply all filters to look if this parent is an appropriated candidate for the current_job valid_dates = JobList._apply_filter(parsed_parent_date, dates_to, associative_list["dates"], "dates") valid_members = JobList._apply_filter(parent.member, members_to, associative_list["members"], "members") valid_chunks = JobList._apply_filter(parent.chunk, chunks_to, associative_list["chunks"], "chunks") - valid_splits = JobList._apply_filter(parent.split, splits_to, associative_list["splits"], "splits") + valid_splits = JobList._apply_filter(parent.split, splits_to, associative_list["splits"], "splits", child, parent) if valid_dates and valid_members and valid_chunks and valid_splits: return True return False diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index dbf93565e..1432abdbe 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -356,7 +356,7 @@ class TestJobList(unittest.TestCase): def test_valid_parent(self): - child = copy.deepcopy(self.mock_job) + # Call the function to get the result date_list = ["20020201", "20020202", "20020203", "20020204", "20020205", "20020206", "20020207", "20020208", "20020209", "20020210"] member_list = ["fc1", "fc2", "fc3"] @@ -375,6 +375,7 @@ class TestJobList(unittest.TestCase): self.mock_job.member = "fc2" self.mock_job.chunk = 1 self.mock_job.split = 1 + child = copy.deepcopy(self.mock_job) result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) # it returns a tuple, the first element is the result, the second is the optional flag self.assertEqual(result, True) @@ -429,6 +430,7 @@ class TestJobList(unittest.TestCase): def test_valid_parent_1_to_1(self): child = copy.deepcopy(self.mock_job) + child.splits = 6 date_list = ["20020201", "20020202", "20020203", "20020204", "20020205", "20020206", "20020207", "20020208", "20020209", "20020210"] member_list = ["fc1", "fc2", "fc3"] @@ -442,6 +444,7 @@ class TestJobList(unittest.TestCase): "CHUNKS_TO": "1,2,3,4,5,6", "SPLITS_TO": "1*,2*,3*,4*,5*,6" } + self.mock_job.splits = 6 self.mock_job.split = 1 self.mock_job.date = datetime.strptime("20020204", "%Y%m%d") self.mock_job.chunk = 5 -- GitLab