From 69eb1c4e63f1d85662328eee9f6ee96ace9241d7 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 3 May 2023 11:20:32 +0200 Subject: [PATCH 01/29] splits --- autosubmit/job/job_list.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 380431d52..257cc4783 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -506,14 +506,14 @@ class JobList(object): @staticmethod def _valid_parent(parent,member_list,date_list,chunk_list,is_a_natural_relation,filters_to_apply): ''' - Check if the parent is valid for the current_job + Check if the parent is valid for the current job :param parent: job to check - :param member_list: - :param date_list: - :param chunk_list: - :param is_a_natural_relation: - :param filters_to_apply: - :return: + :param member_list: list of members + :param date_list: list of dates + :param chunk_list: list of chunks + :param is_a_natural_relation: if the relation is natural or not + :param filters_to_apply: filters to apply + :return: True if the parent is valid, False otherwise ''' #check if current_parent is listed on dependency.relationships optional = False @@ -522,6 +522,7 @@ class JobList(object): dates_to = str(filter_.get("DATES_TO", "natural")).lower() members_to = str(filter_.get("MEMBERS_TO", "natural")).lower() chunks_to = str(filter_.get("CHUNKS_TO", "natural")).lower() + splits_to = str(filter_.get("SPLITS_TO", "natural")).lower() if not is_a_natural_relation: if dates_to == "natural": dates_to = "none" -- GitLab From 846bfe4299cd5d80a219d1c5b4c7477f11560332 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 10 May 2023 18:13:36 +0200 Subject: [PATCH 02/29] splits working --- autosubmit/job/job.py | 2 ++ autosubmit/job/job_dict.py | 3 +- autosubmit/job/job_list.py | 58 +++++++++++++++++++++----------------- 3 files changed, 36 insertions(+), 27 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index fc9e80e68..2784c683b 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -84,6 +84,7 @@ class Job(object): return "{0} STATUS: {1}".format(self.name, self.status) def __init__(self, name, job_id, status, priority): + self.splits = None self.script_name_wrapper = None self.delay_end = datetime.datetime.now() self.delay_retrials = "0" @@ -1113,6 +1114,7 @@ class Job(object): parameters['SDATE'] = date2str(self.date, self.date_format) parameters['MEMBER'] = self.member parameters['SPLIT'] = self.split + parameters['SPLITS'] = self.splits parameters['DELAY'] = self.delay parameters['FREQUENCY'] = self.frequency parameters['SYNCHRONIZE'] = self.synchronize diff --git a/autosubmit/job/job_dict.py b/autosubmit/job/job_dict.py index c6b8e9763..e5be47eb0 100644 --- a/autosubmit/job/job_dict.py +++ b/autosubmit/job/job_dict.py @@ -287,7 +287,7 @@ class DicJobs: else: for d in self._date_list: self._get_date(jobs, dic, d, member, chunk) - if len(jobs) > 1 and isinstance(jobs[0], Iterable): + if isinstance(jobs[0], Iterable): try: jobs_flattened = [job for jobs_to_flatten in jobs for job in jobs_to_flatten] jobs = jobs_flattened @@ -355,6 +355,7 @@ class DicJobs: job.date = date job.member = member job.chunk = chunk + job.splits = self.experiment_data["JOBS"].get(job.section,{}).get("SPLITS", None) job.date_format = self._date_format job.delete_when_edgeless = str(parameters[section].get("DELETE_WHEN_EDGELESS", "true")).lower() diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 257cc4783..97e355c09 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -427,8 +427,8 @@ class JobList(object): """ Check if the current_job_value is included in the filter_value :param relationship: current filter level to check. - :param level_to_check: can be a date, member or chunk. - :param value_to_check: can be a date, member or chunk. + :param level_to_check: can be a date, member, chunk or split. + :param value_to_check: can be a date, member, chunk or split. :return: """ optional = False @@ -457,11 +457,11 @@ class JobList(object): :return: ''' # Search all filters in dependency relationship that affect current_job - # First level can be Date,member or chunk or generic - # Second level can be member or chunk or generic - # Third level can only be chunked. + # First level can be Date,member or chunk or splits generic + # Second level can be member or chunk or splits generic + # Third level can only be chunk or splits. # If the filter is generic, it will be applied to all section jobs. - # Check Date then Member or Chunk then Chunk + # Check Date then Member or Chunk or split then Chunk or split then split optional = False filters_to_apply = [] # this should be a list @@ -479,6 +479,7 @@ class JobList(object): filters_to_apply = filters_to_apply_c elif "CHUNKS_FROM" in filters_to_apply[filter_number]: filters_to_apply,optional = JobList._check_relationship(filters_to_apply[filter_number],"CHUNKS_FROM",current_job.chunk) + # Check Member then Chunk if len(filters_to_apply[0]) == 0: filters_to_apply,optional = JobList._check_relationship(relationships,"MEMBERS_FROM",current_job.member) @@ -486,14 +487,22 @@ class JobList(object): filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply,"CHUNKS_FROM",current_job.chunk) if len(filters_to_apply_c) > 0: filters_to_apply = filters_to_apply_c - #Check Chunk + #Check Chunk then splits if len(filters_to_apply[0]) == 0: filters_to_apply,optional = JobList._check_relationship(relationships,"CHUNKS_FROM",current_job.chunk) + if len(filters_to_apply) > 0 and ( "SPLITS_FROM" in filters_to_apply): + filters_to_apply_s,optional = JobList._check_relationship(filters_to_apply,"SPLITS_FROM",current_job.split) + if len(filters_to_apply_s) > 0: + filters_to_apply = filters_to_apply_s + # Check Splits + if len(filters_to_apply[0]) == 0: + filters_to_apply,optional = JobList._check_relationship(relationships,"SPLITS_FROM",current_job.split) # Generic filter if len(filters_to_apply[0]) == 0: relationships.pop("CHUNKS_FROM",None) relationships.pop("MEMBERS_FROM",None) relationships.pop("DATES_FROM",None) + relationships.pop("SPLITS_FROM",None) filters_to_apply = [relationships] if len(filters_to_apply) == 1 and len(filters_to_apply[0]) == 0: @@ -530,27 +539,34 @@ class JobList(object): members_to = "none" if chunks_to == "natural": chunks_to = "none" + if splits_to == "natural": + splits_to = "none" associative_list["dates"] = date_list associative_list["members"] = member_list associative_list["chunks"] = chunk_list + if parent.splits is not None: + associative_list["splits"] = [ str(split) for split in range(1,int(parent.splits)+1) ] + else: + associative_list["splits"] = None if dates_to == "natural": associative_list["dates"] = [date2str(parent.date)] if parent.date is not None else date_list - if members_to == "natural": associative_list["members"] = [parent.member] if parent.member is not None else member_list if chunks_to == "natural": associative_list["chunks"] = [parent.chunk] if parent.chunk is not None else chunk_list + if splits_to == "natural": + associative_list["splits"] = [parent.split] parsed_parent_date = date2str(parent.date) if parent.date is not None else None # Apply all filters to look if this parent is an appropriated candidate for the current_job valid_dates = JobList._apply_filter(parsed_parent_date, dates_to, associative_list["dates"], "dates") valid_members = JobList._apply_filter(parent.member, members_to, associative_list["members"],"members") valid_chunks = JobList._apply_filter(parent.chunk, chunks_to, associative_list["chunks"], "chunks") - - if valid_dates and valid_members and valid_chunks: - if dates_to.find("?") != -1 or members_to.find("?") != -1 or chunks_to.find("?") != -1: + valid_splits = JobList._apply_filter(parent.split, splits_to, associative_list["splits"], "splits") + if valid_dates and valid_members and valid_chunks and valid_splits: + if dates_to.find("?") != -1 or members_to.find("?") != -1 or chunks_to.find("?") != -1 or splits_to.find("?") != -1: optional = True return True,optional return False,optional @@ -589,7 +605,9 @@ class JobList(object): #if dependency.sign in ["+", "-"]: # natural_jobs = dic_jobs.get_jobs(dependency.section, date, member,chunk) #else: - natural_jobs = dic_jobs.get_jobs(dependency.section, date, member,chunk) + natural_jobs = dic_jobs.get_jobs(dependency.section, date, member, chunk) + if job.split is not None: + natural_jobs = [p_job for p_job in natural_jobs if p_job.split == job.split] if dependency.sign in ['?']: optional_section = True else: @@ -617,28 +635,16 @@ class JobList(object): # Get dates_to, members_to, chunks_to of the deepest level of the relationship. filters_to_apply,optional_from = JobList._filter_current_job(job,copy.deepcopy(dependency.relationships)) if len(filters_to_apply) == 0: - filters_to_apply.append({"DATES_TO": "natural", "MEMBERS_TO": "natural", "CHUNKS_TO": "natural"}) + filters_to_apply.append({"DATES_TO": "natural", "MEMBERS_TO": "natural", "CHUNKS_TO": "natural", "SPLITS_TO": "natural"}) for parent in all_parents: - # Generic for all dependencies - if dependency.delay == -1 or chunk > dependency.delay: - if isinstance(parent, list): - if job.split is not None and len(str(job.split)) > 0: - parent = list(filter( - lambda _parent: _parent.split == job.split, parent)) - parent = parent[0] - else: - if dependency.splits is not None and len(str(dependency.splits)) > 0: - parent = [_parent for _parent in parent if _parent.split in dependency.splits] # If splits is not None, the job is a list of jobs if parent.name == job.name: continue # Check if it is a natural relation based in autosubmit terms ( same date,member,chunk ). - if parent in natural_jobs and ((job.chunk is None or parent.chunk is None or parent.chunk <= job.chunk ) and (parent.split is None or job.split is None or parent.split <= job.split) ) : + if parent in natural_jobs and ((job.chunk is None or parent.chunk is None or parent.chunk <= job.chunk ) and (parent.split is None or job.split is None or parent.split != job.split) ) : natural_relationship = True else: natural_relationship = False - if job.name.find("a002_20120101_0_2_SIM") != -1: - print("test") # Check if the current parent is a valid parent based on the dependencies set on expdef.conf valid,optional_to = JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship,filters_to_apply) if not valid: -- GitLab From b4b68d5650ef85f43d73aa7e0cbcd07aa6f014a5 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 11 May 2023 09:44:07 +0200 Subject: [PATCH 03/29] Normalized dependencies, splits bugs fixed. --- autosubmit/job/job.py | 14 -------------- autosubmit/job/job_list.py | 39 +++++++++----------------------------- 2 files changed, 9 insertions(+), 44 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 2784c683b..581c73fcf 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1186,20 +1186,6 @@ class Job(object): parameters['CHUNK_LAST'] = 'FALSE' parameters['NUMMEMBERS'] = len(as_conf.get_member_list()) parameters['DEPENDENCIES'] = str(as_conf.jobs_data[self.section].get("DEPENDENCIES","")) - # This shouldn't be necessary anymore as now all sub is done in the as_conf.reload() - # if len(self.export) > 0: - # variables = re.findall('%(? 0: - # variables = [variable[1:-1] for variable in variables] - # for key in variables: - # try: - # self.export = re.sub( - # '%(? 0: - aux = [] - for p_split in parents_jobs: - if type(p_split) is not list: - aux.append(p_split) - else: - for aux_job in p_split: - aux.append(aux_job) - parents_jobs = aux - if len(natural_jobs) > 0: - aux = [] - for p_split in natural_jobs: - if type(p_split) is not list: - aux.append(p_split) - else: - for aux_job in p_split: - aux.append(aux_job) - natural_jobs = aux all_parents = list(set(other_parents + parents_jobs)) # Get dates_to, members_to, chunks_to of the deepest level of the relationship. filters_to_apply,optional_from = JobList._filter_current_job(job,copy.deepcopy(dependency.relationships)) @@ -640,8 +619,8 @@ class JobList(object): # If splits is not None, the job is a list of jobs if parent.name == job.name: continue - # Check if it is a natural relation based in autosubmit terms ( same date,member,chunk ). - if parent in natural_jobs and ((job.chunk is None or parent.chunk is None or parent.chunk <= job.chunk ) and (parent.split is None or job.split is None or parent.split != job.split) ) : + # Check if it is a natural relation. The only difference is that a chunk can depend on a chunks <= than the current chunk + if parent in natural_jobs and ((job.chunk is None or parent.chunk is None or parent.chunk <= job.chunk )): natural_relationship = True else: natural_relationship = False -- GitLab From 1cbd185e9eb4f40592f5f8159fa66cc96b868bd3 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 11 May 2023 14:55:04 +0200 Subject: [PATCH 04/29] Normalized dependencies, splits bugs fixed. --- autosubmit/job/job_list.py | 89 +++++++++++++++++++++++++++----------- 1 file changed, 63 insertions(+), 26 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 3a55d84fe..2d8554503 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -439,7 +439,7 @@ class JobList(object): current_filter = {} if filter_type.upper().find(level_to_check.upper()) != -1: for filter_range in filter_data.keys(): - if str(value_to_check) is None or str(filter_range).upper().find(str(value_to_check).upper()) != -1: + if str(filter_range).upper() == "ALL" or str(value_to_check) is None or str(filter_range).upper().find(str(value_to_check).upper()) != -1: if filter_data[filter_range] is not None: if filter_type.find("?") != -1 or filter_range.find("?") != -1: optional = True @@ -449,45 +449,82 @@ class JobList(object): if len(filters) == 0: filters = [{}] return filters,optional + @staticmethod + def _filter_dates(current_job,relationships): + @staticmethod def _filter_current_job(current_job,relationships): ''' Check if the current_job is included in the filter_value ( from) - :param current_job: - :param dependency: - :return: + :param current_job: current job to check + :param dependency: dependency to check + :return: filter_to_apply(dict), boolean ''' - # Search all filters in dependency relationship that affect current_job - # First level can be Date,member or chunk or splits generic - # Second level can be member or chunk or splits generic - # Third level can only be chunk or splits. - # If the filter is generic, it will be applied to all section jobs. - # Check Date then Member or Chunk or split then Chunk or split then split + + # This function will look if the given relationship is set for the given job DATEs,MEMBER,CHUNK,SPLIT ( _from filters ) + # And if it is, it will return the dependencies that need to be activated (_TO filters) + # _FROM behavior: + # DATES_FROM can contain MEMBERS_FROM,CHUNKS_FROM,SPLITS_FROM + # MEMBERS_FROM can contain CHUNKS_FROM,SPLITS_FROM + # CHUNKS_FROM can contain SPLITS_FROM + # SPLITS_FROM can contain nothing + # _TO behavior: + # TO keywords, can be in any of the _FROM filters and they will only affect the _FROM filter they are in. + # There are 4 keywords: + # 1. ALL: all the dependencies will be activated of the given filter type (dates, members, chunks or/and splits) + # 2. NONE: no dependencies will be activated of the given filter type (dates, members, chunks or/and splits) + # 3. NATURAL: this is the normal behavior, represents a way of letting the job to be activated if they would normally be activated. + # 4. ? : this is a weak dependency activation flag, The dependency will be activated but the job can fail without affecting the workflow. + optional = False - filters_to_apply = [] - # this should be a list + filters_to_apply = [{}] + # Check if filter_from-filter_to relationship is set if relationships is not None and len(relationships) > 0: + # CHECK IF DATES FILTERS IS SET filters_to_apply,optional = JobList._check_relationship(relationships,"DATES_FROM",date2str(current_job.date)) + # IF DATES FILTERS IS SET if len(filters_to_apply[0]) > 0: - for filter_number in range(0, len(filters_to_apply)): + for filter_number in range(len(filters_to_apply)): + # CHECK IF MEMBERS FILTERS IS SET if "MEMBERS_FROM" in filters_to_apply[filter_number]: filters_to_apply_m,optional = JobList._check_relationship(filters_to_apply[filter_number],"MEMBERS_FROM",current_job.member) - if len(filters_to_apply_m) > 0: - filters_to_apply,optional = filters_to_apply_m - else: + # IF MEMBERS FILTERS IS SET + if len(filters_to_apply_m[filter_number]) > 0: + filters_to_apply = filters_to_apply_m + if "CHUNKS_FROM" in filters_to_apply[filter_number]: filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply[filter_number],"CHUNKS_FROM",current_job.chunk) - if len(filters_to_apply_c) > 0: + if len(filters_to_apply_c[filter_number]) > 0: filters_to_apply = filters_to_apply_c + if "SPLITS_FROM" in filters_to_apply[filter_number]: + filters_to_apply_s,optional = JobList._check_relationship(filters_to_apply[filter_number],"SPLITS_FROM",current_job.split) + if len(filters_to_apply_s[filter_number]) > 0: + filters_to_apply = filters_to_apply_s + # CHECK IF CHUNKS FILTERS IS SET (MEMBERS FILTERS IS NOT SET) elif "CHUNKS_FROM" in filters_to_apply[filter_number]: - filters_to_apply,optional = JobList._check_relationship(filters_to_apply[filter_number],"CHUNKS_FROM",current_job.chunk) - - # Check Member then Chunk + filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply[filter_number],"CHUNKS_FROM",current_job.chunk) + if len(filters_to_apply[filter_number]) > 0: + filters_to_apply = filters_to_apply_c + if "SPLITS_FROM" in filters_to_apply[filter_number]: + filters_to_apply_s,optional = JobList._check_relationship(filters_to_apply[filter_number],"SPLITS_FROM",current_job.split) + if len(filters_to_apply_s[filter_number]) > 0: + filters_to_apply = filters_to_apply_s + # CHECK IF SPLITS FILTERS IS SET (MEMBERS FILTERS IS NOT SET) + elif "SPLITS_FROM" in filters_to_apply[filter_number]: + filters_to_apply,optional = JobList._check_relationship(filters_to_apply[filter_number],"SPLITS_FROM",current_job.split) + # IF DATES FILTERS IS NOT SET, check if MEMBERS FILTERS IS SET if len(filters_to_apply[0]) == 0: - filters_to_apply,optional = JobList._check_relationship(relationships,"MEMBERS_FROM",current_job.member) - if len(filters_to_apply) > 0 and ( "CHUNKS_FROM" in filters_to_apply): - filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply,"CHUNKS_FROM",current_job.chunk) - if len(filters_to_apply_c) > 0: - filters_to_apply = filters_to_apply_c + for filter_number in range(len(filters_to_apply)): + filters_to_apply_m,optional = JobList._check_relationship(relationships,"MEMBERS_FROM",current_job.member) + if len(filters_to_apply_m[filter_number]) > 0: + filters_to_apply = filters_to_apply_m + if "CHUNKS_FROM" in filters_to_apply: + filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply,"CHUNKS_FROM",current_job.chunk) + if len(filters_to_apply_c) > 0: + filters_to_apply = filters_to_apply_c + if "SPLITS_FROM" in filters_to_apply: + filters_to_apply_s,optional = JobList._check_relationship(filters_to_apply,"SPLITS_FROM",current_job.split) + if len(filters_to_apply_s) > 0: + filters_to_apply = filters_to_apply_s #Check Chunk then splits if len(filters_to_apply[0]) == 0: filters_to_apply,optional = JobList._check_relationship(relationships,"CHUNKS_FROM",current_job.chunk) @@ -498,7 +535,7 @@ class JobList(object): # Check Splits if len(filters_to_apply[0]) == 0: filters_to_apply,optional = JobList._check_relationship(relationships,"SPLITS_FROM",current_job.split) - # Generic filter + # Global filter if len(filters_to_apply[0]) == 0: relationships.pop("CHUNKS_FROM",None) relationships.pop("MEMBERS_FROM",None) -- GitLab From c6103109c84f1160b16a23c2dedb6f77b8195f89 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 15 May 2023 15:01:57 +0200 Subject: [PATCH 05/29] All working now, todo pipeline tests --- autosubmit/job/job_list.py | 107 ++++++++++++++++++----- docs/source/userguide/wrappers/index.rst | 2 +- 2 files changed, 87 insertions(+), 22 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 2d8554503..facbc05db 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -449,8 +449,59 @@ class JobList(object): if len(filters) == 0: filters = [{}] return filters,optional + + @staticmethod + def _check_dates(relationships, current_job): + filters_to_apply, optional = JobList._check_relationship(relationships, "DATES_FROM", date2str(current_job.date)) + for filter in filters_to_apply: + if "MEMBERS_FROM" in filter: + filters_to_apply_m, optional_m = JobList._check_members(filter, current_job) + if len(filters_to_apply_m) > 0: + filters_to_apply = filters_to_apply_m + optional = optional_m + if "CHUNKS_FROM" in filter: + filters_to_apply_c, optional_c = JobList._check_chunks(filter, current_job) + if len(filters_to_apply_c) > 0: + filters_to_apply = filters_to_apply_c + optional = optional_c + if "SPLITS_FROM" in filter: + filters_to_apply_s, optional_s = JobList._check_splits(filter, current_job) + if len(filters_to_apply_s) > 0: + filters_to_apply = filters_to_apply_s + optional = optional_s + return filters_to_apply, optional + @staticmethod - def _filter_dates(current_job,relationships): + def _check_members(relationships, current_job): + filters_to_apply, optional = JobList._check_relationship(relationships, "MEMBERS_FROM", current_job.member) + for filter in filters_to_apply: + if "CHUNKS_FROM" in filter: + filters_to_apply_c, optional_c = JobList._check_chunks(filter, current_job) + if len(filters_to_apply_c) > 0: + filters_to_apply = filters_to_apply_c + optional = optional_c + if "SPLITS_FROM" in filter: + filters_to_apply_s, optional_s = JobList._check_splits(filter, current_job) + if len(filters_to_apply_s) > 0: + filters_to_apply = filters_to_apply_s + optional = optional_s + return filters_to_apply, optional + + @staticmethod + def _check_chunks(relationships, current_job): + filters_to_apply, optional = JobList._check_relationship(relationships, "CHUNKS_FROM", current_job.chunk) + for filter in filters_to_apply: + if "SPLITS_FROM" in filter: + filters_to_apply_s, optional_s = JobList._check_splits(filter, current_job) + if len(filters_to_apply_s) > 0: + filters_to_apply = filters_to_apply_s + optional = optional_s + return filters_to_apply, optional + + @staticmethod + def _check_splits(relationships, current_job): + filters_to_apply, optional = JobList._check_relationship(relationships, "SPLITS_FROM", current_job.split) + return filters_to_apply, optional @staticmethod def _filter_current_job(current_job,relationships): @@ -481,7 +532,21 @@ class JobList(object): # Check if filter_from-filter_to relationship is set if relationships is not None and len(relationships) > 0: # CHECK IF DATES FILTERS IS SET - filters_to_apply,optional = JobList._check_relationship(relationships,"DATES_FROM",date2str(current_job.date)) + if "DATES_FROM" in relationships: + filters_to_apply, optional = JobList._check_dates(relationships, current_job) + elif "MEMBERS_FROM" in relationships: + filters_to_apply, optional = JobList._check_members(relationships, current_job) + elif "CHUNKS_FROM" in relationships: + filters_to_apply, optional = JobList._check_chunks(relationships, current_job) + elif "SPLITS_FROM" in relationships: + filters_to_apply, optional = JobList._check_splits(relationships, current_job) + else: + relationships.pop("CHUNKS_FROM", None) + relationships.pop("MEMBERS_FROM", None) + relationships.pop("DATES_FROM", None) + relationships.pop("SPLITS_FROM", None) + filters_to_apply = [relationships] + return filters_to_apply,optional # IF DATES FILTERS IS SET if len(filters_to_apply[0]) > 0: for filter_number in range(len(filters_to_apply)): @@ -489,16 +554,16 @@ class JobList(object): if "MEMBERS_FROM" in filters_to_apply[filter_number]: filters_to_apply_m,optional = JobList._check_relationship(filters_to_apply[filter_number],"MEMBERS_FROM",current_job.member) # IF MEMBERS FILTERS IS SET - if len(filters_to_apply_m[filter_number]) > 0: - filters_to_apply = filters_to_apply_m - if "CHUNKS_FROM" in filters_to_apply[filter_number]: - filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply[filter_number],"CHUNKS_FROM",current_job.chunk) - if len(filters_to_apply_c[filter_number]) > 0: - filters_to_apply = filters_to_apply_c - if "SPLITS_FROM" in filters_to_apply[filter_number]: - filters_to_apply_s,optional = JobList._check_relationship(filters_to_apply[filter_number],"SPLITS_FROM",current_job.split) - if len(filters_to_apply_s[filter_number]) > 0: - filters_to_apply = filters_to_apply_s + if len(filters_to_apply_m[filter_number]) > 0: + filters_to_apply = filters_to_apply_m + if "CHUNKS_FROM" in filters_to_apply[filter_number]: + filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply[filter_number],"CHUNKS_FROM",current_job.chunk) + if len(filters_to_apply_c[filter_number]) > 0: + filters_to_apply = filters_to_apply_c + if "SPLITS_FROM" in filters_to_apply[filter_number]: + filters_to_apply_s,optional = JobList._check_relationship(filters_to_apply[filter_number],"SPLITS_FROM",current_job.split) + if len(filters_to_apply_s[filter_number]) > 0: + filters_to_apply = filters_to_apply_s # CHECK IF CHUNKS FILTERS IS SET (MEMBERS FILTERS IS NOT SET) elif "CHUNKS_FROM" in filters_to_apply[filter_number]: filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply[filter_number],"CHUNKS_FROM",current_job.chunk) @@ -513,10 +578,8 @@ class JobList(object): filters_to_apply,optional = JobList._check_relationship(filters_to_apply[filter_number],"SPLITS_FROM",current_job.split) # IF DATES FILTERS IS NOT SET, check if MEMBERS FILTERS IS SET if len(filters_to_apply[0]) == 0: + filters_to_apply, optional = JobList._check_relationship(relationships, "MEMBERS_FROM", current_job.member) for filter_number in range(len(filters_to_apply)): - filters_to_apply_m,optional = JobList._check_relationship(relationships,"MEMBERS_FROM",current_job.member) - if len(filters_to_apply_m[filter_number]) > 0: - filters_to_apply = filters_to_apply_m if "CHUNKS_FROM" in filters_to_apply: filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply,"CHUNKS_FROM",current_job.chunk) if len(filters_to_apply_c) > 0: @@ -527,14 +590,16 @@ class JobList(object): filters_to_apply = filters_to_apply_s #Check Chunk then splits if len(filters_to_apply[0]) == 0: - filters_to_apply,optional = JobList._check_relationship(relationships,"CHUNKS_FROM",current_job.chunk) - if len(filters_to_apply) > 0 and ( "SPLITS_FROM" in filters_to_apply): - filters_to_apply_s,optional = JobList._check_relationship(filters_to_apply,"SPLITS_FROM",current_job.split) - if len(filters_to_apply_s) > 0: - filters_to_apply = filters_to_apply_s + filters_to_apply, optional = JobList._check_relationship(relationships, "CHUNKS_FROM", current_job.chunk) + for filter_number in range(len(filters_to_apply)): + if "SPLITS_FROM" in filters_to_apply[filter_number]: + filters_to_apply_s,optional = JobList._filter_splits(filters_to_apply, "SPLITS_FROM", current_job.split) + if len(filters_to_apply_s) > 0: + filters_to_apply = filters_to_apply_s # Check Splits if len(filters_to_apply[0]) == 0: - filters_to_apply,optional = JobList._check_relationship(relationships,"SPLITS_FROM",current_job.split) + filters_to_apply, optional = JobList._check_relationship(relationships, "SPLITS_FROM",current_job.split) + # Global filter if len(filters_to_apply[0]) == 0: relationships.pop("CHUNKS_FROM",None) diff --git a/docs/source/userguide/wrappers/index.rst b/docs/source/userguide/wrappers/index.rst index 168e5afa8..155d4d66a 100644 --- a/docs/source/userguide/wrappers/index.rst +++ b/docs/source/userguide/wrappers/index.rst @@ -392,7 +392,7 @@ Considering the following configuration: "20120201": CHUNKS_FROM: 1: - DATES_TO: "20120101" + DATES_TO: "º" CHUNKS_TO: "1" RUNNING: chunk SYNCHRONIZE: member -- GitLab From 92b061d084afeb44fdca748fd1d6f25d7ab8c19d Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 25 May 2023 11:57:17 +0200 Subject: [PATCH 06/29] fixed tests --- autosubmit/job/job_dict.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit/job/job_dict.py b/autosubmit/job/job_dict.py index e5be47eb0..e2f673563 100644 --- a/autosubmit/job/job_dict.py +++ b/autosubmit/job/job_dict.py @@ -287,7 +287,7 @@ class DicJobs: else: for d in self._date_list: self._get_date(jobs, dic, d, member, chunk) - if isinstance(jobs[0], Iterable): + if len(jobs) > 0 and isinstance(jobs[0], list): try: jobs_flattened = [job for jobs_to_flatten in jobs for job in jobs_to_flatten] jobs = jobs_flattened -- GitLab From 56edf2932e861040ef02834c5bc98daa0dc7149a Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 26 May 2023 15:53:40 +0200 Subject: [PATCH 07/29] Fix algorithm --- autosubmit/job/job_list.py | 301 ++++++++++++++++--------------------- 1 file changed, 133 insertions(+), 168 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index facbc05db..dcb2e3650 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -429,79 +429,118 @@ class JobList(object): Check if the current_job_value is included in the filter_value :param relationship: current filter level to check. :param level_to_check: can be a date, member, chunk or split. - :param value_to_check: can be a date, member, chunk or split. + :param value_to_check: Can be None, a date, a member, a chunk or a split. :return: """ - optional = False filters = [] - for filter_type, filter_data in relationships.items(): - if isinstance(filter_data, collections.abc.Mapping): - current_filter = {} - if filter_type.upper().find(level_to_check.upper()) != -1: - for filter_range in filter_data.keys(): - if str(filter_range).upper() == "ALL" or str(value_to_check) is None or str(filter_range).upper().find(str(value_to_check).upper()) != -1: - if filter_data[filter_range] is not None: - if filter_type.find("?") != -1 or filter_range.find("?") != -1: - optional = True - current_filter.update(filter_data[filter_range]) - filters.append(current_filter) + for filter_range,filter_data in relationships.get(level_to_check,{}).items(): + if not str(value_to_check) or str(filter_range).upper() in "ALL" or str(value_to_check).upper() in str(filter_range).upper(): + if filter_data: + if "?" in filter_range: + filter_data["OPTIONAL"] = True + else: + filter_data["OPTIONAL"] = relationships["OPTIONAL"] + filters.append(filter_data) # Normalize the filter return if len(filters) == 0: filters = [{}] - return filters,optional + return filters @staticmethod def _check_dates(relationships, current_job): - filters_to_apply, optional = JobList._check_relationship(relationships, "DATES_FROM", date2str(current_job.date)) - for filter in filters_to_apply: + filters_to_apply = JobList._check_relationship(relationships, "DATES_FROM", date2str(current_job.date)) + for i,filter in enumerate(filters_to_apply): + optional = filter.pop("OPTIONAL", False) if "MEMBERS_FROM" in filter: - filters_to_apply_m, optional_m = JobList._check_members(filter, current_job) + filters_to_apply_m = JobList._check_members({"MEMBERS_FROM": (filter.pop("MEMBERS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_m) > 0: - filters_to_apply = filters_to_apply_m - optional = optional_m + filters_to_apply[i] = filters_to_apply_m if "CHUNKS_FROM" in filter: - filters_to_apply_c, optional_c = JobList._check_chunks(filter, current_job) - if len(filters_to_apply_c) > 0: - filters_to_apply = filters_to_apply_c - optional = optional_c + filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM")),"OPTIONAL":optional}, current_job) + if len(filters_to_apply_c) > 0 and len(filters_to_apply_c[0]) > 0: + filters_to_apply[i] = filters_to_apply_c if "SPLITS_FROM" in filter: - filters_to_apply_s, optional_s = JobList._check_splits(filter, current_job) + filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_s) > 0: - filters_to_apply = filters_to_apply_s - optional = optional_s - return filters_to_apply, optional + filters_to_apply[i] = filters_to_apply_s + filters_to_apply = JobList._unify_to_filters(filters_to_apply) + return filters_to_apply @staticmethod def _check_members(relationships, current_job): - filters_to_apply, optional = JobList._check_relationship(relationships, "MEMBERS_FROM", current_job.member) - for filter in filters_to_apply: + filters_to_apply = JobList._check_relationship(relationships, "MEMBERS_FROM", current_job.member) + for i,filter in enumerate(filters_to_apply): + optional = filter.pop("OPTIONAL", False) if "CHUNKS_FROM" in filter: - filters_to_apply_c, optional_c = JobList._check_chunks(filter, current_job) + filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_c) > 0: - filters_to_apply = filters_to_apply_c - optional = optional_c + filters_to_apply[i] = filters_to_apply_c if "SPLITS_FROM" in filter: - filters_to_apply_s, optional_s = JobList._check_splits(filter, current_job) + filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_s) > 0: - filters_to_apply = filters_to_apply_s - optional = optional_s - return filters_to_apply, optional + filters_to_apply[i] = filters_to_apply_s + filters_to_apply = JobList._unify_to_filters(filters_to_apply) + return filters_to_apply @staticmethod def _check_chunks(relationships, current_job): - filters_to_apply, optional = JobList._check_relationship(relationships, "CHUNKS_FROM", current_job.chunk) - for filter in filters_to_apply: + filters_to_apply = JobList._check_relationship(relationships, "CHUNKS_FROM", current_job.chunk) + for i,filter in enumerate(filters_to_apply): + optional = filter.pop("OPTIONAL", False) if "SPLITS_FROM" in filter: - filters_to_apply_s, optional_s = JobList._check_splits(filter, current_job) + filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_s) > 0: - filters_to_apply = filters_to_apply_s - optional = optional_s - return filters_to_apply, optional + filters_to_apply[i] = filters_to_apply_s + filters_to_apply = JobList._unify_to_filters(filters_to_apply) + return filters_to_apply @staticmethod def _check_splits(relationships, current_job): - filters_to_apply, optional = JobList._check_relationship(relationships, "SPLITS_FROM", current_job.split) - return filters_to_apply, optional + filters_to_apply = JobList._check_relationship(relationships, "SPLITS_FROM", current_job.split) + # No more FROM sections to check, unify _to FILTERS and return + filters_to_apply = JobList._unify_to_filters(filters_to_apply) + return filters_to_apply + + @staticmethod + def _unify_to_filter(unified_filter,filter_to,filter_type): + if "all" not in unified_filter[filter_type]: + aux = filter_to.pop(filter_type, None) + if aux: + aux = aux.split(",") + for element in aux: + if element.lower().strip("?") in ["natural","none"] and len(unified_filter[filter_type]) > 0: + continue + else: + if filter_to.get("OPTIONAL",False) and element[-1] != "?": + element += "?" + unified_filter[filter_type].add(element) + @staticmethod + def _normalize_to_filters(filter_to,filter_type): + if len(filter_to[filter_type]) == 0: + filter_to.pop(filter_type, None) + elif "all" in filter_to[filter_type]: + filter_to[filter_type] = "all" + else: + # transform to str separated by commas if multiple elements + filter_to[filter_type] = ",".join(filter_to[filter_type]) + + @staticmethod + def _unify_to_filters(filter_to_apply): + unified_filter = {"DATES_TO": set(), "MEMBERS_TO": set(), "CHUNKS_TO": set(), "SPLITS_TO": set()} + for filter_to in filter_to_apply: + if len(filter_to) > 0: + JobList._unify_to_filter(unified_filter,filter_to,"DATES_TO") + JobList._unify_to_filter(unified_filter,filter_to,"MEMBERS_TO") + JobList._unify_to_filter(unified_filter,filter_to,"CHUNKS_TO") + JobList._unify_to_filter(unified_filter,filter_to,"SPLITS_TO") + filter_to.pop("OPTIONAL", None) + + JobList._normalize_to_filters(unified_filter,"DATES_TO") + JobList._normalize_to_filters(unified_filter,"MEMBERS_TO") + JobList._normalize_to_filters(unified_filter,"CHUNKS_TO") + JobList._normalize_to_filters(unified_filter,"SPLITS_TO") + + return unified_filter @staticmethod def _filter_current_job(current_job,relationships): @@ -527,96 +566,33 @@ class JobList(object): # 3. NATURAL: this is the normal behavior, represents a way of letting the job to be activated if they would normally be activated. # 4. ? : this is a weak dependency activation flag, The dependency will be activated but the job can fail without affecting the workflow. - optional = False filters_to_apply = [{}] # Check if filter_from-filter_to relationship is set + relationships["OPTIONAL"] = False + if current_job.section.lower() == "opa": + print("debugging") if relationships is not None and len(relationships) > 0: - # CHECK IF DATES FILTERS IS SET + # Look for a starting point if "DATES_FROM" in relationships: - filters_to_apply, optional = JobList._check_dates(relationships, current_job) + filters_to_apply = JobList._check_dates(relationships, current_job) elif "MEMBERS_FROM" in relationships: - filters_to_apply, optional = JobList._check_members(relationships, current_job) + filters_to_apply = JobList._check_members(relationships, current_job) elif "CHUNKS_FROM" in relationships: - filters_to_apply, optional = JobList._check_chunks(relationships, current_job) + filters_to_apply = JobList._check_chunks(relationships, current_job) elif "SPLITS_FROM" in relationships: - filters_to_apply, optional = JobList._check_splits(relationships, current_job) + filters_to_apply = JobList._check_splits(relationships, current_job) else: relationships.pop("CHUNKS_FROM", None) relationships.pop("MEMBERS_FROM", None) relationships.pop("DATES_FROM", None) relationships.pop("SPLITS_FROM", None) filters_to_apply = [relationships] - return filters_to_apply,optional - # IF DATES FILTERS IS SET - if len(filters_to_apply[0]) > 0: - for filter_number in range(len(filters_to_apply)): - # CHECK IF MEMBERS FILTERS IS SET - if "MEMBERS_FROM" in filters_to_apply[filter_number]: - filters_to_apply_m,optional = JobList._check_relationship(filters_to_apply[filter_number],"MEMBERS_FROM",current_job.member) - # IF MEMBERS FILTERS IS SET - if len(filters_to_apply_m[filter_number]) > 0: - filters_to_apply = filters_to_apply_m - if "CHUNKS_FROM" in filters_to_apply[filter_number]: - filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply[filter_number],"CHUNKS_FROM",current_job.chunk) - if len(filters_to_apply_c[filter_number]) > 0: - filters_to_apply = filters_to_apply_c - if "SPLITS_FROM" in filters_to_apply[filter_number]: - filters_to_apply_s,optional = JobList._check_relationship(filters_to_apply[filter_number],"SPLITS_FROM",current_job.split) - if len(filters_to_apply_s[filter_number]) > 0: - filters_to_apply = filters_to_apply_s - # CHECK IF CHUNKS FILTERS IS SET (MEMBERS FILTERS IS NOT SET) - elif "CHUNKS_FROM" in filters_to_apply[filter_number]: - filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply[filter_number],"CHUNKS_FROM",current_job.chunk) - if len(filters_to_apply[filter_number]) > 0: - filters_to_apply = filters_to_apply_c - if "SPLITS_FROM" in filters_to_apply[filter_number]: - filters_to_apply_s,optional = JobList._check_relationship(filters_to_apply[filter_number],"SPLITS_FROM",current_job.split) - if len(filters_to_apply_s[filter_number]) > 0: - filters_to_apply = filters_to_apply_s - # CHECK IF SPLITS FILTERS IS SET (MEMBERS FILTERS IS NOT SET) - elif "SPLITS_FROM" in filters_to_apply[filter_number]: - filters_to_apply,optional = JobList._check_relationship(filters_to_apply[filter_number],"SPLITS_FROM",current_job.split) - # IF DATES FILTERS IS NOT SET, check if MEMBERS FILTERS IS SET - if len(filters_to_apply[0]) == 0: - filters_to_apply, optional = JobList._check_relationship(relationships, "MEMBERS_FROM", current_job.member) - for filter_number in range(len(filters_to_apply)): - if "CHUNKS_FROM" in filters_to_apply: - filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply,"CHUNKS_FROM",current_job.chunk) - if len(filters_to_apply_c) > 0: - filters_to_apply = filters_to_apply_c - if "SPLITS_FROM" in filters_to_apply: - filters_to_apply_s,optional = JobList._check_relationship(filters_to_apply,"SPLITS_FROM",current_job.split) - if len(filters_to_apply_s) > 0: - filters_to_apply = filters_to_apply_s - #Check Chunk then splits - if len(filters_to_apply[0]) == 0: - filters_to_apply, optional = JobList._check_relationship(relationships, "CHUNKS_FROM", current_job.chunk) - for filter_number in range(len(filters_to_apply)): - if "SPLITS_FROM" in filters_to_apply[filter_number]: - filters_to_apply_s,optional = JobList._filter_splits(filters_to_apply, "SPLITS_FROM", current_job.split) - if len(filters_to_apply_s) > 0: - filters_to_apply = filters_to_apply_s - # Check Splits - if len(filters_to_apply[0]) == 0: - filters_to_apply, optional = JobList._check_relationship(relationships, "SPLITS_FROM",current_job.split) - - # Global filter - if len(filters_to_apply[0]) == 0: - relationships.pop("CHUNKS_FROM",None) - relationships.pop("MEMBERS_FROM",None) - relationships.pop("DATES_FROM",None) - relationships.pop("SPLITS_FROM",None) - filters_to_apply = [relationships] - - if len(filters_to_apply) == 1 and len(filters_to_apply[0]) == 0: - return [],optional - else: - return filters_to_apply,optional + return filters_to_apply @staticmethod - def _valid_parent(parent,member_list,date_list,chunk_list,is_a_natural_relation,filters_to_apply): + def _valid_parent(parent,member_list,date_list,chunk_list,is_a_natural_relation,filter_): ''' Check if the parent is valid for the current job :param parent: job to check @@ -629,49 +605,46 @@ class JobList(object): ''' #check if current_parent is listed on dependency.relationships optional = False - for filter_ in filters_to_apply: - associative_list = {} - dates_to = str(filter_.get("DATES_TO", "natural")).lower() - members_to = str(filter_.get("MEMBERS_TO", "natural")).lower() - chunks_to = str(filter_.get("CHUNKS_TO", "natural")).lower() - splits_to = str(filter_.get("SPLITS_TO", "natural")).lower() - if not is_a_natural_relation: - if dates_to == "natural": - dates_to = "none" - if members_to == "natural": - members_to = "none" - if chunks_to == "natural": - chunks_to = "none" - if splits_to == "natural": - splits_to = "none" - - - associative_list["dates"] = date_list - associative_list["members"] = member_list - associative_list["chunks"] = chunk_list - if parent.splits is not None: - associative_list["splits"] = [ str(split) for split in range(1,int(parent.splits)+1) ] - else: - associative_list["splits"] = None - + associative_list = {} + dates_to = str(filter_.get("DATES_TO", "natural")).lower() + members_to = str(filter_.get("MEMBERS_TO", "natural")).lower() + chunks_to = str(filter_.get("CHUNKS_TO", "natural")).lower() + splits_to = str(filter_.get("SPLITS_TO", "natural")).lower() + if not is_a_natural_relation: if dates_to == "natural": - associative_list["dates"] = [date2str(parent.date)] if parent.date is not None else date_list + dates_to = "none" if members_to == "natural": - associative_list["members"] = [parent.member] if parent.member is not None else member_list + members_to = "none" if chunks_to == "natural": - associative_list["chunks"] = [parent.chunk] if parent.chunk is not None else chunk_list + chunks_to = "none" if splits_to == "natural": - associative_list["splits"] = [parent.split] if parent.split is not None else parent.splits - parsed_parent_date = date2str(parent.date) if parent.date is not None else None - # Apply all filters to look if this parent is an appropriated candidate for the current_job - valid_dates = JobList._apply_filter(parsed_parent_date, dates_to, associative_list["dates"], "dates") - valid_members = JobList._apply_filter(parent.member, members_to, associative_list["members"],"members") - valid_chunks = JobList._apply_filter(parent.chunk, chunks_to, associative_list["chunks"], "chunks") - valid_splits = JobList._apply_filter(parent.split, splits_to, associative_list["splits"], "splits") - if valid_dates and valid_members and valid_chunks and valid_splits: - if dates_to.find("?") != -1 or members_to.find("?") != -1 or chunks_to.find("?") != -1 or splits_to.find("?") != -1: - optional = True - return True,optional + splits_to = "none" + associative_list["dates"] = date_list + associative_list["members"] = member_list + associative_list["chunks"] = chunk_list + if parent.splits is not None: + associative_list["splits"] = [ str(split) for split in range(1,int(parent.splits)+1) ] + else: + associative_list["splits"] = None + + if dates_to == "natural": + associative_list["dates"] = [date2str(parent.date)] if parent.date is not None else date_list + if members_to == "natural": + associative_list["members"] = [parent.member] if parent.member is not None else member_list + if chunks_to == "natural": + associative_list["chunks"] = [parent.chunk] if parent.chunk is not None else chunk_list + if splits_to == "natural": + associative_list["splits"] = [parent.split] if parent.split is not None else parent.splits + parsed_parent_date = date2str(parent.date) if parent.date is not None else None + # Apply all filters to look if this parent is an appropriated candidate for the current_job + valid_dates = JobList._apply_filter(parsed_parent_date, dates_to, associative_list["dates"], "dates") + valid_members = JobList._apply_filter(parent.member, members_to, associative_list["members"],"members") + valid_chunks = JobList._apply_filter(parent.chunk, chunks_to, associative_list["chunks"], "chunks") + valid_splits = JobList._apply_filter(parent.split, splits_to, associative_list["splits"], "splits") + if valid_dates and valid_members and valid_chunks and valid_splits: + if dates_to.find("?") != -1 or members_to.find("?") != -1 or chunks_to.find("?") != -1 or splits_to.find("?") != -1: + optional = True + return True,optional return False,optional @staticmethod def _manage_job_dependencies(dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys, dependencies, @@ -706,28 +679,20 @@ class JobList(object): other_parents = dic_jobs.get_jobs(dependency.section, None, None, None) parents_jobs = dic_jobs.get_jobs(dependency.section, date, member, chunk) natural_jobs = dic_jobs.get_jobs(dependency.section, date, member, chunk) - #if job.split is not None: - # natural_jobs = [p_job for p_job in natural_jobs if p_job.split == job.split or p_job.split is None] - if dependency.sign in ['?']: - optional_section = True - else: - optional_section = False all_parents = list(set(other_parents + parents_jobs)) # Get dates_to, members_to, chunks_to of the deepest level of the relationship. - filters_to_apply,optional_from = JobList._filter_current_job(job,copy.deepcopy(dependency.relationships)) - if len(filters_to_apply) == 0: - filters_to_apply.append({"DATES_TO": "natural", "MEMBERS_TO": "natural", "CHUNKS_TO": "natural", "SPLITS_TO": "natural"}) + filters_to_apply = JobList._filter_current_job(job,copy.deepcopy(dependency.relationships)) for parent in all_parents: # If splits is not None, the job is a list of jobs if parent.name == job.name: continue # Check if it is a natural relation. The only difference is that a chunk can depend on a chunks <= than the current chunk - if parent in natural_jobs and ((job.chunk is None or parent.chunk is None or parent.chunk <= job.chunk )): + if parent in natural_jobs and (job.chunk is None or parent.chunk is None or parent.chunk <= job.chunk ): natural_relationship = True else: natural_relationship = False # Check if the current parent is a valid parent based on the dependencies set on expdef.conf - valid,optional_to = JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship,filters_to_apply) + valid,optional = JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship,filters_to_apply) if not valid: continue else: @@ -736,7 +701,7 @@ class JobList(object): job.add_parent(parent) JobList._add_edge(graph, job, parent) # Could be more variables in the future - if optional_to or optional_from or optional_section: + if optional: job.add_edge_info(parent.name,special_variables={"optional":True}) JobList.handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member, member_list, dependency.section, graph, other_parents) -- GitLab From 703a31ea0448d31d791cd8d6803b805095d09792 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 26 May 2023 16:13:14 +0200 Subject: [PATCH 08/29] commented the function --- autosubmit/job/job_list.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index dcb2e3650..0252c742d 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -449,21 +449,51 @@ class JobList(object): @staticmethod def _check_dates(relationships, current_job): filters_to_apply = JobList._check_relationship(relationships, "DATES_FROM", date2str(current_job.date)) + # there could be multiple filters that apply... per example + # Current task date is 20020201, and member is fc2 + # Dummy example, not specially usefull in a real case + #DATES_FROM: + #all: + #MEMBERS_FROM: + #ALL: ... + #CHUNKS_FROM: + #ALL: ... + #20020201: + #MEMBERS_FROM: + #fc2: + #DATES_TO: "20020201" + #MEMBERS_TO: "fc2" + #CHUNKS_TO: "ALL" + #SPLITS_FROM: + #ALL: + #SPLITS_TO: "1" + # this "for" iterates for ALL and fc2 as current task is selected in both filters + # The dict in this step is: + # [{MEMBERS_FROM{..},CHUNKS_FROM{...}},{MEMBERS_FROM{..},SPLITS_FROM{...}}] for i,filter in enumerate(filters_to_apply): + # {MEMBERS_FROM{..},CHUNKS_FROM{...}} I want too look ALL filters not only one, but I want to go recursivily until get the _TO filter optional = filter.pop("OPTIONAL", False) + # This is not an if_else, because the current level ( dates ) could have two different filters. + # Second case commented: ( date_from 20020201 ) + # Will enter, go recursivily to the similar methods and in the end it will do: + # Will enter members_from, and obtain [{DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", CHUNKS_FROM{...}] if "MEMBERS_FROM" in filter: filters_to_apply_m = JobList._check_members({"MEMBERS_FROM": (filter.pop("MEMBERS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_m) > 0: filters_to_apply[i] = filters_to_apply_m + # Will enter chunks_from, and obtain [{DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"] if "CHUNKS_FROM" in filter: filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_c) > 0 and len(filters_to_apply_c[0]) > 0: filters_to_apply[i] = filters_to_apply_c + #IGNORED if "SPLITS_FROM" in filter: filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_s) > 0: filters_to_apply[i] = filters_to_apply_s + # Unify filters from all filters_from where the current job is included to have a single SET of filters_to filters_to_apply = JobList._unify_to_filters(filters_to_apply) + # {DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"} return filters_to_apply @staticmethod @@ -572,7 +602,7 @@ class JobList(object): if current_job.section.lower() == "opa": print("debugging") if relationships is not None and len(relationships) > 0: - # Look for a starting point + # Look for a starting point, this can be if else becasue they're exclusive as a DATE_FROM can't be in a MEMBER_FROM and so on if "DATES_FROM" in relationships: filters_to_apply = JobList._check_dates(relationships, current_job) elif "MEMBERS_FROM" in relationships: -- GitLab From 2f52b6a27a60cb1e4a35cec910516074b0157c2f Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 26 May 2023 16:32:18 +0200 Subject: [PATCH 09/29] Added docstrings --- autosubmit/job/job_list.py | 50 +++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 0252c742d..5b5533f87 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -448,6 +448,12 @@ class JobList(object): @staticmethod def _check_dates(relationships, current_job): + """ + Check if the current_job_value is included in the filter_from and retrieve filter_to value + :param relationships: Remaining filters to apply. + :param current_job: Current job to check. + :return: filters_to_apply + """ filters_to_apply = JobList._check_relationship(relationships, "DATES_FROM", date2str(current_job.date)) # there could be multiple filters that apply... per example # Current task date is 20020201, and member is fc2 @@ -498,6 +504,12 @@ class JobList(object): @staticmethod def _check_members(relationships, current_job): + """ + Check if the current_job_value is included in the filter_from and retrieve filter_to value + :param relationships: Remaining filters to apply. + :param current_job: Current job to check. + :return: filters_to_apply + """ filters_to_apply = JobList._check_relationship(relationships, "MEMBERS_FROM", current_job.member) for i,filter in enumerate(filters_to_apply): optional = filter.pop("OPTIONAL", False) @@ -514,6 +526,12 @@ class JobList(object): @staticmethod def _check_chunks(relationships, current_job): + """ + Check if the current_job_value is included in the filter_from and retrieve filter_to value + :param relationships: Remaining filters to apply. + :param current_job: Current job to check. + :return: filters_to_apply + """ filters_to_apply = JobList._check_relationship(relationships, "CHUNKS_FROM", current_job.chunk) for i,filter in enumerate(filters_to_apply): optional = filter.pop("OPTIONAL", False) @@ -526,6 +544,12 @@ class JobList(object): @staticmethod def _check_splits(relationships, current_job): + """ + Check if the current_job_value is included in the filter_from and retrieve filter_to value + :param relationships: Remaining filters to apply. + :param current_job: Current job to check. + :return: filters_to_apply + """ filters_to_apply = JobList._check_relationship(relationships, "SPLITS_FROM", current_job.split) # No more FROM sections to check, unify _to FILTERS and return filters_to_apply = JobList._unify_to_filters(filters_to_apply) @@ -533,6 +557,13 @@ class JobList(object): @staticmethod def _unify_to_filter(unified_filter,filter_to,filter_type): + """ + Unify filter_to filters into a single dictionary + :param unified_filter: Single dictionary with all filters_to + :param filter_to: Current dictionary that contains the filters_to + :param filter_type: "DATES_TO", "MEMBERS_TO", "CHUNKS_TO", "SPLITS_TO" + :return: unified_filter + """ if "all" not in unified_filter[filter_type]: aux = filter_to.pop(filter_type, None) if aux: @@ -546,6 +577,12 @@ class JobList(object): unified_filter[filter_type].add(element) @staticmethod def _normalize_to_filters(filter_to,filter_type): + """ + Normalize filter_to filters to a single string or "all" + :param filter_to: Unified filter_to dictionary + :param filter_type: "DATES_TO", "MEMBERS_TO", "CHUNKS_TO", "SPLITS_TO" + :return: + """ if len(filter_to[filter_type]) == 0: filter_to.pop(filter_type, None) elif "all" in filter_to[filter_type]: @@ -556,6 +593,11 @@ class JobList(object): @staticmethod def _unify_to_filters(filter_to_apply): + """ + Unify all filter_to filters into a single dictionary ( of current selection ) + :param filter_to_apply: Filters to apply + :return: Single dictionary with all filters_to + """ unified_filter = {"DATES_TO": set(), "MEMBERS_TO": set(), "CHUNKS_TO": set(), "SPLITS_TO": set()} for filter_to in filter_to_apply: if len(filter_to) > 0: @@ -575,10 +617,10 @@ class JobList(object): @staticmethod def _filter_current_job(current_job,relationships): ''' - Check if the current_job is included in the filter_value ( from) - :param current_job: current job to check - :param dependency: dependency to check - :return: filter_to_apply(dict), boolean + This function will filter the current job based on the relationships given + :param current_job: Current job to filter + :param relationships: Relationships to apply + :return: dict() with the filters to apply, or empty dict() if no filters to apply ''' # This function will look if the given relationship is set for the given job DATEs,MEMBER,CHUNK,SPLIT ( _from filters ) -- GitLab From f1eef4e82a5c4aa0b44df118ff8ccf5fce449954 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 26 May 2023 16:33:40 +0200 Subject: [PATCH 10/29] fixed typo --- docs/source/userguide/wrappers/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/userguide/wrappers/index.rst b/docs/source/userguide/wrappers/index.rst index 155d4d66a..168e5afa8 100644 --- a/docs/source/userguide/wrappers/index.rst +++ b/docs/source/userguide/wrappers/index.rst @@ -392,7 +392,7 @@ Considering the following configuration: "20120201": CHUNKS_FROM: 1: - DATES_TO: "º" + DATES_TO: "20120101" CHUNKS_TO: "1" RUNNING: chunk SYNCHRONIZE: member -- GitLab From 4624b027f1ba3104256b152b259634ca385216d4 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 26 May 2023 16:37:45 +0200 Subject: [PATCH 11/29] Reverting change --- autosubmit/job/job_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 5b5533f87..7fb0e0d4b 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -417,7 +417,7 @@ class JobList(object): else: to_filter.append(filter_value) - if str(to_filter).find(str(parent_value).upper()) != -1: + if str(parent_value).upper() in str(to_filter).upper(): return True else: return False -- GitLab From 3e4ba5cf654481f2b66e2a41f947c358531047fe Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 31 May 2023 17:03:56 +0200 Subject: [PATCH 12/29] added testcases fixed few things --- autosubmit/job/job_list.py | 76 ++++++++------- test/unit/test_dependencies.py | 173 +++++++++++++++++++++++++++++++++ 2 files changed, 214 insertions(+), 35 deletions(-) create mode 100644 test/unit/test_dependencies.py diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 7fb0e0d4b..3cccba1c4 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -434,7 +434,7 @@ class JobList(object): """ filters = [] for filter_range,filter_data in relationships.get(level_to_check,{}).items(): - if not str(value_to_check) or str(filter_range).upper() in "ALL" or str(value_to_check).upper() in str(filter_range).upper(): + if not value_to_check or str(filter_range).upper() in "ALL" or str(value_to_check).upper() in str(filter_range).upper(): if filter_data: if "?" in filter_range: filter_data["OPTIONAL"] = True @@ -454,6 +454,7 @@ class JobList(object): :param current_job: Current job to check. :return: filters_to_apply """ + optional = False filters_to_apply = JobList._check_relationship(relationships, "DATES_FROM", date2str(current_job.date)) # there could be multiple filters that apply... per example # Current task date is 20020201, and member is fc2 @@ -486,18 +487,21 @@ class JobList(object): if "MEMBERS_FROM" in filter: filters_to_apply_m = JobList._check_members({"MEMBERS_FROM": (filter.pop("MEMBERS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_m) > 0: - filters_to_apply[i] = filters_to_apply_m + filters_to_apply[i].update(filters_to_apply_m) # Will enter chunks_from, and obtain [{DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"] if "CHUNKS_FROM" in filter: filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_c) > 0 and len(filters_to_apply_c[0]) > 0: - filters_to_apply[i] = filters_to_apply_c + filters_to_apply[i].update(filters_to_apply_c) #IGNORED if "SPLITS_FROM" in filter: filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_s) > 0: - filters_to_apply[i] = filters_to_apply_s + filters_to_apply[i].update(filters_to_apply_s) # Unify filters from all filters_from where the current job is included to have a single SET of filters_to + if optional: + for i,filter in enumerate(filters_to_apply): + filters_to_apply[i]["OPTIONAL"] = True filters_to_apply = JobList._unify_to_filters(filters_to_apply) # {DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"} return filters_to_apply @@ -511,16 +515,21 @@ class JobList(object): :return: filters_to_apply """ filters_to_apply = JobList._check_relationship(relationships, "MEMBERS_FROM", current_job.member) + optional = False for i,filter in enumerate(filters_to_apply): optional = filter.pop("OPTIONAL", False) if "CHUNKS_FROM" in filter: filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_c) > 0: - filters_to_apply[i] = filters_to_apply_c + filters_to_apply[i].update(filters_to_apply_c) + if "SPLITS_FROM" in filter: filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_s) > 0: - filters_to_apply[i] = filters_to_apply_s + filters_to_apply[i].update(filters_to_apply_s) + if optional: + for i,filter in enumerate(filters_to_apply): + filters_to_apply[i]["OPTIONAL"] = True filters_to_apply = JobList._unify_to_filters(filters_to_apply) return filters_to_apply @@ -532,13 +541,17 @@ class JobList(object): :param current_job: Current job to check. :return: filters_to_apply """ + optional = False filters_to_apply = JobList._check_relationship(relationships, "CHUNKS_FROM", current_job.chunk) for i,filter in enumerate(filters_to_apply): optional = filter.pop("OPTIONAL", False) if "SPLITS_FROM" in filter: filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_s) > 0: - filters_to_apply[i] = filters_to_apply_s + filters_to_apply[i].update(filters_to_apply_s) + if optional: + for i,filter in enumerate(filters_to_apply): + filters_to_apply[i]["OPTIONAL"] = True filters_to_apply = JobList._unify_to_filters(filters_to_apply) return filters_to_apply @@ -550,6 +563,7 @@ class JobList(object): :param current_job: Current job to check. :return: filters_to_apply """ + filters_to_apply = JobList._check_relationship(relationships, "SPLITS_FROM", current_job.split) # No more FROM sections to check, unify _to FILTERS and return filters_to_apply = JobList._unify_to_filters(filters_to_apply) @@ -638,12 +652,11 @@ class JobList(object): # 3. NATURAL: this is the normal behavior, represents a way of letting the job to be activated if they would normally be activated. # 4. ? : this is a weak dependency activation flag, The dependency will be activated but the job can fail without affecting the workflow. - filters_to_apply = [{}] + filters_to_apply = {} # Check if filter_from-filter_to relationship is set - relationships["OPTIONAL"] = False - if current_job.section.lower() == "opa": - print("debugging") if relationships is not None and len(relationships) > 0: + if "OPTIONAL" not in relationships: + relationships["OPTIONAL"] = False # Look for a starting point, this can be if else becasue they're exclusive as a DATE_FROM can't be in a MEMBER_FROM and so on if "DATES_FROM" in relationships: filters_to_apply = JobList._check_dates(relationships, current_job) @@ -658,7 +671,7 @@ class JobList(object): relationships.pop("MEMBERS_FROM", None) relationships.pop("DATES_FROM", None) relationships.pop("SPLITS_FROM", None) - filters_to_apply = [relationships] + filters_to_apply = relationships return filters_to_apply @@ -676,8 +689,14 @@ class JobList(object): :return: True if the parent is valid, False otherwise ''' #check if current_parent is listed on dependency.relationships - optional = False associative_list = {} + associative_list["dates"] = date_list + associative_list["members"] = member_list + associative_list["chunks"] = chunk_list + if parent.splits is not None: + associative_list["splits"] = [ str(split) for split in range(1,int(parent.splits)+1) ] + else: + associative_list["splits"] = None dates_to = str(filter_.get("DATES_TO", "natural")).lower() members_to = str(filter_.get("MEMBERS_TO", "natural")).lower() chunks_to = str(filter_.get("CHUNKS_TO", "natural")).lower() @@ -691,14 +710,6 @@ class JobList(object): chunks_to = "none" if splits_to == "natural": splits_to = "none" - associative_list["dates"] = date_list - associative_list["members"] = member_list - associative_list["chunks"] = chunk_list - if parent.splits is not None: - associative_list["splits"] = [ str(split) for split in range(1,int(parent.splits)+1) ] - else: - associative_list["splits"] = None - if dates_to == "natural": associative_list["dates"] = [date2str(parent.date)] if parent.date is not None else date_list if members_to == "natural": @@ -710,14 +721,12 @@ class JobList(object): parsed_parent_date = date2str(parent.date) if parent.date is not None else None # Apply all filters to look if this parent is an appropriated candidate for the current_job valid_dates = JobList._apply_filter(parsed_parent_date, dates_to, associative_list["dates"], "dates") - valid_members = JobList._apply_filter(parent.member, members_to, associative_list["members"],"members") + valid_members = JobList._apply_filter(parent.member, members_to, associative_list["members"], "members") valid_chunks = JobList._apply_filter(parent.chunk, chunks_to, associative_list["chunks"], "chunks") valid_splits = JobList._apply_filter(parent.split, splits_to, associative_list["splits"], "splits") if valid_dates and valid_members and valid_chunks and valid_splits: - if dates_to.find("?") != -1 or members_to.find("?") != -1 or chunks_to.find("?") != -1 or splits_to.find("?") != -1: - optional = True - return True,optional - return False,optional + return True,( "?" in [dates_to,members_to,chunks_to,splits_to] ) + return False,False @staticmethod def _manage_job_dependencies(dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys, dependencies, graph): @@ -765,16 +774,13 @@ class JobList(object): natural_relationship = False # Check if the current parent is a valid parent based on the dependencies set on expdef.conf valid,optional = JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship,filters_to_apply) - if not valid: - continue - else: - pass # If the parent is valid, add it to the graph - job.add_parent(parent) - JobList._add_edge(graph, job, parent) - # Could be more variables in the future - if optional: - job.add_edge_info(parent.name,special_variables={"optional":True}) + if valid: + job.add_parent(parent) + JobList._add_edge(graph, job, parent) + # Could be more variables in the future + if optional: + job.add_edge_info(parent.name,special_variables={"optional":True}) JobList.handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member, member_list, dependency.section, graph, other_parents) diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py new file mode 100644 index 000000000..64ddb8f6b --- /dev/null +++ b/test/unit/test_dependencies.py @@ -0,0 +1,173 @@ +import unittest + +import mock +from copy import deepcopy +from autosubmit.job.job_list import JobList +from autosubmit.job.job import Job +from autosubmit.job.job_common import Status +from datetime import datetime +class TestJobList(unittest.TestCase): + def setUp(self): + # Define common test case inputs here + self.relationships_dates = { + "OPTIONAL": False, + "DATES_FROM": { + "20020201": { + "MEMBERS_FROM": { + "fc2": { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL" + } + }, + "SPLITS_FROM": { + "ALL": { + "SPLITS_TO": "1" + } + } + } + } + } + self.relationships_dates_optional = deepcopy(self.relationships_dates) + self.relationships_dates_optional["DATES_FROM"]["20020201"]["MEMBERS_FROM"] = { "fc2?": { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", "CHUNKS_TO": "ALL", "SPLITS_TO": "5" } } + self.relationships_dates_optional["DATES_FROM"]["20020201"]["SPLITS_FROM"] = { "ALL": { "SPLITS_TO": "1?" } } + + self.relationships_members = { + "OPTIONAL": False, + "MEMBERS_FROM": { + "fc2": { + "SPLITS_FROM": { + "ALL": { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + } + } + } + } + self.relationships_chunks = { + "OPTIONAL": False, + "CHUNKS_FROM": { + "1": { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + } + } + self.relationships_splits = { + "OPTIONAL": False, + "SPLITS_FROM": { + "1": { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + } + } + + # Create a mock Job object + self.mock_job = mock.MagicMock(spec=Job) + + # Set the attributes on the mock object + self.mock_job.name = "Job1" + self.mock_job.job_id = 1 + self.mock_job.status = Status.READY + self.mock_job.priority = 1 + self.mock_job.date = None + self.mock_job.member = None + self.mock_job.chunk = None + self.mock_job.split = None + def test_simple_dependency(self): + result_d = JobList._check_dates({}, self.mock_job) + result_m = JobList._check_members({}, self.mock_job) + result_c = JobList._check_chunks({}, self.mock_job) + result_s = JobList._check_splits({}, self.mock_job) + self.assertEqual(result_d, {}) + self.assertEqual(result_m, {}) + self.assertEqual(result_c, {}) + self.assertEqual(result_s, {}) + def test_check_dates_optional(self): + self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") + self.mock_job.member = "fc2" + self.mock_job.chunk = 1 + self.mock_job.split = 1 + result = JobList._check_dates(self.relationships_dates_optional, self.mock_job) + expected_output = { + "DATES_TO": "20020201?", + "MEMBERS_TO": "fc2?", + "CHUNKS_TO": "ALL?", + "SPLITS_TO": "1?" + } + self.assertEqual(result, expected_output) + def test_check_dates(self): + # Call the function to get the result + self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") + self.mock_job.member = "fc2" + self.mock_job.chunk = 1 + self.mock_job.split = 1 + result = JobList._check_dates(self.relationships_dates, self.mock_job) + expected_output = { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + self.assertEqual(result, expected_output) + self.mock_job.date = datetime.strptime("20020202", "%Y%m%d") + result = JobList._check_dates(self.relationships_dates, self.mock_job) + self.assertEqual(result, {}) + def test_check_members(self): + # Call the function to get the result + self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") + self.mock_job.member = "fc2" + + result = JobList._check_members(self.relationships_members, self.mock_job) + expected_output = { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + self.assertEqual(result, expected_output) + self.mock_job.member = "fc3" + result = JobList._check_members(self.relationships_members, self.mock_job) + self.assertEqual(result, {}) + + def test_check_splits(self): + # Call the function to get the result + + self.mock_job.split = 1 + result = JobList._check_splits(self.relationships_splits, self.mock_job) + expected_output = { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + self.assertEqual(result, expected_output) + self.mock_job.split = 2 + result = JobList._check_splits(self.relationships_splits, self.mock_job) + self.assertEqual(result, {}) + def test_check_chunks(self): + # Call the function to get the result + + self.mock_job.chunk = 1 + + result = JobList._check_chunks(self.relationships_chunks, self.mock_job) + expected_output = { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + self.assertEqual(result, expected_output) + self.mock_job.chunk = 2 + result = JobList._check_chunks(self.relationships_chunks, self.mock_job) + self.assertEqual(result, {}) +if __name__ == '__main__': + unittest.main() -- GitLab From 9867f96829a96d407bb092336f27e61c92602c56 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 3 May 2023 11:20:32 +0200 Subject: [PATCH 13/29] splits --- autosubmit/job/job_list.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 380431d52..257cc4783 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -506,14 +506,14 @@ class JobList(object): @staticmethod def _valid_parent(parent,member_list,date_list,chunk_list,is_a_natural_relation,filters_to_apply): ''' - Check if the parent is valid for the current_job + Check if the parent is valid for the current job :param parent: job to check - :param member_list: - :param date_list: - :param chunk_list: - :param is_a_natural_relation: - :param filters_to_apply: - :return: + :param member_list: list of members + :param date_list: list of dates + :param chunk_list: list of chunks + :param is_a_natural_relation: if the relation is natural or not + :param filters_to_apply: filters to apply + :return: True if the parent is valid, False otherwise ''' #check if current_parent is listed on dependency.relationships optional = False @@ -522,6 +522,7 @@ class JobList(object): dates_to = str(filter_.get("DATES_TO", "natural")).lower() members_to = str(filter_.get("MEMBERS_TO", "natural")).lower() chunks_to = str(filter_.get("CHUNKS_TO", "natural")).lower() + splits_to = str(filter_.get("SPLITS_TO", "natural")).lower() if not is_a_natural_relation: if dates_to == "natural": dates_to = "none" -- GitLab From de2c58de5c16e1f097c6421cb2c7c25a3182eb78 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 10 May 2023 18:13:36 +0200 Subject: [PATCH 14/29] splits working --- autosubmit/job/job.py | 2 ++ autosubmit/job/job_dict.py | 3 +- autosubmit/job/job_list.py | 58 +++++++++++++++++++++----------------- 3 files changed, 36 insertions(+), 27 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index ab754734e..faeb411eb 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -135,6 +135,7 @@ class Job(object): return "{0} STATUS: {1}".format(self.name, self.status) def __init__(self, name, job_id, status, priority): + self.splits = None self.script_name_wrapper = None self.delay_end = datetime.datetime.now() self._delay_retrials = "0" @@ -1333,6 +1334,7 @@ class Job(object): parameters['SDATE'] = self.sdate parameters['MEMBER'] = self.member parameters['SPLIT'] = self.split + parameters['SPLITS'] = self.splits parameters['DELAY'] = self.delay parameters['FREQUENCY'] = self.frequency parameters['SYNCHRONIZE'] = self.synchronize diff --git a/autosubmit/job/job_dict.py b/autosubmit/job/job_dict.py index c6b8e9763..e5be47eb0 100644 --- a/autosubmit/job/job_dict.py +++ b/autosubmit/job/job_dict.py @@ -287,7 +287,7 @@ class DicJobs: else: for d in self._date_list: self._get_date(jobs, dic, d, member, chunk) - if len(jobs) > 1 and isinstance(jobs[0], Iterable): + if isinstance(jobs[0], Iterable): try: jobs_flattened = [job for jobs_to_flatten in jobs for job in jobs_to_flatten] jobs = jobs_flattened @@ -355,6 +355,7 @@ class DicJobs: job.date = date job.member = member job.chunk = chunk + job.splits = self.experiment_data["JOBS"].get(job.section,{}).get("SPLITS", None) job.date_format = self._date_format job.delete_when_edgeless = str(parameters[section].get("DELETE_WHEN_EDGELESS", "true")).lower() diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 257cc4783..97e355c09 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -427,8 +427,8 @@ class JobList(object): """ Check if the current_job_value is included in the filter_value :param relationship: current filter level to check. - :param level_to_check: can be a date, member or chunk. - :param value_to_check: can be a date, member or chunk. + :param level_to_check: can be a date, member, chunk or split. + :param value_to_check: can be a date, member, chunk or split. :return: """ optional = False @@ -457,11 +457,11 @@ class JobList(object): :return: ''' # Search all filters in dependency relationship that affect current_job - # First level can be Date,member or chunk or generic - # Second level can be member or chunk or generic - # Third level can only be chunked. + # First level can be Date,member or chunk or splits generic + # Second level can be member or chunk or splits generic + # Third level can only be chunk or splits. # If the filter is generic, it will be applied to all section jobs. - # Check Date then Member or Chunk then Chunk + # Check Date then Member or Chunk or split then Chunk or split then split optional = False filters_to_apply = [] # this should be a list @@ -479,6 +479,7 @@ class JobList(object): filters_to_apply = filters_to_apply_c elif "CHUNKS_FROM" in filters_to_apply[filter_number]: filters_to_apply,optional = JobList._check_relationship(filters_to_apply[filter_number],"CHUNKS_FROM",current_job.chunk) + # Check Member then Chunk if len(filters_to_apply[0]) == 0: filters_to_apply,optional = JobList._check_relationship(relationships,"MEMBERS_FROM",current_job.member) @@ -486,14 +487,22 @@ class JobList(object): filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply,"CHUNKS_FROM",current_job.chunk) if len(filters_to_apply_c) > 0: filters_to_apply = filters_to_apply_c - #Check Chunk + #Check Chunk then splits if len(filters_to_apply[0]) == 0: filters_to_apply,optional = JobList._check_relationship(relationships,"CHUNKS_FROM",current_job.chunk) + if len(filters_to_apply) > 0 and ( "SPLITS_FROM" in filters_to_apply): + filters_to_apply_s,optional = JobList._check_relationship(filters_to_apply,"SPLITS_FROM",current_job.split) + if len(filters_to_apply_s) > 0: + filters_to_apply = filters_to_apply_s + # Check Splits + if len(filters_to_apply[0]) == 0: + filters_to_apply,optional = JobList._check_relationship(relationships,"SPLITS_FROM",current_job.split) # Generic filter if len(filters_to_apply[0]) == 0: relationships.pop("CHUNKS_FROM",None) relationships.pop("MEMBERS_FROM",None) relationships.pop("DATES_FROM",None) + relationships.pop("SPLITS_FROM",None) filters_to_apply = [relationships] if len(filters_to_apply) == 1 and len(filters_to_apply[0]) == 0: @@ -530,27 +539,34 @@ class JobList(object): members_to = "none" if chunks_to == "natural": chunks_to = "none" + if splits_to == "natural": + splits_to = "none" associative_list["dates"] = date_list associative_list["members"] = member_list associative_list["chunks"] = chunk_list + if parent.splits is not None: + associative_list["splits"] = [ str(split) for split in range(1,int(parent.splits)+1) ] + else: + associative_list["splits"] = None if dates_to == "natural": associative_list["dates"] = [date2str(parent.date)] if parent.date is not None else date_list - if members_to == "natural": associative_list["members"] = [parent.member] if parent.member is not None else member_list if chunks_to == "natural": associative_list["chunks"] = [parent.chunk] if parent.chunk is not None else chunk_list + if splits_to == "natural": + associative_list["splits"] = [parent.split] parsed_parent_date = date2str(parent.date) if parent.date is not None else None # Apply all filters to look if this parent is an appropriated candidate for the current_job valid_dates = JobList._apply_filter(parsed_parent_date, dates_to, associative_list["dates"], "dates") valid_members = JobList._apply_filter(parent.member, members_to, associative_list["members"],"members") valid_chunks = JobList._apply_filter(parent.chunk, chunks_to, associative_list["chunks"], "chunks") - - if valid_dates and valid_members and valid_chunks: - if dates_to.find("?") != -1 or members_to.find("?") != -1 or chunks_to.find("?") != -1: + valid_splits = JobList._apply_filter(parent.split, splits_to, associative_list["splits"], "splits") + if valid_dates and valid_members and valid_chunks and valid_splits: + if dates_to.find("?") != -1 or members_to.find("?") != -1 or chunks_to.find("?") != -1 or splits_to.find("?") != -1: optional = True return True,optional return False,optional @@ -589,7 +605,9 @@ class JobList(object): #if dependency.sign in ["+", "-"]: # natural_jobs = dic_jobs.get_jobs(dependency.section, date, member,chunk) #else: - natural_jobs = dic_jobs.get_jobs(dependency.section, date, member,chunk) + natural_jobs = dic_jobs.get_jobs(dependency.section, date, member, chunk) + if job.split is not None: + natural_jobs = [p_job for p_job in natural_jobs if p_job.split == job.split] if dependency.sign in ['?']: optional_section = True else: @@ -617,28 +635,16 @@ class JobList(object): # Get dates_to, members_to, chunks_to of the deepest level of the relationship. filters_to_apply,optional_from = JobList._filter_current_job(job,copy.deepcopy(dependency.relationships)) if len(filters_to_apply) == 0: - filters_to_apply.append({"DATES_TO": "natural", "MEMBERS_TO": "natural", "CHUNKS_TO": "natural"}) + filters_to_apply.append({"DATES_TO": "natural", "MEMBERS_TO": "natural", "CHUNKS_TO": "natural", "SPLITS_TO": "natural"}) for parent in all_parents: - # Generic for all dependencies - if dependency.delay == -1 or chunk > dependency.delay: - if isinstance(parent, list): - if job.split is not None and len(str(job.split)) > 0: - parent = list(filter( - lambda _parent: _parent.split == job.split, parent)) - parent = parent[0] - else: - if dependency.splits is not None and len(str(dependency.splits)) > 0: - parent = [_parent for _parent in parent if _parent.split in dependency.splits] # If splits is not None, the job is a list of jobs if parent.name == job.name: continue # Check if it is a natural relation based in autosubmit terms ( same date,member,chunk ). - if parent in natural_jobs and ((job.chunk is None or parent.chunk is None or parent.chunk <= job.chunk ) and (parent.split is None or job.split is None or parent.split <= job.split) ) : + if parent in natural_jobs and ((job.chunk is None or parent.chunk is None or parent.chunk <= job.chunk ) and (parent.split is None or job.split is None or parent.split != job.split) ) : natural_relationship = True else: natural_relationship = False - if job.name.find("a002_20120101_0_2_SIM") != -1: - print("test") # Check if the current parent is a valid parent based on the dependencies set on expdef.conf valid,optional_to = JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship,filters_to_apply) if not valid: -- GitLab From 70d05395f74feedc02637646f6a663abc898c359 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 11 May 2023 09:44:07 +0200 Subject: [PATCH 15/29] Normalized dependencies, splits bugs fixed. --- autosubmit/job/job.py | 14 -------------- autosubmit/job/job_list.py | 39 +++++++++----------------------------- 2 files changed, 9 insertions(+), 44 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index faeb411eb..601a2d581 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1406,20 +1406,6 @@ class Job(object): parameters['CHUNK_LAST'] = 'FALSE' parameters['NUMMEMBERS'] = len(as_conf.get_member_list()) parameters['DEPENDENCIES'] = str(as_conf.jobs_data[self.section].get("DEPENDENCIES","")) - # This shouldn't be necessary anymore as now all sub is done in the as_conf.reload() - # if len(self.export) > 0: - # variables = re.findall('%(? 0: - # variables = [variable[1:-1] for variable in variables] - # for key in variables: - # try: - # self.export = re.sub( - # '%(? 0: - aux = [] - for p_split in parents_jobs: - if type(p_split) is not list: - aux.append(p_split) - else: - for aux_job in p_split: - aux.append(aux_job) - parents_jobs = aux - if len(natural_jobs) > 0: - aux = [] - for p_split in natural_jobs: - if type(p_split) is not list: - aux.append(p_split) - else: - for aux_job in p_split: - aux.append(aux_job) - natural_jobs = aux all_parents = list(set(other_parents + parents_jobs)) # Get dates_to, members_to, chunks_to of the deepest level of the relationship. filters_to_apply,optional_from = JobList._filter_current_job(job,copy.deepcopy(dependency.relationships)) @@ -640,8 +619,8 @@ class JobList(object): # If splits is not None, the job is a list of jobs if parent.name == job.name: continue - # Check if it is a natural relation based in autosubmit terms ( same date,member,chunk ). - if parent in natural_jobs and ((job.chunk is None or parent.chunk is None or parent.chunk <= job.chunk ) and (parent.split is None or job.split is None or parent.split != job.split) ) : + # Check if it is a natural relation. The only difference is that a chunk can depend on a chunks <= than the current chunk + if parent in natural_jobs and ((job.chunk is None or parent.chunk is None or parent.chunk <= job.chunk )): natural_relationship = True else: natural_relationship = False -- GitLab From 9b6dab915dd245edda5c37f58806c6520b3361ac Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 11 May 2023 14:55:04 +0200 Subject: [PATCH 16/29] Normalized dependencies, splits bugs fixed. --- autosubmit/job/job_list.py | 89 +++++++++++++++++++++++++++----------- 1 file changed, 63 insertions(+), 26 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 3a55d84fe..2d8554503 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -439,7 +439,7 @@ class JobList(object): current_filter = {} if filter_type.upper().find(level_to_check.upper()) != -1: for filter_range in filter_data.keys(): - if str(value_to_check) is None or str(filter_range).upper().find(str(value_to_check).upper()) != -1: + if str(filter_range).upper() == "ALL" or str(value_to_check) is None or str(filter_range).upper().find(str(value_to_check).upper()) != -1: if filter_data[filter_range] is not None: if filter_type.find("?") != -1 or filter_range.find("?") != -1: optional = True @@ -449,45 +449,82 @@ class JobList(object): if len(filters) == 0: filters = [{}] return filters,optional + @staticmethod + def _filter_dates(current_job,relationships): + @staticmethod def _filter_current_job(current_job,relationships): ''' Check if the current_job is included in the filter_value ( from) - :param current_job: - :param dependency: - :return: + :param current_job: current job to check + :param dependency: dependency to check + :return: filter_to_apply(dict), boolean ''' - # Search all filters in dependency relationship that affect current_job - # First level can be Date,member or chunk or splits generic - # Second level can be member or chunk or splits generic - # Third level can only be chunk or splits. - # If the filter is generic, it will be applied to all section jobs. - # Check Date then Member or Chunk or split then Chunk or split then split + + # This function will look if the given relationship is set for the given job DATEs,MEMBER,CHUNK,SPLIT ( _from filters ) + # And if it is, it will return the dependencies that need to be activated (_TO filters) + # _FROM behavior: + # DATES_FROM can contain MEMBERS_FROM,CHUNKS_FROM,SPLITS_FROM + # MEMBERS_FROM can contain CHUNKS_FROM,SPLITS_FROM + # CHUNKS_FROM can contain SPLITS_FROM + # SPLITS_FROM can contain nothing + # _TO behavior: + # TO keywords, can be in any of the _FROM filters and they will only affect the _FROM filter they are in. + # There are 4 keywords: + # 1. ALL: all the dependencies will be activated of the given filter type (dates, members, chunks or/and splits) + # 2. NONE: no dependencies will be activated of the given filter type (dates, members, chunks or/and splits) + # 3. NATURAL: this is the normal behavior, represents a way of letting the job to be activated if they would normally be activated. + # 4. ? : this is a weak dependency activation flag, The dependency will be activated but the job can fail without affecting the workflow. + optional = False - filters_to_apply = [] - # this should be a list + filters_to_apply = [{}] + # Check if filter_from-filter_to relationship is set if relationships is not None and len(relationships) > 0: + # CHECK IF DATES FILTERS IS SET filters_to_apply,optional = JobList._check_relationship(relationships,"DATES_FROM",date2str(current_job.date)) + # IF DATES FILTERS IS SET if len(filters_to_apply[0]) > 0: - for filter_number in range(0, len(filters_to_apply)): + for filter_number in range(len(filters_to_apply)): + # CHECK IF MEMBERS FILTERS IS SET if "MEMBERS_FROM" in filters_to_apply[filter_number]: filters_to_apply_m,optional = JobList._check_relationship(filters_to_apply[filter_number],"MEMBERS_FROM",current_job.member) - if len(filters_to_apply_m) > 0: - filters_to_apply,optional = filters_to_apply_m - else: + # IF MEMBERS FILTERS IS SET + if len(filters_to_apply_m[filter_number]) > 0: + filters_to_apply = filters_to_apply_m + if "CHUNKS_FROM" in filters_to_apply[filter_number]: filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply[filter_number],"CHUNKS_FROM",current_job.chunk) - if len(filters_to_apply_c) > 0: + if len(filters_to_apply_c[filter_number]) > 0: filters_to_apply = filters_to_apply_c + if "SPLITS_FROM" in filters_to_apply[filter_number]: + filters_to_apply_s,optional = JobList._check_relationship(filters_to_apply[filter_number],"SPLITS_FROM",current_job.split) + if len(filters_to_apply_s[filter_number]) > 0: + filters_to_apply = filters_to_apply_s + # CHECK IF CHUNKS FILTERS IS SET (MEMBERS FILTERS IS NOT SET) elif "CHUNKS_FROM" in filters_to_apply[filter_number]: - filters_to_apply,optional = JobList._check_relationship(filters_to_apply[filter_number],"CHUNKS_FROM",current_job.chunk) - - # Check Member then Chunk + filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply[filter_number],"CHUNKS_FROM",current_job.chunk) + if len(filters_to_apply[filter_number]) > 0: + filters_to_apply = filters_to_apply_c + if "SPLITS_FROM" in filters_to_apply[filter_number]: + filters_to_apply_s,optional = JobList._check_relationship(filters_to_apply[filter_number],"SPLITS_FROM",current_job.split) + if len(filters_to_apply_s[filter_number]) > 0: + filters_to_apply = filters_to_apply_s + # CHECK IF SPLITS FILTERS IS SET (MEMBERS FILTERS IS NOT SET) + elif "SPLITS_FROM" in filters_to_apply[filter_number]: + filters_to_apply,optional = JobList._check_relationship(filters_to_apply[filter_number],"SPLITS_FROM",current_job.split) + # IF DATES FILTERS IS NOT SET, check if MEMBERS FILTERS IS SET if len(filters_to_apply[0]) == 0: - filters_to_apply,optional = JobList._check_relationship(relationships,"MEMBERS_FROM",current_job.member) - if len(filters_to_apply) > 0 and ( "CHUNKS_FROM" in filters_to_apply): - filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply,"CHUNKS_FROM",current_job.chunk) - if len(filters_to_apply_c) > 0: - filters_to_apply = filters_to_apply_c + for filter_number in range(len(filters_to_apply)): + filters_to_apply_m,optional = JobList._check_relationship(relationships,"MEMBERS_FROM",current_job.member) + if len(filters_to_apply_m[filter_number]) > 0: + filters_to_apply = filters_to_apply_m + if "CHUNKS_FROM" in filters_to_apply: + filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply,"CHUNKS_FROM",current_job.chunk) + if len(filters_to_apply_c) > 0: + filters_to_apply = filters_to_apply_c + if "SPLITS_FROM" in filters_to_apply: + filters_to_apply_s,optional = JobList._check_relationship(filters_to_apply,"SPLITS_FROM",current_job.split) + if len(filters_to_apply_s) > 0: + filters_to_apply = filters_to_apply_s #Check Chunk then splits if len(filters_to_apply[0]) == 0: filters_to_apply,optional = JobList._check_relationship(relationships,"CHUNKS_FROM",current_job.chunk) @@ -498,7 +535,7 @@ class JobList(object): # Check Splits if len(filters_to_apply[0]) == 0: filters_to_apply,optional = JobList._check_relationship(relationships,"SPLITS_FROM",current_job.split) - # Generic filter + # Global filter if len(filters_to_apply[0]) == 0: relationships.pop("CHUNKS_FROM",None) relationships.pop("MEMBERS_FROM",None) -- GitLab From 1d624061d9ddd3d74bec4e60a9376855a3aae243 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 15 May 2023 15:01:57 +0200 Subject: [PATCH 17/29] All working now, todo pipeline tests --- autosubmit/job/job_list.py | 107 ++++++++++++++++++----- docs/source/userguide/wrappers/index.rst | 2 +- 2 files changed, 87 insertions(+), 22 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 2d8554503..facbc05db 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -449,8 +449,59 @@ class JobList(object): if len(filters) == 0: filters = [{}] return filters,optional + + @staticmethod + def _check_dates(relationships, current_job): + filters_to_apply, optional = JobList._check_relationship(relationships, "DATES_FROM", date2str(current_job.date)) + for filter in filters_to_apply: + if "MEMBERS_FROM" in filter: + filters_to_apply_m, optional_m = JobList._check_members(filter, current_job) + if len(filters_to_apply_m) > 0: + filters_to_apply = filters_to_apply_m + optional = optional_m + if "CHUNKS_FROM" in filter: + filters_to_apply_c, optional_c = JobList._check_chunks(filter, current_job) + if len(filters_to_apply_c) > 0: + filters_to_apply = filters_to_apply_c + optional = optional_c + if "SPLITS_FROM" in filter: + filters_to_apply_s, optional_s = JobList._check_splits(filter, current_job) + if len(filters_to_apply_s) > 0: + filters_to_apply = filters_to_apply_s + optional = optional_s + return filters_to_apply, optional + @staticmethod - def _filter_dates(current_job,relationships): + def _check_members(relationships, current_job): + filters_to_apply, optional = JobList._check_relationship(relationships, "MEMBERS_FROM", current_job.member) + for filter in filters_to_apply: + if "CHUNKS_FROM" in filter: + filters_to_apply_c, optional_c = JobList._check_chunks(filter, current_job) + if len(filters_to_apply_c) > 0: + filters_to_apply = filters_to_apply_c + optional = optional_c + if "SPLITS_FROM" in filter: + filters_to_apply_s, optional_s = JobList._check_splits(filter, current_job) + if len(filters_to_apply_s) > 0: + filters_to_apply = filters_to_apply_s + optional = optional_s + return filters_to_apply, optional + + @staticmethod + def _check_chunks(relationships, current_job): + filters_to_apply, optional = JobList._check_relationship(relationships, "CHUNKS_FROM", current_job.chunk) + for filter in filters_to_apply: + if "SPLITS_FROM" in filter: + filters_to_apply_s, optional_s = JobList._check_splits(filter, current_job) + if len(filters_to_apply_s) > 0: + filters_to_apply = filters_to_apply_s + optional = optional_s + return filters_to_apply, optional + + @staticmethod + def _check_splits(relationships, current_job): + filters_to_apply, optional = JobList._check_relationship(relationships, "SPLITS_FROM", current_job.split) + return filters_to_apply, optional @staticmethod def _filter_current_job(current_job,relationships): @@ -481,7 +532,21 @@ class JobList(object): # Check if filter_from-filter_to relationship is set if relationships is not None and len(relationships) > 0: # CHECK IF DATES FILTERS IS SET - filters_to_apply,optional = JobList._check_relationship(relationships,"DATES_FROM",date2str(current_job.date)) + if "DATES_FROM" in relationships: + filters_to_apply, optional = JobList._check_dates(relationships, current_job) + elif "MEMBERS_FROM" in relationships: + filters_to_apply, optional = JobList._check_members(relationships, current_job) + elif "CHUNKS_FROM" in relationships: + filters_to_apply, optional = JobList._check_chunks(relationships, current_job) + elif "SPLITS_FROM" in relationships: + filters_to_apply, optional = JobList._check_splits(relationships, current_job) + else: + relationships.pop("CHUNKS_FROM", None) + relationships.pop("MEMBERS_FROM", None) + relationships.pop("DATES_FROM", None) + relationships.pop("SPLITS_FROM", None) + filters_to_apply = [relationships] + return filters_to_apply,optional # IF DATES FILTERS IS SET if len(filters_to_apply[0]) > 0: for filter_number in range(len(filters_to_apply)): @@ -489,16 +554,16 @@ class JobList(object): if "MEMBERS_FROM" in filters_to_apply[filter_number]: filters_to_apply_m,optional = JobList._check_relationship(filters_to_apply[filter_number],"MEMBERS_FROM",current_job.member) # IF MEMBERS FILTERS IS SET - if len(filters_to_apply_m[filter_number]) > 0: - filters_to_apply = filters_to_apply_m - if "CHUNKS_FROM" in filters_to_apply[filter_number]: - filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply[filter_number],"CHUNKS_FROM",current_job.chunk) - if len(filters_to_apply_c[filter_number]) > 0: - filters_to_apply = filters_to_apply_c - if "SPLITS_FROM" in filters_to_apply[filter_number]: - filters_to_apply_s,optional = JobList._check_relationship(filters_to_apply[filter_number],"SPLITS_FROM",current_job.split) - if len(filters_to_apply_s[filter_number]) > 0: - filters_to_apply = filters_to_apply_s + if len(filters_to_apply_m[filter_number]) > 0: + filters_to_apply = filters_to_apply_m + if "CHUNKS_FROM" in filters_to_apply[filter_number]: + filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply[filter_number],"CHUNKS_FROM",current_job.chunk) + if len(filters_to_apply_c[filter_number]) > 0: + filters_to_apply = filters_to_apply_c + if "SPLITS_FROM" in filters_to_apply[filter_number]: + filters_to_apply_s,optional = JobList._check_relationship(filters_to_apply[filter_number],"SPLITS_FROM",current_job.split) + if len(filters_to_apply_s[filter_number]) > 0: + filters_to_apply = filters_to_apply_s # CHECK IF CHUNKS FILTERS IS SET (MEMBERS FILTERS IS NOT SET) elif "CHUNKS_FROM" in filters_to_apply[filter_number]: filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply[filter_number],"CHUNKS_FROM",current_job.chunk) @@ -513,10 +578,8 @@ class JobList(object): filters_to_apply,optional = JobList._check_relationship(filters_to_apply[filter_number],"SPLITS_FROM",current_job.split) # IF DATES FILTERS IS NOT SET, check if MEMBERS FILTERS IS SET if len(filters_to_apply[0]) == 0: + filters_to_apply, optional = JobList._check_relationship(relationships, "MEMBERS_FROM", current_job.member) for filter_number in range(len(filters_to_apply)): - filters_to_apply_m,optional = JobList._check_relationship(relationships,"MEMBERS_FROM",current_job.member) - if len(filters_to_apply_m[filter_number]) > 0: - filters_to_apply = filters_to_apply_m if "CHUNKS_FROM" in filters_to_apply: filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply,"CHUNKS_FROM",current_job.chunk) if len(filters_to_apply_c) > 0: @@ -527,14 +590,16 @@ class JobList(object): filters_to_apply = filters_to_apply_s #Check Chunk then splits if len(filters_to_apply[0]) == 0: - filters_to_apply,optional = JobList._check_relationship(relationships,"CHUNKS_FROM",current_job.chunk) - if len(filters_to_apply) > 0 and ( "SPLITS_FROM" in filters_to_apply): - filters_to_apply_s,optional = JobList._check_relationship(filters_to_apply,"SPLITS_FROM",current_job.split) - if len(filters_to_apply_s) > 0: - filters_to_apply = filters_to_apply_s + filters_to_apply, optional = JobList._check_relationship(relationships, "CHUNKS_FROM", current_job.chunk) + for filter_number in range(len(filters_to_apply)): + if "SPLITS_FROM" in filters_to_apply[filter_number]: + filters_to_apply_s,optional = JobList._filter_splits(filters_to_apply, "SPLITS_FROM", current_job.split) + if len(filters_to_apply_s) > 0: + filters_to_apply = filters_to_apply_s # Check Splits if len(filters_to_apply[0]) == 0: - filters_to_apply,optional = JobList._check_relationship(relationships,"SPLITS_FROM",current_job.split) + filters_to_apply, optional = JobList._check_relationship(relationships, "SPLITS_FROM",current_job.split) + # Global filter if len(filters_to_apply[0]) == 0: relationships.pop("CHUNKS_FROM",None) diff --git a/docs/source/userguide/wrappers/index.rst b/docs/source/userguide/wrappers/index.rst index 168e5afa8..155d4d66a 100644 --- a/docs/source/userguide/wrappers/index.rst +++ b/docs/source/userguide/wrappers/index.rst @@ -392,7 +392,7 @@ Considering the following configuration: "20120201": CHUNKS_FROM: 1: - DATES_TO: "20120101" + DATES_TO: "º" CHUNKS_TO: "1" RUNNING: chunk SYNCHRONIZE: member -- GitLab From bf000b81d5d3396409488e119ebf4c734db46aed Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 25 May 2023 11:57:17 +0200 Subject: [PATCH 18/29] fixed tests --- autosubmit/job/job_dict.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit/job/job_dict.py b/autosubmit/job/job_dict.py index e5be47eb0..e2f673563 100644 --- a/autosubmit/job/job_dict.py +++ b/autosubmit/job/job_dict.py @@ -287,7 +287,7 @@ class DicJobs: else: for d in self._date_list: self._get_date(jobs, dic, d, member, chunk) - if isinstance(jobs[0], Iterable): + if len(jobs) > 0 and isinstance(jobs[0], list): try: jobs_flattened = [job for jobs_to_flatten in jobs for job in jobs_to_flatten] jobs = jobs_flattened -- GitLab From 391768acfb5498027f3b3f199e53ac07251ed20d Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 26 May 2023 15:53:40 +0200 Subject: [PATCH 19/29] Fix algorithm --- autosubmit/job/job_list.py | 301 ++++++++++++++++--------------------- 1 file changed, 133 insertions(+), 168 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index facbc05db..dcb2e3650 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -429,79 +429,118 @@ class JobList(object): Check if the current_job_value is included in the filter_value :param relationship: current filter level to check. :param level_to_check: can be a date, member, chunk or split. - :param value_to_check: can be a date, member, chunk or split. + :param value_to_check: Can be None, a date, a member, a chunk or a split. :return: """ - optional = False filters = [] - for filter_type, filter_data in relationships.items(): - if isinstance(filter_data, collections.abc.Mapping): - current_filter = {} - if filter_type.upper().find(level_to_check.upper()) != -1: - for filter_range in filter_data.keys(): - if str(filter_range).upper() == "ALL" or str(value_to_check) is None or str(filter_range).upper().find(str(value_to_check).upper()) != -1: - if filter_data[filter_range] is not None: - if filter_type.find("?") != -1 or filter_range.find("?") != -1: - optional = True - current_filter.update(filter_data[filter_range]) - filters.append(current_filter) + for filter_range,filter_data in relationships.get(level_to_check,{}).items(): + if not str(value_to_check) or str(filter_range).upper() in "ALL" or str(value_to_check).upper() in str(filter_range).upper(): + if filter_data: + if "?" in filter_range: + filter_data["OPTIONAL"] = True + else: + filter_data["OPTIONAL"] = relationships["OPTIONAL"] + filters.append(filter_data) # Normalize the filter return if len(filters) == 0: filters = [{}] - return filters,optional + return filters @staticmethod def _check_dates(relationships, current_job): - filters_to_apply, optional = JobList._check_relationship(relationships, "DATES_FROM", date2str(current_job.date)) - for filter in filters_to_apply: + filters_to_apply = JobList._check_relationship(relationships, "DATES_FROM", date2str(current_job.date)) + for i,filter in enumerate(filters_to_apply): + optional = filter.pop("OPTIONAL", False) if "MEMBERS_FROM" in filter: - filters_to_apply_m, optional_m = JobList._check_members(filter, current_job) + filters_to_apply_m = JobList._check_members({"MEMBERS_FROM": (filter.pop("MEMBERS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_m) > 0: - filters_to_apply = filters_to_apply_m - optional = optional_m + filters_to_apply[i] = filters_to_apply_m if "CHUNKS_FROM" in filter: - filters_to_apply_c, optional_c = JobList._check_chunks(filter, current_job) - if len(filters_to_apply_c) > 0: - filters_to_apply = filters_to_apply_c - optional = optional_c + filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM")),"OPTIONAL":optional}, current_job) + if len(filters_to_apply_c) > 0 and len(filters_to_apply_c[0]) > 0: + filters_to_apply[i] = filters_to_apply_c if "SPLITS_FROM" in filter: - filters_to_apply_s, optional_s = JobList._check_splits(filter, current_job) + filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_s) > 0: - filters_to_apply = filters_to_apply_s - optional = optional_s - return filters_to_apply, optional + filters_to_apply[i] = filters_to_apply_s + filters_to_apply = JobList._unify_to_filters(filters_to_apply) + return filters_to_apply @staticmethod def _check_members(relationships, current_job): - filters_to_apply, optional = JobList._check_relationship(relationships, "MEMBERS_FROM", current_job.member) - for filter in filters_to_apply: + filters_to_apply = JobList._check_relationship(relationships, "MEMBERS_FROM", current_job.member) + for i,filter in enumerate(filters_to_apply): + optional = filter.pop("OPTIONAL", False) if "CHUNKS_FROM" in filter: - filters_to_apply_c, optional_c = JobList._check_chunks(filter, current_job) + filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_c) > 0: - filters_to_apply = filters_to_apply_c - optional = optional_c + filters_to_apply[i] = filters_to_apply_c if "SPLITS_FROM" in filter: - filters_to_apply_s, optional_s = JobList._check_splits(filter, current_job) + filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_s) > 0: - filters_to_apply = filters_to_apply_s - optional = optional_s - return filters_to_apply, optional + filters_to_apply[i] = filters_to_apply_s + filters_to_apply = JobList._unify_to_filters(filters_to_apply) + return filters_to_apply @staticmethod def _check_chunks(relationships, current_job): - filters_to_apply, optional = JobList._check_relationship(relationships, "CHUNKS_FROM", current_job.chunk) - for filter in filters_to_apply: + filters_to_apply = JobList._check_relationship(relationships, "CHUNKS_FROM", current_job.chunk) + for i,filter in enumerate(filters_to_apply): + optional = filter.pop("OPTIONAL", False) if "SPLITS_FROM" in filter: - filters_to_apply_s, optional_s = JobList._check_splits(filter, current_job) + filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_s) > 0: - filters_to_apply = filters_to_apply_s - optional = optional_s - return filters_to_apply, optional + filters_to_apply[i] = filters_to_apply_s + filters_to_apply = JobList._unify_to_filters(filters_to_apply) + return filters_to_apply @staticmethod def _check_splits(relationships, current_job): - filters_to_apply, optional = JobList._check_relationship(relationships, "SPLITS_FROM", current_job.split) - return filters_to_apply, optional + filters_to_apply = JobList._check_relationship(relationships, "SPLITS_FROM", current_job.split) + # No more FROM sections to check, unify _to FILTERS and return + filters_to_apply = JobList._unify_to_filters(filters_to_apply) + return filters_to_apply + + @staticmethod + def _unify_to_filter(unified_filter,filter_to,filter_type): + if "all" not in unified_filter[filter_type]: + aux = filter_to.pop(filter_type, None) + if aux: + aux = aux.split(",") + for element in aux: + if element.lower().strip("?") in ["natural","none"] and len(unified_filter[filter_type]) > 0: + continue + else: + if filter_to.get("OPTIONAL",False) and element[-1] != "?": + element += "?" + unified_filter[filter_type].add(element) + @staticmethod + def _normalize_to_filters(filter_to,filter_type): + if len(filter_to[filter_type]) == 0: + filter_to.pop(filter_type, None) + elif "all" in filter_to[filter_type]: + filter_to[filter_type] = "all" + else: + # transform to str separated by commas if multiple elements + filter_to[filter_type] = ",".join(filter_to[filter_type]) + + @staticmethod + def _unify_to_filters(filter_to_apply): + unified_filter = {"DATES_TO": set(), "MEMBERS_TO": set(), "CHUNKS_TO": set(), "SPLITS_TO": set()} + for filter_to in filter_to_apply: + if len(filter_to) > 0: + JobList._unify_to_filter(unified_filter,filter_to,"DATES_TO") + JobList._unify_to_filter(unified_filter,filter_to,"MEMBERS_TO") + JobList._unify_to_filter(unified_filter,filter_to,"CHUNKS_TO") + JobList._unify_to_filter(unified_filter,filter_to,"SPLITS_TO") + filter_to.pop("OPTIONAL", None) + + JobList._normalize_to_filters(unified_filter,"DATES_TO") + JobList._normalize_to_filters(unified_filter,"MEMBERS_TO") + JobList._normalize_to_filters(unified_filter,"CHUNKS_TO") + JobList._normalize_to_filters(unified_filter,"SPLITS_TO") + + return unified_filter @staticmethod def _filter_current_job(current_job,relationships): @@ -527,96 +566,33 @@ class JobList(object): # 3. NATURAL: this is the normal behavior, represents a way of letting the job to be activated if they would normally be activated. # 4. ? : this is a weak dependency activation flag, The dependency will be activated but the job can fail without affecting the workflow. - optional = False filters_to_apply = [{}] # Check if filter_from-filter_to relationship is set + relationships["OPTIONAL"] = False + if current_job.section.lower() == "opa": + print("debugging") if relationships is not None and len(relationships) > 0: - # CHECK IF DATES FILTERS IS SET + # Look for a starting point if "DATES_FROM" in relationships: - filters_to_apply, optional = JobList._check_dates(relationships, current_job) + filters_to_apply = JobList._check_dates(relationships, current_job) elif "MEMBERS_FROM" in relationships: - filters_to_apply, optional = JobList._check_members(relationships, current_job) + filters_to_apply = JobList._check_members(relationships, current_job) elif "CHUNKS_FROM" in relationships: - filters_to_apply, optional = JobList._check_chunks(relationships, current_job) + filters_to_apply = JobList._check_chunks(relationships, current_job) elif "SPLITS_FROM" in relationships: - filters_to_apply, optional = JobList._check_splits(relationships, current_job) + filters_to_apply = JobList._check_splits(relationships, current_job) else: relationships.pop("CHUNKS_FROM", None) relationships.pop("MEMBERS_FROM", None) relationships.pop("DATES_FROM", None) relationships.pop("SPLITS_FROM", None) filters_to_apply = [relationships] - return filters_to_apply,optional - # IF DATES FILTERS IS SET - if len(filters_to_apply[0]) > 0: - for filter_number in range(len(filters_to_apply)): - # CHECK IF MEMBERS FILTERS IS SET - if "MEMBERS_FROM" in filters_to_apply[filter_number]: - filters_to_apply_m,optional = JobList._check_relationship(filters_to_apply[filter_number],"MEMBERS_FROM",current_job.member) - # IF MEMBERS FILTERS IS SET - if len(filters_to_apply_m[filter_number]) > 0: - filters_to_apply = filters_to_apply_m - if "CHUNKS_FROM" in filters_to_apply[filter_number]: - filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply[filter_number],"CHUNKS_FROM",current_job.chunk) - if len(filters_to_apply_c[filter_number]) > 0: - filters_to_apply = filters_to_apply_c - if "SPLITS_FROM" in filters_to_apply[filter_number]: - filters_to_apply_s,optional = JobList._check_relationship(filters_to_apply[filter_number],"SPLITS_FROM",current_job.split) - if len(filters_to_apply_s[filter_number]) > 0: - filters_to_apply = filters_to_apply_s - # CHECK IF CHUNKS FILTERS IS SET (MEMBERS FILTERS IS NOT SET) - elif "CHUNKS_FROM" in filters_to_apply[filter_number]: - filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply[filter_number],"CHUNKS_FROM",current_job.chunk) - if len(filters_to_apply[filter_number]) > 0: - filters_to_apply = filters_to_apply_c - if "SPLITS_FROM" in filters_to_apply[filter_number]: - filters_to_apply_s,optional = JobList._check_relationship(filters_to_apply[filter_number],"SPLITS_FROM",current_job.split) - if len(filters_to_apply_s[filter_number]) > 0: - filters_to_apply = filters_to_apply_s - # CHECK IF SPLITS FILTERS IS SET (MEMBERS FILTERS IS NOT SET) - elif "SPLITS_FROM" in filters_to_apply[filter_number]: - filters_to_apply,optional = JobList._check_relationship(filters_to_apply[filter_number],"SPLITS_FROM",current_job.split) - # IF DATES FILTERS IS NOT SET, check if MEMBERS FILTERS IS SET - if len(filters_to_apply[0]) == 0: - filters_to_apply, optional = JobList._check_relationship(relationships, "MEMBERS_FROM", current_job.member) - for filter_number in range(len(filters_to_apply)): - if "CHUNKS_FROM" in filters_to_apply: - filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply,"CHUNKS_FROM",current_job.chunk) - if len(filters_to_apply_c) > 0: - filters_to_apply = filters_to_apply_c - if "SPLITS_FROM" in filters_to_apply: - filters_to_apply_s,optional = JobList._check_relationship(filters_to_apply,"SPLITS_FROM",current_job.split) - if len(filters_to_apply_s) > 0: - filters_to_apply = filters_to_apply_s - #Check Chunk then splits - if len(filters_to_apply[0]) == 0: - filters_to_apply, optional = JobList._check_relationship(relationships, "CHUNKS_FROM", current_job.chunk) - for filter_number in range(len(filters_to_apply)): - if "SPLITS_FROM" in filters_to_apply[filter_number]: - filters_to_apply_s,optional = JobList._filter_splits(filters_to_apply, "SPLITS_FROM", current_job.split) - if len(filters_to_apply_s) > 0: - filters_to_apply = filters_to_apply_s - # Check Splits - if len(filters_to_apply[0]) == 0: - filters_to_apply, optional = JobList._check_relationship(relationships, "SPLITS_FROM",current_job.split) - - # Global filter - if len(filters_to_apply[0]) == 0: - relationships.pop("CHUNKS_FROM",None) - relationships.pop("MEMBERS_FROM",None) - relationships.pop("DATES_FROM",None) - relationships.pop("SPLITS_FROM",None) - filters_to_apply = [relationships] - - if len(filters_to_apply) == 1 and len(filters_to_apply[0]) == 0: - return [],optional - else: - return filters_to_apply,optional + return filters_to_apply @staticmethod - def _valid_parent(parent,member_list,date_list,chunk_list,is_a_natural_relation,filters_to_apply): + def _valid_parent(parent,member_list,date_list,chunk_list,is_a_natural_relation,filter_): ''' Check if the parent is valid for the current job :param parent: job to check @@ -629,49 +605,46 @@ class JobList(object): ''' #check if current_parent is listed on dependency.relationships optional = False - for filter_ in filters_to_apply: - associative_list = {} - dates_to = str(filter_.get("DATES_TO", "natural")).lower() - members_to = str(filter_.get("MEMBERS_TO", "natural")).lower() - chunks_to = str(filter_.get("CHUNKS_TO", "natural")).lower() - splits_to = str(filter_.get("SPLITS_TO", "natural")).lower() - if not is_a_natural_relation: - if dates_to == "natural": - dates_to = "none" - if members_to == "natural": - members_to = "none" - if chunks_to == "natural": - chunks_to = "none" - if splits_to == "natural": - splits_to = "none" - - - associative_list["dates"] = date_list - associative_list["members"] = member_list - associative_list["chunks"] = chunk_list - if parent.splits is not None: - associative_list["splits"] = [ str(split) for split in range(1,int(parent.splits)+1) ] - else: - associative_list["splits"] = None - + associative_list = {} + dates_to = str(filter_.get("DATES_TO", "natural")).lower() + members_to = str(filter_.get("MEMBERS_TO", "natural")).lower() + chunks_to = str(filter_.get("CHUNKS_TO", "natural")).lower() + splits_to = str(filter_.get("SPLITS_TO", "natural")).lower() + if not is_a_natural_relation: if dates_to == "natural": - associative_list["dates"] = [date2str(parent.date)] if parent.date is not None else date_list + dates_to = "none" if members_to == "natural": - associative_list["members"] = [parent.member] if parent.member is not None else member_list + members_to = "none" if chunks_to == "natural": - associative_list["chunks"] = [parent.chunk] if parent.chunk is not None else chunk_list + chunks_to = "none" if splits_to == "natural": - associative_list["splits"] = [parent.split] if parent.split is not None else parent.splits - parsed_parent_date = date2str(parent.date) if parent.date is not None else None - # Apply all filters to look if this parent is an appropriated candidate for the current_job - valid_dates = JobList._apply_filter(parsed_parent_date, dates_to, associative_list["dates"], "dates") - valid_members = JobList._apply_filter(parent.member, members_to, associative_list["members"],"members") - valid_chunks = JobList._apply_filter(parent.chunk, chunks_to, associative_list["chunks"], "chunks") - valid_splits = JobList._apply_filter(parent.split, splits_to, associative_list["splits"], "splits") - if valid_dates and valid_members and valid_chunks and valid_splits: - if dates_to.find("?") != -1 or members_to.find("?") != -1 or chunks_to.find("?") != -1 or splits_to.find("?") != -1: - optional = True - return True,optional + splits_to = "none" + associative_list["dates"] = date_list + associative_list["members"] = member_list + associative_list["chunks"] = chunk_list + if parent.splits is not None: + associative_list["splits"] = [ str(split) for split in range(1,int(parent.splits)+1) ] + else: + associative_list["splits"] = None + + if dates_to == "natural": + associative_list["dates"] = [date2str(parent.date)] if parent.date is not None else date_list + if members_to == "natural": + associative_list["members"] = [parent.member] if parent.member is not None else member_list + if chunks_to == "natural": + associative_list["chunks"] = [parent.chunk] if parent.chunk is not None else chunk_list + if splits_to == "natural": + associative_list["splits"] = [parent.split] if parent.split is not None else parent.splits + parsed_parent_date = date2str(parent.date) if parent.date is not None else None + # Apply all filters to look if this parent is an appropriated candidate for the current_job + valid_dates = JobList._apply_filter(parsed_parent_date, dates_to, associative_list["dates"], "dates") + valid_members = JobList._apply_filter(parent.member, members_to, associative_list["members"],"members") + valid_chunks = JobList._apply_filter(parent.chunk, chunks_to, associative_list["chunks"], "chunks") + valid_splits = JobList._apply_filter(parent.split, splits_to, associative_list["splits"], "splits") + if valid_dates and valid_members and valid_chunks and valid_splits: + if dates_to.find("?") != -1 or members_to.find("?") != -1 or chunks_to.find("?") != -1 or splits_to.find("?") != -1: + optional = True + return True,optional return False,optional @staticmethod def _manage_job_dependencies(dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys, dependencies, @@ -706,28 +679,20 @@ class JobList(object): other_parents = dic_jobs.get_jobs(dependency.section, None, None, None) parents_jobs = dic_jobs.get_jobs(dependency.section, date, member, chunk) natural_jobs = dic_jobs.get_jobs(dependency.section, date, member, chunk) - #if job.split is not None: - # natural_jobs = [p_job for p_job in natural_jobs if p_job.split == job.split or p_job.split is None] - if dependency.sign in ['?']: - optional_section = True - else: - optional_section = False all_parents = list(set(other_parents + parents_jobs)) # Get dates_to, members_to, chunks_to of the deepest level of the relationship. - filters_to_apply,optional_from = JobList._filter_current_job(job,copy.deepcopy(dependency.relationships)) - if len(filters_to_apply) == 0: - filters_to_apply.append({"DATES_TO": "natural", "MEMBERS_TO": "natural", "CHUNKS_TO": "natural", "SPLITS_TO": "natural"}) + filters_to_apply = JobList._filter_current_job(job,copy.deepcopy(dependency.relationships)) for parent in all_parents: # If splits is not None, the job is a list of jobs if parent.name == job.name: continue # Check if it is a natural relation. The only difference is that a chunk can depend on a chunks <= than the current chunk - if parent in natural_jobs and ((job.chunk is None or parent.chunk is None or parent.chunk <= job.chunk )): + if parent in natural_jobs and (job.chunk is None or parent.chunk is None or parent.chunk <= job.chunk ): natural_relationship = True else: natural_relationship = False # Check if the current parent is a valid parent based on the dependencies set on expdef.conf - valid,optional_to = JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship,filters_to_apply) + valid,optional = JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship,filters_to_apply) if not valid: continue else: @@ -736,7 +701,7 @@ class JobList(object): job.add_parent(parent) JobList._add_edge(graph, job, parent) # Could be more variables in the future - if optional_to or optional_from or optional_section: + if optional: job.add_edge_info(parent.name,special_variables={"optional":True}) JobList.handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member, member_list, dependency.section, graph, other_parents) -- GitLab From 4a6c5c63538e8763d2e01506e160a8d71fc82ef4 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 26 May 2023 16:13:14 +0200 Subject: [PATCH 20/29] commented the function --- autosubmit/job/job_list.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index dcb2e3650..0252c742d 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -449,21 +449,51 @@ class JobList(object): @staticmethod def _check_dates(relationships, current_job): filters_to_apply = JobList._check_relationship(relationships, "DATES_FROM", date2str(current_job.date)) + # there could be multiple filters that apply... per example + # Current task date is 20020201, and member is fc2 + # Dummy example, not specially usefull in a real case + #DATES_FROM: + #all: + #MEMBERS_FROM: + #ALL: ... + #CHUNKS_FROM: + #ALL: ... + #20020201: + #MEMBERS_FROM: + #fc2: + #DATES_TO: "20020201" + #MEMBERS_TO: "fc2" + #CHUNKS_TO: "ALL" + #SPLITS_FROM: + #ALL: + #SPLITS_TO: "1" + # this "for" iterates for ALL and fc2 as current task is selected in both filters + # The dict in this step is: + # [{MEMBERS_FROM{..},CHUNKS_FROM{...}},{MEMBERS_FROM{..},SPLITS_FROM{...}}] for i,filter in enumerate(filters_to_apply): + # {MEMBERS_FROM{..},CHUNKS_FROM{...}} I want too look ALL filters not only one, but I want to go recursivily until get the _TO filter optional = filter.pop("OPTIONAL", False) + # This is not an if_else, because the current level ( dates ) could have two different filters. + # Second case commented: ( date_from 20020201 ) + # Will enter, go recursivily to the similar methods and in the end it will do: + # Will enter members_from, and obtain [{DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", CHUNKS_FROM{...}] if "MEMBERS_FROM" in filter: filters_to_apply_m = JobList._check_members({"MEMBERS_FROM": (filter.pop("MEMBERS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_m) > 0: filters_to_apply[i] = filters_to_apply_m + # Will enter chunks_from, and obtain [{DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"] if "CHUNKS_FROM" in filter: filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_c) > 0 and len(filters_to_apply_c[0]) > 0: filters_to_apply[i] = filters_to_apply_c + #IGNORED if "SPLITS_FROM" in filter: filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_s) > 0: filters_to_apply[i] = filters_to_apply_s + # Unify filters from all filters_from where the current job is included to have a single SET of filters_to filters_to_apply = JobList._unify_to_filters(filters_to_apply) + # {DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"} return filters_to_apply @staticmethod @@ -572,7 +602,7 @@ class JobList(object): if current_job.section.lower() == "opa": print("debugging") if relationships is not None and len(relationships) > 0: - # Look for a starting point + # Look for a starting point, this can be if else becasue they're exclusive as a DATE_FROM can't be in a MEMBER_FROM and so on if "DATES_FROM" in relationships: filters_to_apply = JobList._check_dates(relationships, current_job) elif "MEMBERS_FROM" in relationships: -- GitLab From 787cb4a83154606b610ff67ab0789fa2c5680eee Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 26 May 2023 16:32:18 +0200 Subject: [PATCH 21/29] Added docstrings --- autosubmit/job/job_list.py | 50 +++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 0252c742d..5b5533f87 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -448,6 +448,12 @@ class JobList(object): @staticmethod def _check_dates(relationships, current_job): + """ + Check if the current_job_value is included in the filter_from and retrieve filter_to value + :param relationships: Remaining filters to apply. + :param current_job: Current job to check. + :return: filters_to_apply + """ filters_to_apply = JobList._check_relationship(relationships, "DATES_FROM", date2str(current_job.date)) # there could be multiple filters that apply... per example # Current task date is 20020201, and member is fc2 @@ -498,6 +504,12 @@ class JobList(object): @staticmethod def _check_members(relationships, current_job): + """ + Check if the current_job_value is included in the filter_from and retrieve filter_to value + :param relationships: Remaining filters to apply. + :param current_job: Current job to check. + :return: filters_to_apply + """ filters_to_apply = JobList._check_relationship(relationships, "MEMBERS_FROM", current_job.member) for i,filter in enumerate(filters_to_apply): optional = filter.pop("OPTIONAL", False) @@ -514,6 +526,12 @@ class JobList(object): @staticmethod def _check_chunks(relationships, current_job): + """ + Check if the current_job_value is included in the filter_from and retrieve filter_to value + :param relationships: Remaining filters to apply. + :param current_job: Current job to check. + :return: filters_to_apply + """ filters_to_apply = JobList._check_relationship(relationships, "CHUNKS_FROM", current_job.chunk) for i,filter in enumerate(filters_to_apply): optional = filter.pop("OPTIONAL", False) @@ -526,6 +544,12 @@ class JobList(object): @staticmethod def _check_splits(relationships, current_job): + """ + Check if the current_job_value is included in the filter_from and retrieve filter_to value + :param relationships: Remaining filters to apply. + :param current_job: Current job to check. + :return: filters_to_apply + """ filters_to_apply = JobList._check_relationship(relationships, "SPLITS_FROM", current_job.split) # No more FROM sections to check, unify _to FILTERS and return filters_to_apply = JobList._unify_to_filters(filters_to_apply) @@ -533,6 +557,13 @@ class JobList(object): @staticmethod def _unify_to_filter(unified_filter,filter_to,filter_type): + """ + Unify filter_to filters into a single dictionary + :param unified_filter: Single dictionary with all filters_to + :param filter_to: Current dictionary that contains the filters_to + :param filter_type: "DATES_TO", "MEMBERS_TO", "CHUNKS_TO", "SPLITS_TO" + :return: unified_filter + """ if "all" not in unified_filter[filter_type]: aux = filter_to.pop(filter_type, None) if aux: @@ -546,6 +577,12 @@ class JobList(object): unified_filter[filter_type].add(element) @staticmethod def _normalize_to_filters(filter_to,filter_type): + """ + Normalize filter_to filters to a single string or "all" + :param filter_to: Unified filter_to dictionary + :param filter_type: "DATES_TO", "MEMBERS_TO", "CHUNKS_TO", "SPLITS_TO" + :return: + """ if len(filter_to[filter_type]) == 0: filter_to.pop(filter_type, None) elif "all" in filter_to[filter_type]: @@ -556,6 +593,11 @@ class JobList(object): @staticmethod def _unify_to_filters(filter_to_apply): + """ + Unify all filter_to filters into a single dictionary ( of current selection ) + :param filter_to_apply: Filters to apply + :return: Single dictionary with all filters_to + """ unified_filter = {"DATES_TO": set(), "MEMBERS_TO": set(), "CHUNKS_TO": set(), "SPLITS_TO": set()} for filter_to in filter_to_apply: if len(filter_to) > 0: @@ -575,10 +617,10 @@ class JobList(object): @staticmethod def _filter_current_job(current_job,relationships): ''' - Check if the current_job is included in the filter_value ( from) - :param current_job: current job to check - :param dependency: dependency to check - :return: filter_to_apply(dict), boolean + This function will filter the current job based on the relationships given + :param current_job: Current job to filter + :param relationships: Relationships to apply + :return: dict() with the filters to apply, or empty dict() if no filters to apply ''' # This function will look if the given relationship is set for the given job DATEs,MEMBER,CHUNK,SPLIT ( _from filters ) -- GitLab From 4ddba0a44b1c7bda5c4c301e3842e93fc5ff9076 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 26 May 2023 16:33:40 +0200 Subject: [PATCH 22/29] fixed typo --- docs/source/userguide/wrappers/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/userguide/wrappers/index.rst b/docs/source/userguide/wrappers/index.rst index 155d4d66a..168e5afa8 100644 --- a/docs/source/userguide/wrappers/index.rst +++ b/docs/source/userguide/wrappers/index.rst @@ -392,7 +392,7 @@ Considering the following configuration: "20120201": CHUNKS_FROM: 1: - DATES_TO: "º" + DATES_TO: "20120101" CHUNKS_TO: "1" RUNNING: chunk SYNCHRONIZE: member -- GitLab From f71ace4347a4c83265237213b8034f73b1784916 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 26 May 2023 16:37:45 +0200 Subject: [PATCH 23/29] Reverting change --- autosubmit/job/job_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 5b5533f87..7fb0e0d4b 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -417,7 +417,7 @@ class JobList(object): else: to_filter.append(filter_value) - if str(to_filter).find(str(parent_value).upper()) != -1: + if str(parent_value).upper() in str(to_filter).upper(): return True else: return False -- GitLab From 4ea678f393c235a771ab5c436d410e1edd46d365 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 31 May 2023 17:03:56 +0200 Subject: [PATCH 24/29] added testcases fixed few things --- autosubmit/job/job_list.py | 76 ++++++++------- test/unit/test_dependencies.py | 173 +++++++++++++++++++++++++++++++++ 2 files changed, 214 insertions(+), 35 deletions(-) create mode 100644 test/unit/test_dependencies.py diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 7fb0e0d4b..3cccba1c4 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -434,7 +434,7 @@ class JobList(object): """ filters = [] for filter_range,filter_data in relationships.get(level_to_check,{}).items(): - if not str(value_to_check) or str(filter_range).upper() in "ALL" or str(value_to_check).upper() in str(filter_range).upper(): + if not value_to_check or str(filter_range).upper() in "ALL" or str(value_to_check).upper() in str(filter_range).upper(): if filter_data: if "?" in filter_range: filter_data["OPTIONAL"] = True @@ -454,6 +454,7 @@ class JobList(object): :param current_job: Current job to check. :return: filters_to_apply """ + optional = False filters_to_apply = JobList._check_relationship(relationships, "DATES_FROM", date2str(current_job.date)) # there could be multiple filters that apply... per example # Current task date is 20020201, and member is fc2 @@ -486,18 +487,21 @@ class JobList(object): if "MEMBERS_FROM" in filter: filters_to_apply_m = JobList._check_members({"MEMBERS_FROM": (filter.pop("MEMBERS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_m) > 0: - filters_to_apply[i] = filters_to_apply_m + filters_to_apply[i].update(filters_to_apply_m) # Will enter chunks_from, and obtain [{DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"] if "CHUNKS_FROM" in filter: filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_c) > 0 and len(filters_to_apply_c[0]) > 0: - filters_to_apply[i] = filters_to_apply_c + filters_to_apply[i].update(filters_to_apply_c) #IGNORED if "SPLITS_FROM" in filter: filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_s) > 0: - filters_to_apply[i] = filters_to_apply_s + filters_to_apply[i].update(filters_to_apply_s) # Unify filters from all filters_from where the current job is included to have a single SET of filters_to + if optional: + for i,filter in enumerate(filters_to_apply): + filters_to_apply[i]["OPTIONAL"] = True filters_to_apply = JobList._unify_to_filters(filters_to_apply) # {DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"} return filters_to_apply @@ -511,16 +515,21 @@ class JobList(object): :return: filters_to_apply """ filters_to_apply = JobList._check_relationship(relationships, "MEMBERS_FROM", current_job.member) + optional = False for i,filter in enumerate(filters_to_apply): optional = filter.pop("OPTIONAL", False) if "CHUNKS_FROM" in filter: filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_c) > 0: - filters_to_apply[i] = filters_to_apply_c + filters_to_apply[i].update(filters_to_apply_c) + if "SPLITS_FROM" in filter: filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_s) > 0: - filters_to_apply[i] = filters_to_apply_s + filters_to_apply[i].update(filters_to_apply_s) + if optional: + for i,filter in enumerate(filters_to_apply): + filters_to_apply[i]["OPTIONAL"] = True filters_to_apply = JobList._unify_to_filters(filters_to_apply) return filters_to_apply @@ -532,13 +541,17 @@ class JobList(object): :param current_job: Current job to check. :return: filters_to_apply """ + optional = False filters_to_apply = JobList._check_relationship(relationships, "CHUNKS_FROM", current_job.chunk) for i,filter in enumerate(filters_to_apply): optional = filter.pop("OPTIONAL", False) if "SPLITS_FROM" in filter: filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_s) > 0: - filters_to_apply[i] = filters_to_apply_s + filters_to_apply[i].update(filters_to_apply_s) + if optional: + for i,filter in enumerate(filters_to_apply): + filters_to_apply[i]["OPTIONAL"] = True filters_to_apply = JobList._unify_to_filters(filters_to_apply) return filters_to_apply @@ -550,6 +563,7 @@ class JobList(object): :param current_job: Current job to check. :return: filters_to_apply """ + filters_to_apply = JobList._check_relationship(relationships, "SPLITS_FROM", current_job.split) # No more FROM sections to check, unify _to FILTERS and return filters_to_apply = JobList._unify_to_filters(filters_to_apply) @@ -638,12 +652,11 @@ class JobList(object): # 3. NATURAL: this is the normal behavior, represents a way of letting the job to be activated if they would normally be activated. # 4. ? : this is a weak dependency activation flag, The dependency will be activated but the job can fail without affecting the workflow. - filters_to_apply = [{}] + filters_to_apply = {} # Check if filter_from-filter_to relationship is set - relationships["OPTIONAL"] = False - if current_job.section.lower() == "opa": - print("debugging") if relationships is not None and len(relationships) > 0: + if "OPTIONAL" not in relationships: + relationships["OPTIONAL"] = False # Look for a starting point, this can be if else becasue they're exclusive as a DATE_FROM can't be in a MEMBER_FROM and so on if "DATES_FROM" in relationships: filters_to_apply = JobList._check_dates(relationships, current_job) @@ -658,7 +671,7 @@ class JobList(object): relationships.pop("MEMBERS_FROM", None) relationships.pop("DATES_FROM", None) relationships.pop("SPLITS_FROM", None) - filters_to_apply = [relationships] + filters_to_apply = relationships return filters_to_apply @@ -676,8 +689,14 @@ class JobList(object): :return: True if the parent is valid, False otherwise ''' #check if current_parent is listed on dependency.relationships - optional = False associative_list = {} + associative_list["dates"] = date_list + associative_list["members"] = member_list + associative_list["chunks"] = chunk_list + if parent.splits is not None: + associative_list["splits"] = [ str(split) for split in range(1,int(parent.splits)+1) ] + else: + associative_list["splits"] = None dates_to = str(filter_.get("DATES_TO", "natural")).lower() members_to = str(filter_.get("MEMBERS_TO", "natural")).lower() chunks_to = str(filter_.get("CHUNKS_TO", "natural")).lower() @@ -691,14 +710,6 @@ class JobList(object): chunks_to = "none" if splits_to == "natural": splits_to = "none" - associative_list["dates"] = date_list - associative_list["members"] = member_list - associative_list["chunks"] = chunk_list - if parent.splits is not None: - associative_list["splits"] = [ str(split) for split in range(1,int(parent.splits)+1) ] - else: - associative_list["splits"] = None - if dates_to == "natural": associative_list["dates"] = [date2str(parent.date)] if parent.date is not None else date_list if members_to == "natural": @@ -710,14 +721,12 @@ class JobList(object): parsed_parent_date = date2str(parent.date) if parent.date is not None else None # Apply all filters to look if this parent is an appropriated candidate for the current_job valid_dates = JobList._apply_filter(parsed_parent_date, dates_to, associative_list["dates"], "dates") - valid_members = JobList._apply_filter(parent.member, members_to, associative_list["members"],"members") + valid_members = JobList._apply_filter(parent.member, members_to, associative_list["members"], "members") valid_chunks = JobList._apply_filter(parent.chunk, chunks_to, associative_list["chunks"], "chunks") valid_splits = JobList._apply_filter(parent.split, splits_to, associative_list["splits"], "splits") if valid_dates and valid_members and valid_chunks and valid_splits: - if dates_to.find("?") != -1 or members_to.find("?") != -1 or chunks_to.find("?") != -1 or splits_to.find("?") != -1: - optional = True - return True,optional - return False,optional + return True,( "?" in [dates_to,members_to,chunks_to,splits_to] ) + return False,False @staticmethod def _manage_job_dependencies(dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys, dependencies, graph): @@ -765,16 +774,13 @@ class JobList(object): natural_relationship = False # Check if the current parent is a valid parent based on the dependencies set on expdef.conf valid,optional = JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship,filters_to_apply) - if not valid: - continue - else: - pass # If the parent is valid, add it to the graph - job.add_parent(parent) - JobList._add_edge(graph, job, parent) - # Could be more variables in the future - if optional: - job.add_edge_info(parent.name,special_variables={"optional":True}) + if valid: + job.add_parent(parent) + JobList._add_edge(graph, job, parent) + # Could be more variables in the future + if optional: + job.add_edge_info(parent.name,special_variables={"optional":True}) JobList.handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member, member_list, dependency.section, graph, other_parents) diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py new file mode 100644 index 000000000..64ddb8f6b --- /dev/null +++ b/test/unit/test_dependencies.py @@ -0,0 +1,173 @@ +import unittest + +import mock +from copy import deepcopy +from autosubmit.job.job_list import JobList +from autosubmit.job.job import Job +from autosubmit.job.job_common import Status +from datetime import datetime +class TestJobList(unittest.TestCase): + def setUp(self): + # Define common test case inputs here + self.relationships_dates = { + "OPTIONAL": False, + "DATES_FROM": { + "20020201": { + "MEMBERS_FROM": { + "fc2": { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL" + } + }, + "SPLITS_FROM": { + "ALL": { + "SPLITS_TO": "1" + } + } + } + } + } + self.relationships_dates_optional = deepcopy(self.relationships_dates) + self.relationships_dates_optional["DATES_FROM"]["20020201"]["MEMBERS_FROM"] = { "fc2?": { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", "CHUNKS_TO": "ALL", "SPLITS_TO": "5" } } + self.relationships_dates_optional["DATES_FROM"]["20020201"]["SPLITS_FROM"] = { "ALL": { "SPLITS_TO": "1?" } } + + self.relationships_members = { + "OPTIONAL": False, + "MEMBERS_FROM": { + "fc2": { + "SPLITS_FROM": { + "ALL": { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + } + } + } + } + self.relationships_chunks = { + "OPTIONAL": False, + "CHUNKS_FROM": { + "1": { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + } + } + self.relationships_splits = { + "OPTIONAL": False, + "SPLITS_FROM": { + "1": { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + } + } + + # Create a mock Job object + self.mock_job = mock.MagicMock(spec=Job) + + # Set the attributes on the mock object + self.mock_job.name = "Job1" + self.mock_job.job_id = 1 + self.mock_job.status = Status.READY + self.mock_job.priority = 1 + self.mock_job.date = None + self.mock_job.member = None + self.mock_job.chunk = None + self.mock_job.split = None + def test_simple_dependency(self): + result_d = JobList._check_dates({}, self.mock_job) + result_m = JobList._check_members({}, self.mock_job) + result_c = JobList._check_chunks({}, self.mock_job) + result_s = JobList._check_splits({}, self.mock_job) + self.assertEqual(result_d, {}) + self.assertEqual(result_m, {}) + self.assertEqual(result_c, {}) + self.assertEqual(result_s, {}) + def test_check_dates_optional(self): + self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") + self.mock_job.member = "fc2" + self.mock_job.chunk = 1 + self.mock_job.split = 1 + result = JobList._check_dates(self.relationships_dates_optional, self.mock_job) + expected_output = { + "DATES_TO": "20020201?", + "MEMBERS_TO": "fc2?", + "CHUNKS_TO": "ALL?", + "SPLITS_TO": "1?" + } + self.assertEqual(result, expected_output) + def test_check_dates(self): + # Call the function to get the result + self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") + self.mock_job.member = "fc2" + self.mock_job.chunk = 1 + self.mock_job.split = 1 + result = JobList._check_dates(self.relationships_dates, self.mock_job) + expected_output = { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + self.assertEqual(result, expected_output) + self.mock_job.date = datetime.strptime("20020202", "%Y%m%d") + result = JobList._check_dates(self.relationships_dates, self.mock_job) + self.assertEqual(result, {}) + def test_check_members(self): + # Call the function to get the result + self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") + self.mock_job.member = "fc2" + + result = JobList._check_members(self.relationships_members, self.mock_job) + expected_output = { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + self.assertEqual(result, expected_output) + self.mock_job.member = "fc3" + result = JobList._check_members(self.relationships_members, self.mock_job) + self.assertEqual(result, {}) + + def test_check_splits(self): + # Call the function to get the result + + self.mock_job.split = 1 + result = JobList._check_splits(self.relationships_splits, self.mock_job) + expected_output = { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + self.assertEqual(result, expected_output) + self.mock_job.split = 2 + result = JobList._check_splits(self.relationships_splits, self.mock_job) + self.assertEqual(result, {}) + def test_check_chunks(self): + # Call the function to get the result + + self.mock_job.chunk = 1 + + result = JobList._check_chunks(self.relationships_chunks, self.mock_job) + expected_output = { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + self.assertEqual(result, expected_output) + self.mock_job.chunk = 2 + result = JobList._check_chunks(self.relationships_chunks, self.mock_job) + self.assertEqual(result, {}) +if __name__ == '__main__': + unittest.main() -- GitLab From 2cb9c9d7c436e1fa4f5ee11299f57f5b8cb97787 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 9 Jun 2023 17:07:38 +0200 Subject: [PATCH 25/29] added split property --- autosubmit/job/job.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 601a2d581..8798a0ebd 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -441,6 +441,14 @@ class Job(object): @custom_directives.setter def custom_directives(self, value): self._custom_directives = value + @property + @autosubmit_parameter(name='splits') + def splits(self): + """Number of splits.""" + return self._splits + @splits.setter + def splits(self, value): + self._splits = value def __getstate__(self): odict = self.__dict__ -- GitLab From 6d05bcc5e00bb22114a11cb58eb1ba60cf679de8 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 9 Jun 2023 17:08:47 +0200 Subject: [PATCH 26/29] added split property --- autosubmit/job/job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 8798a0ebd..568a9550f 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -444,7 +444,7 @@ class Job(object): @property @autosubmit_parameter(name='splits') def splits(self): - """Number of splits.""" + """Max number of splits.""" return self._splits @splits.setter def splits(self, value): -- GitLab From f144a10cb920f1722e8ea92565c85be4e59f7b6a Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 9 Jun 2023 18:33:02 +0200 Subject: [PATCH 27/29] added tests --- autosubmit/autosubmit.py | 1 + autosubmit/job/job_list.py | 6 +++++- test/unit/test_dependencies.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index c7635b73b..804be36cf 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -808,6 +808,7 @@ class Autosubmit: os.mkdir(aslogs_path) if owner: os.chmod(tmp_path, 0o775) + Log.set_file(os.path.join(aslogs_path, args.command + '.log'), "out", log_level) Log.set_file(os.path.join(aslogs_path, args.command + '_err.log'), "err") if args.command in ["run"]: diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 3cccba1c4..3fe0291d4 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -693,6 +693,7 @@ class JobList(object): associative_list["dates"] = date_list associative_list["members"] = member_list associative_list["chunks"] = chunk_list + if parent.splits is not None: associative_list["splits"] = [ str(split) for split in range(1,int(parent.splits)+1) ] else: @@ -725,7 +726,10 @@ class JobList(object): valid_chunks = JobList._apply_filter(parent.chunk, chunks_to, associative_list["chunks"], "chunks") valid_splits = JobList._apply_filter(parent.split, splits_to, associative_list["splits"], "splits") if valid_dates and valid_members and valid_chunks and valid_splits: - return True,( "?" in [dates_to,members_to,chunks_to,splits_to] ) + for value in [dates_to, members_to, chunks_to, splits_to]: + if "?" in value: + return True, True + return True, False return False,False @staticmethod def _manage_job_dependencies(dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys, dependencies, diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 64ddb8f6b..c9af72477 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -169,5 +169,37 @@ class TestJobList(unittest.TestCase): self.mock_job.chunk = 2 result = JobList._check_chunks(self.relationships_chunks, self.mock_job) self.assertEqual(result, {}) + def test_valid_parent(self): + # Call the function to get the result + + date_list = ["20020201"] + member_list = ["fc1", "fc2", "fc3"] + chunk_list = [1, 2, 3] + is_a_natural_relation = False + # Filter_to values + filter_ = { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + # PArent job values + self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") + self.mock_job.member = "fc2" + self.mock_job.chunk = 1 + self.mock_job.split = 1 + result = JobList._valid_parent(self.mock_job, date_list, member_list, chunk_list, is_a_natural_relation, filter_) + # it returns a tuple, the first element is the result, the second is the optional flag + self.assertEqual(result, (True,False)) + filter_ = { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1?" + } + result = JobList._valid_parent(self.mock_job, date_list, member_list, chunk_list, is_a_natural_relation, filter_) + self.assertEqual(result, (True,True)) + + if __name__ == '__main__': unittest.main() -- GitLab From 6a0aa9fffe2fbf7a30c4abddea84222c2add8513 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 12 Jun 2023 08:19:42 +0200 Subject: [PATCH 28/29] testing split_from --- test/unit/test_dependencies.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index c9af72477..22d6548b2 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -58,6 +58,26 @@ class TestJobList(unittest.TestCase): } } } + self.relationships_chunks2 = { + "OPTIONAL": False, + "CHUNKS_FROM": { + "1": { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + }, + "2": { + "SPLITS_FROM": { + "5": { + "SPLITS_TO": "2" + } + } + } + } + } + + self.relationships_splits = { "OPTIONAL": False, "SPLITS_FROM": { @@ -157,7 +177,6 @@ class TestJobList(unittest.TestCase): # Call the function to get the result self.mock_job.chunk = 1 - result = JobList._check_chunks(self.relationships_chunks, self.mock_job) expected_output = { "DATES_TO": "20020201", @@ -169,6 +188,17 @@ class TestJobList(unittest.TestCase): self.mock_job.chunk = 2 result = JobList._check_chunks(self.relationships_chunks, self.mock_job) self.assertEqual(result, {}) + # test splits_from + self.mock_job.split = 5 + result = JobList._check_chunks(self.relationships_chunks2, self.mock_job) + expected_output2 = { + "SPLITS_TO": "2" + } + self.assertEqual(result, expected_output2) + self.mock_job.split = 1 + result = JobList._check_chunks(self.relationships_chunks2, self.mock_job) + self.assertEqual(result, {}) + def test_valid_parent(self): # Call the function to get the result -- GitLab From a6f81d6614efb3fcf63be08d97887af8f295d866 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 12 Jun 2023 08:29:36 +0200 Subject: [PATCH 29/29] Test general filter --- autosubmit/job/job_list.py | 1 + test/unit/test_dependencies.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 3fe0291d4..edd67d1c5 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -667,6 +667,7 @@ class JobList(object): elif "SPLITS_FROM" in relationships: filters_to_apply = JobList._check_splits(relationships, current_job) else: + relationships.pop("OPTIONAL", None) relationships.pop("CHUNKS_FROM", None) relationships.pop("MEMBERS_FROM", None) relationships.pop("DATES_FROM", None) diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 22d6548b2..a08a4a73d 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -90,6 +90,12 @@ class TestJobList(unittest.TestCase): } } + self.relationships_general = { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } # Create a mock Job object self.mock_job = mock.MagicMock(spec=Job) @@ -199,6 +205,23 @@ class TestJobList(unittest.TestCase): result = JobList._check_chunks(self.relationships_chunks2, self.mock_job) self.assertEqual(result, {}) + def test_check_general(self): + # Call the function to get the result + + self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") + self.mock_job.member = "fc2" + self.mock_job.chunk = 1 + self.mock_job.split = 1 + result = JobList._filter_current_job(self.mock_job,self.relationships_general) + expected_output = { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + self.assertEqual(result, expected_output) + + def test_valid_parent(self): # Call the function to get the result -- GitLab