diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index c7635b73b7f6430dacb97090d70ddcab96beb4d6..804be36cf5ff5e2dfc348db81ef78042f518cfb0 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -808,6 +808,7 @@ class Autosubmit: os.mkdir(aslogs_path) if owner: os.chmod(tmp_path, 0o775) + Log.set_file(os.path.join(aslogs_path, args.command + '.log'), "out", log_level) Log.set_file(os.path.join(aslogs_path, args.command + '_err.log'), "err") if args.command in ["run"]: diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index ab754734e6f6ddc0eeeb0931b8d1a21e1595f9fb..568a9550fc3f6a127c9188e69c2592965c8312f4 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -135,6 +135,7 @@ class Job(object): return "{0} STATUS: {1}".format(self.name, self.status) def __init__(self, name, job_id, status, priority): + self.splits = None self.script_name_wrapper = None self.delay_end = datetime.datetime.now() self._delay_retrials = "0" @@ -440,6 +441,14 @@ class Job(object): @custom_directives.setter def custom_directives(self, value): self._custom_directives = value + @property + @autosubmit_parameter(name='splits') + def splits(self): + """Max number of splits.""" + return self._splits + @splits.setter + def splits(self, value): + self._splits = value def __getstate__(self): odict = self.__dict__ @@ -1333,6 +1342,7 @@ class Job(object): parameters['SDATE'] = self.sdate parameters['MEMBER'] = self.member parameters['SPLIT'] = self.split + parameters['SPLITS'] = self.splits parameters['DELAY'] = self.delay parameters['FREQUENCY'] = self.frequency parameters['SYNCHRONIZE'] = self.synchronize @@ -1404,20 +1414,6 @@ class Job(object): parameters['CHUNK_LAST'] = 'FALSE' parameters['NUMMEMBERS'] = len(as_conf.get_member_list()) parameters['DEPENDENCIES'] = str(as_conf.jobs_data[self.section].get("DEPENDENCIES","")) - # This shouldn't be necessary anymore as now all sub is done in the as_conf.reload() - # if len(self.export) > 0: - # variables = re.findall('%(? 0: - # variables = [variable[1:-1] for variable in variables] - # for key in variables: - # try: - # self.export = re.sub( - # '%(? 1 and isinstance(jobs[0], Iterable): + if len(jobs) > 0 and isinstance(jobs[0], list): try: jobs_flattened = [job for jobs_to_flatten in jobs for job in jobs_to_flatten] jobs = jobs_flattened @@ -355,6 +355,7 @@ class DicJobs: job.date = date job.member = member job.chunk = chunk + job.splits = self.experiment_data["JOBS"].get(job.section,{}).get("SPLITS", None) job.date_format = self._date_format job.delete_when_edgeless = str(parameters[section].get("DELETE_WHEN_EDGELESS", "true")).lower() diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 380431d52ce4e5f0355dd06931e798e4b429d601..edd67d1c5ff9d1d3461f6420840614069352ee51 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -394,7 +394,7 @@ class JobList(object): return False elif filter_value.find(",") != -1: aux_filter = filter_value.split(",") - if filter_type != "chunks": + if filter_type not in ["chunks", "splits"]: for value in aux_filter: if str(value).isdigit(): to_filter.append(associative_list[int(value)]) @@ -408,7 +408,7 @@ class JobList(object): start = start_end[0].strip("[]") end = start_end[1].strip("[]") del start_end - if filter_type == "chunks": # chunk directly + if filter_type not in ["chunks", "splits"]: # chunk directly for value in range(int(start), int(end) + 1): to_filter.append(value) else: # index @@ -416,6 +416,7 @@ class JobList(object): to_filter.append(value) else: to_filter.append(filter_value) + if str(parent_value).upper() in str(to_filter).upper(): return True else: @@ -427,132 +428,310 @@ class JobList(object): """ Check if the current_job_value is included in the filter_value :param relationship: current filter level to check. - :param level_to_check: can be a date, member or chunk. - :param value_to_check: can be a date, member or chunk. + :param level_to_check: can be a date, member, chunk or split. + :param value_to_check: Can be None, a date, a member, a chunk or a split. :return: """ - optional = False filters = [] - for filter_type, filter_data in relationships.items(): - if isinstance(filter_data, collections.abc.Mapping): - current_filter = {} - if filter_type.upper().find(level_to_check.upper()) != -1: - for filter_range in filter_data.keys(): - if str(value_to_check) is None or str(filter_range).upper().find(str(value_to_check).upper()) != -1: - if filter_data[filter_range] is not None: - if filter_type.find("?") != -1 or filter_range.find("?") != -1: - optional = True - current_filter.update(filter_data[filter_range]) - filters.append(current_filter) + for filter_range,filter_data in relationships.get(level_to_check,{}).items(): + if not value_to_check or str(filter_range).upper() in "ALL" or str(value_to_check).upper() in str(filter_range).upper(): + if filter_data: + if "?" in filter_range: + filter_data["OPTIONAL"] = True + else: + filter_data["OPTIONAL"] = relationships["OPTIONAL"] + filters.append(filter_data) # Normalize the filter return if len(filters) == 0: filters = [{}] - return filters,optional + return filters + + @staticmethod + def _check_dates(relationships, current_job): + """ + Check if the current_job_value is included in the filter_from and retrieve filter_to value + :param relationships: Remaining filters to apply. + :param current_job: Current job to check. + :return: filters_to_apply + """ + optional = False + filters_to_apply = JobList._check_relationship(relationships, "DATES_FROM", date2str(current_job.date)) + # there could be multiple filters that apply... per example + # Current task date is 20020201, and member is fc2 + # Dummy example, not specially usefull in a real case + #DATES_FROM: + #all: + #MEMBERS_FROM: + #ALL: ... + #CHUNKS_FROM: + #ALL: ... + #20020201: + #MEMBERS_FROM: + #fc2: + #DATES_TO: "20020201" + #MEMBERS_TO: "fc2" + #CHUNKS_TO: "ALL" + #SPLITS_FROM: + #ALL: + #SPLITS_TO: "1" + # this "for" iterates for ALL and fc2 as current task is selected in both filters + # The dict in this step is: + # [{MEMBERS_FROM{..},CHUNKS_FROM{...}},{MEMBERS_FROM{..},SPLITS_FROM{...}}] + for i,filter in enumerate(filters_to_apply): + # {MEMBERS_FROM{..},CHUNKS_FROM{...}} I want too look ALL filters not only one, but I want to go recursivily until get the _TO filter + optional = filter.pop("OPTIONAL", False) + # This is not an if_else, because the current level ( dates ) could have two different filters. + # Second case commented: ( date_from 20020201 ) + # Will enter, go recursivily to the similar methods and in the end it will do: + # Will enter members_from, and obtain [{DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", CHUNKS_FROM{...}] + if "MEMBERS_FROM" in filter: + filters_to_apply_m = JobList._check_members({"MEMBERS_FROM": (filter.pop("MEMBERS_FROM")),"OPTIONAL":optional}, current_job) + if len(filters_to_apply_m) > 0: + filters_to_apply[i].update(filters_to_apply_m) + # Will enter chunks_from, and obtain [{DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"] + if "CHUNKS_FROM" in filter: + filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM")),"OPTIONAL":optional}, current_job) + if len(filters_to_apply_c) > 0 and len(filters_to_apply_c[0]) > 0: + filters_to_apply[i].update(filters_to_apply_c) + #IGNORED + if "SPLITS_FROM" in filter: + filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) + if len(filters_to_apply_s) > 0: + filters_to_apply[i].update(filters_to_apply_s) + # Unify filters from all filters_from where the current job is included to have a single SET of filters_to + if optional: + for i,filter in enumerate(filters_to_apply): + filters_to_apply[i]["OPTIONAL"] = True + filters_to_apply = JobList._unify_to_filters(filters_to_apply) + # {DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"} + return filters_to_apply + + @staticmethod + def _check_members(relationships, current_job): + """ + Check if the current_job_value is included in the filter_from and retrieve filter_to value + :param relationships: Remaining filters to apply. + :param current_job: Current job to check. + :return: filters_to_apply + """ + filters_to_apply = JobList._check_relationship(relationships, "MEMBERS_FROM", current_job.member) + optional = False + for i,filter in enumerate(filters_to_apply): + optional = filter.pop("OPTIONAL", False) + if "CHUNKS_FROM" in filter: + filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM")),"OPTIONAL":optional}, current_job) + if len(filters_to_apply_c) > 0: + filters_to_apply[i].update(filters_to_apply_c) + + if "SPLITS_FROM" in filter: + filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) + if len(filters_to_apply_s) > 0: + filters_to_apply[i].update(filters_to_apply_s) + if optional: + for i,filter in enumerate(filters_to_apply): + filters_to_apply[i]["OPTIONAL"] = True + filters_to_apply = JobList._unify_to_filters(filters_to_apply) + return filters_to_apply + + @staticmethod + def _check_chunks(relationships, current_job): + """ + Check if the current_job_value is included in the filter_from and retrieve filter_to value + :param relationships: Remaining filters to apply. + :param current_job: Current job to check. + :return: filters_to_apply + """ + optional = False + filters_to_apply = JobList._check_relationship(relationships, "CHUNKS_FROM", current_job.chunk) + for i,filter in enumerate(filters_to_apply): + optional = filter.pop("OPTIONAL", False) + if "SPLITS_FROM" in filter: + filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) + if len(filters_to_apply_s) > 0: + filters_to_apply[i].update(filters_to_apply_s) + if optional: + for i,filter in enumerate(filters_to_apply): + filters_to_apply[i]["OPTIONAL"] = True + filters_to_apply = JobList._unify_to_filters(filters_to_apply) + return filters_to_apply + + @staticmethod + def _check_splits(relationships, current_job): + """ + Check if the current_job_value is included in the filter_from and retrieve filter_to value + :param relationships: Remaining filters to apply. + :param current_job: Current job to check. + :return: filters_to_apply + """ + + filters_to_apply = JobList._check_relationship(relationships, "SPLITS_FROM", current_job.split) + # No more FROM sections to check, unify _to FILTERS and return + filters_to_apply = JobList._unify_to_filters(filters_to_apply) + return filters_to_apply + + @staticmethod + def _unify_to_filter(unified_filter,filter_to,filter_type): + """ + Unify filter_to filters into a single dictionary + :param unified_filter: Single dictionary with all filters_to + :param filter_to: Current dictionary that contains the filters_to + :param filter_type: "DATES_TO", "MEMBERS_TO", "CHUNKS_TO", "SPLITS_TO" + :return: unified_filter + """ + if "all" not in unified_filter[filter_type]: + aux = filter_to.pop(filter_type, None) + if aux: + aux = aux.split(",") + for element in aux: + if element.lower().strip("?") in ["natural","none"] and len(unified_filter[filter_type]) > 0: + continue + else: + if filter_to.get("OPTIONAL",False) and element[-1] != "?": + element += "?" + unified_filter[filter_type].add(element) + @staticmethod + def _normalize_to_filters(filter_to,filter_type): + """ + Normalize filter_to filters to a single string or "all" + :param filter_to: Unified filter_to dictionary + :param filter_type: "DATES_TO", "MEMBERS_TO", "CHUNKS_TO", "SPLITS_TO" + :return: + """ + if len(filter_to[filter_type]) == 0: + filter_to.pop(filter_type, None) + elif "all" in filter_to[filter_type]: + filter_to[filter_type] = "all" + else: + # transform to str separated by commas if multiple elements + filter_to[filter_type] = ",".join(filter_to[filter_type]) + + @staticmethod + def _unify_to_filters(filter_to_apply): + """ + Unify all filter_to filters into a single dictionary ( of current selection ) + :param filter_to_apply: Filters to apply + :return: Single dictionary with all filters_to + """ + unified_filter = {"DATES_TO": set(), "MEMBERS_TO": set(), "CHUNKS_TO": set(), "SPLITS_TO": set()} + for filter_to in filter_to_apply: + if len(filter_to) > 0: + JobList._unify_to_filter(unified_filter,filter_to,"DATES_TO") + JobList._unify_to_filter(unified_filter,filter_to,"MEMBERS_TO") + JobList._unify_to_filter(unified_filter,filter_to,"CHUNKS_TO") + JobList._unify_to_filter(unified_filter,filter_to,"SPLITS_TO") + filter_to.pop("OPTIONAL", None) + + JobList._normalize_to_filters(unified_filter,"DATES_TO") + JobList._normalize_to_filters(unified_filter,"MEMBERS_TO") + JobList._normalize_to_filters(unified_filter,"CHUNKS_TO") + JobList._normalize_to_filters(unified_filter,"SPLITS_TO") + + return unified_filter + @staticmethod def _filter_current_job(current_job,relationships): ''' - Check if the current_job is included in the filter_value ( from) - :param current_job: - :param dependency: - :return: + This function will filter the current job based on the relationships given + :param current_job: Current job to filter + :param relationships: Relationships to apply + :return: dict() with the filters to apply, or empty dict() if no filters to apply ''' - # Search all filters in dependency relationship that affect current_job - # First level can be Date,member or chunk or generic - # Second level can be member or chunk or generic - # Third level can only be chunked. - # If the filter is generic, it will be applied to all section jobs. - # Check Date then Member or Chunk then Chunk - optional = False - filters_to_apply = [] - # this should be a list + + # This function will look if the given relationship is set for the given job DATEs,MEMBER,CHUNK,SPLIT ( _from filters ) + # And if it is, it will return the dependencies that need to be activated (_TO filters) + # _FROM behavior: + # DATES_FROM can contain MEMBERS_FROM,CHUNKS_FROM,SPLITS_FROM + # MEMBERS_FROM can contain CHUNKS_FROM,SPLITS_FROM + # CHUNKS_FROM can contain SPLITS_FROM + # SPLITS_FROM can contain nothing + # _TO behavior: + # TO keywords, can be in any of the _FROM filters and they will only affect the _FROM filter they are in. + # There are 4 keywords: + # 1. ALL: all the dependencies will be activated of the given filter type (dates, members, chunks or/and splits) + # 2. NONE: no dependencies will be activated of the given filter type (dates, members, chunks or/and splits) + # 3. NATURAL: this is the normal behavior, represents a way of letting the job to be activated if they would normally be activated. + # 4. ? : this is a weak dependency activation flag, The dependency will be activated but the job can fail without affecting the workflow. + + filters_to_apply = {} + # Check if filter_from-filter_to relationship is set if relationships is not None and len(relationships) > 0: - filters_to_apply,optional = JobList._check_relationship(relationships,"DATES_FROM",date2str(current_job.date)) - if len(filters_to_apply[0]) > 0: - for filter_number in range(0, len(filters_to_apply)): - if "MEMBERS_FROM" in filters_to_apply[filter_number]: - filters_to_apply_m,optional = JobList._check_relationship(filters_to_apply[filter_number],"MEMBERS_FROM",current_job.member) - if len(filters_to_apply_m) > 0: - filters_to_apply,optional = filters_to_apply_m - else: - filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply[filter_number],"CHUNKS_FROM",current_job.chunk) - if len(filters_to_apply_c) > 0: - filters_to_apply = filters_to_apply_c - elif "CHUNKS_FROM" in filters_to_apply[filter_number]: - filters_to_apply,optional = JobList._check_relationship(filters_to_apply[filter_number],"CHUNKS_FROM",current_job.chunk) - # Check Member then Chunk - if len(filters_to_apply[0]) == 0: - filters_to_apply,optional = JobList._check_relationship(relationships,"MEMBERS_FROM",current_job.member) - if len(filters_to_apply) > 0 and ( "CHUNKS_FROM" in filters_to_apply): - filters_to_apply_c,optional = JobList._check_relationship(filters_to_apply,"CHUNKS_FROM",current_job.chunk) - if len(filters_to_apply_c) > 0: - filters_to_apply = filters_to_apply_c - #Check Chunk - if len(filters_to_apply[0]) == 0: - filters_to_apply,optional = JobList._check_relationship(relationships,"CHUNKS_FROM",current_job.chunk) - # Generic filter - if len(filters_to_apply[0]) == 0: - relationships.pop("CHUNKS_FROM",None) - relationships.pop("MEMBERS_FROM",None) - relationships.pop("DATES_FROM",None) - filters_to_apply = [relationships] - - if len(filters_to_apply) == 1 and len(filters_to_apply[0]) == 0: - return [],optional - else: - return filters_to_apply,optional + if "OPTIONAL" not in relationships: + relationships["OPTIONAL"] = False + # Look for a starting point, this can be if else becasue they're exclusive as a DATE_FROM can't be in a MEMBER_FROM and so on + if "DATES_FROM" in relationships: + filters_to_apply = JobList._check_dates(relationships, current_job) + elif "MEMBERS_FROM" in relationships: + filters_to_apply = JobList._check_members(relationships, current_job) + elif "CHUNKS_FROM" in relationships: + filters_to_apply = JobList._check_chunks(relationships, current_job) + elif "SPLITS_FROM" in relationships: + filters_to_apply = JobList._check_splits(relationships, current_job) + else: + relationships.pop("OPTIONAL", None) + relationships.pop("CHUNKS_FROM", None) + relationships.pop("MEMBERS_FROM", None) + relationships.pop("DATES_FROM", None) + relationships.pop("SPLITS_FROM", None) + filters_to_apply = relationships + return filters_to_apply @staticmethod - def _valid_parent(parent,member_list,date_list,chunk_list,is_a_natural_relation,filters_to_apply): + def _valid_parent(parent,member_list,date_list,chunk_list,is_a_natural_relation,filter_): ''' - Check if the parent is valid for the current_job + Check if the parent is valid for the current job :param parent: job to check - :param member_list: - :param date_list: - :param chunk_list: - :param is_a_natural_relation: - :param filters_to_apply: - :return: + :param member_list: list of members + :param date_list: list of dates + :param chunk_list: list of chunks + :param is_a_natural_relation: if the relation is natural or not + :param filters_to_apply: filters to apply + :return: True if the parent is valid, False otherwise ''' #check if current_parent is listed on dependency.relationships - optional = False - for filter_ in filters_to_apply: - associative_list = {} - dates_to = str(filter_.get("DATES_TO", "natural")).lower() - members_to = str(filter_.get("MEMBERS_TO", "natural")).lower() - chunks_to = str(filter_.get("CHUNKS_TO", "natural")).lower() - if not is_a_natural_relation: - if dates_to == "natural": - dates_to = "none" - if members_to == "natural": - members_to = "none" - if chunks_to == "natural": - chunks_to = "none" - - - associative_list["dates"] = date_list - associative_list["members"] = member_list - associative_list["chunks"] = chunk_list + associative_list = {} + associative_list["dates"] = date_list + associative_list["members"] = member_list + associative_list["chunks"] = chunk_list + if parent.splits is not None: + associative_list["splits"] = [ str(split) for split in range(1,int(parent.splits)+1) ] + else: + associative_list["splits"] = None + dates_to = str(filter_.get("DATES_TO", "natural")).lower() + members_to = str(filter_.get("MEMBERS_TO", "natural")).lower() + chunks_to = str(filter_.get("CHUNKS_TO", "natural")).lower() + splits_to = str(filter_.get("SPLITS_TO", "natural")).lower() + if not is_a_natural_relation: if dates_to == "natural": - associative_list["dates"] = [date2str(parent.date)] if parent.date is not None else date_list - + dates_to = "none" if members_to == "natural": - associative_list["members"] = [parent.member] if parent.member is not None else member_list + members_to = "none" if chunks_to == "natural": - associative_list["chunks"] = [parent.chunk] if parent.chunk is not None else chunk_list - parsed_parent_date = date2str(parent.date) if parent.date is not None else None - # Apply all filters to look if this parent is an appropriated candidate for the current_job - valid_dates = JobList._apply_filter(parsed_parent_date, dates_to, associative_list["dates"], "dates") - valid_members = JobList._apply_filter(parent.member, members_to, associative_list["members"],"members") - valid_chunks = JobList._apply_filter(parent.chunk, chunks_to, associative_list["chunks"], "chunks") - - if valid_dates and valid_members and valid_chunks: - if dates_to.find("?") != -1 or members_to.find("?") != -1 or chunks_to.find("?") != -1: - optional = True - return True,optional - return False,optional + chunks_to = "none" + if splits_to == "natural": + splits_to = "none" + if dates_to == "natural": + associative_list["dates"] = [date2str(parent.date)] if parent.date is not None else date_list + if members_to == "natural": + associative_list["members"] = [parent.member] if parent.member is not None else member_list + if chunks_to == "natural": + associative_list["chunks"] = [parent.chunk] if parent.chunk is not None else chunk_list + if splits_to == "natural": + associative_list["splits"] = [parent.split] if parent.split is not None else parent.splits + parsed_parent_date = date2str(parent.date) if parent.date is not None else None + # Apply all filters to look if this parent is an appropriated candidate for the current_job + valid_dates = JobList._apply_filter(parsed_parent_date, dates_to, associative_list["dates"], "dates") + valid_members = JobList._apply_filter(parent.member, members_to, associative_list["members"], "members") + valid_chunks = JobList._apply_filter(parent.chunk, chunks_to, associative_list["chunks"], "chunks") + valid_splits = JobList._apply_filter(parent.split, splits_to, associative_list["splits"], "splits") + if valid_dates and valid_members and valid_chunks and valid_splits: + for value in [dates_to, members_to, chunks_to, splits_to]: + if "?" in value: + return True, True + return True, False + return False,False @staticmethod def _manage_job_dependencies(dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys, dependencies, graph): @@ -585,71 +764,28 @@ class JobList(object): other_parents = dic_jobs.get_jobs(dependency.section, None, None, None) parents_jobs = dic_jobs.get_jobs(dependency.section, date, member, chunk) - #if dependency.sign in ["+", "-"]: - # natural_jobs = dic_jobs.get_jobs(dependency.section, date, member,chunk) - #else: - natural_jobs = dic_jobs.get_jobs(dependency.section, date, member,chunk) - if dependency.sign in ['?']: - optional_section = True - else: - optional_section = False - # Convert multi_array list into 1d list - if len(parents_jobs) > 0: - aux = [] - for p_split in parents_jobs: - if type(p_split) is not list: - aux.append(p_split) - else: - for aux_job in p_split: - aux.append(aux_job) - parents_jobs = aux - if len(natural_jobs) > 0: - aux = [] - for p_split in natural_jobs: - if type(p_split) is not list: - aux.append(p_split) - else: - for aux_job in p_split: - aux.append(aux_job) - natural_jobs = aux + natural_jobs = dic_jobs.get_jobs(dependency.section, date, member, chunk) all_parents = list(set(other_parents + parents_jobs)) # Get dates_to, members_to, chunks_to of the deepest level of the relationship. - filters_to_apply,optional_from = JobList._filter_current_job(job,copy.deepcopy(dependency.relationships)) - if len(filters_to_apply) == 0: - filters_to_apply.append({"DATES_TO": "natural", "MEMBERS_TO": "natural", "CHUNKS_TO": "natural"}) + filters_to_apply = JobList._filter_current_job(job,copy.deepcopy(dependency.relationships)) for parent in all_parents: - # Generic for all dependencies - if dependency.delay == -1 or chunk > dependency.delay: - if isinstance(parent, list): - if job.split is not None and len(str(job.split)) > 0: - parent = list(filter( - lambda _parent: _parent.split == job.split, parent)) - parent = parent[0] - else: - if dependency.splits is not None and len(str(dependency.splits)) > 0: - parent = [_parent for _parent in parent if _parent.split in dependency.splits] # If splits is not None, the job is a list of jobs if parent.name == job.name: continue - # Check if it is a natural relation based in autosubmit terms ( same date,member,chunk ). - if parent in natural_jobs and ((job.chunk is None or parent.chunk is None or parent.chunk <= job.chunk ) and (parent.split is None or job.split is None or parent.split <= job.split) ) : + # Check if it is a natural relation. The only difference is that a chunk can depend on a chunks <= than the current chunk + if parent in natural_jobs and (job.chunk is None or parent.chunk is None or parent.chunk <= job.chunk ): natural_relationship = True else: natural_relationship = False - if job.name.find("a002_20120101_0_2_SIM") != -1: - print("test") # Check if the current parent is a valid parent based on the dependencies set on expdef.conf - valid,optional_to = JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship,filters_to_apply) - if not valid: - continue - else: - pass + valid,optional = JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship,filters_to_apply) # If the parent is valid, add it to the graph - job.add_parent(parent) - JobList._add_edge(graph, job, parent) - # Could be more variables in the future - if optional_to or optional_from or optional_section: - job.add_edge_info(parent.name,special_variables={"optional":True}) + if valid: + job.add_parent(parent) + JobList._add_edge(graph, job, parent) + # Could be more variables in the future + if optional: + job.add_edge_info(parent.name,special_variables={"optional":True}) JobList.handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member, member_list, dependency.section, graph, other_parents) diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py new file mode 100644 index 0000000000000000000000000000000000000000..a08a4a73d713e0ce3bd103eb2bba5be1ab698edf --- /dev/null +++ b/test/unit/test_dependencies.py @@ -0,0 +1,258 @@ +import unittest + +import mock +from copy import deepcopy +from autosubmit.job.job_list import JobList +from autosubmit.job.job import Job +from autosubmit.job.job_common import Status +from datetime import datetime +class TestJobList(unittest.TestCase): + def setUp(self): + # Define common test case inputs here + self.relationships_dates = { + "OPTIONAL": False, + "DATES_FROM": { + "20020201": { + "MEMBERS_FROM": { + "fc2": { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL" + } + }, + "SPLITS_FROM": { + "ALL": { + "SPLITS_TO": "1" + } + } + } + } + } + self.relationships_dates_optional = deepcopy(self.relationships_dates) + self.relationships_dates_optional["DATES_FROM"]["20020201"]["MEMBERS_FROM"] = { "fc2?": { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", "CHUNKS_TO": "ALL", "SPLITS_TO": "5" } } + self.relationships_dates_optional["DATES_FROM"]["20020201"]["SPLITS_FROM"] = { "ALL": { "SPLITS_TO": "1?" } } + + self.relationships_members = { + "OPTIONAL": False, + "MEMBERS_FROM": { + "fc2": { + "SPLITS_FROM": { + "ALL": { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + } + } + } + } + self.relationships_chunks = { + "OPTIONAL": False, + "CHUNKS_FROM": { + "1": { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + } + } + self.relationships_chunks2 = { + "OPTIONAL": False, + "CHUNKS_FROM": { + "1": { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + }, + "2": { + "SPLITS_FROM": { + "5": { + "SPLITS_TO": "2" + } + } + } + } + } + + + self.relationships_splits = { + "OPTIONAL": False, + "SPLITS_FROM": { + "1": { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + } + } + + self.relationships_general = { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + # Create a mock Job object + self.mock_job = mock.MagicMock(spec=Job) + + # Set the attributes on the mock object + self.mock_job.name = "Job1" + self.mock_job.job_id = 1 + self.mock_job.status = Status.READY + self.mock_job.priority = 1 + self.mock_job.date = None + self.mock_job.member = None + self.mock_job.chunk = None + self.mock_job.split = None + def test_simple_dependency(self): + result_d = JobList._check_dates({}, self.mock_job) + result_m = JobList._check_members({}, self.mock_job) + result_c = JobList._check_chunks({}, self.mock_job) + result_s = JobList._check_splits({}, self.mock_job) + self.assertEqual(result_d, {}) + self.assertEqual(result_m, {}) + self.assertEqual(result_c, {}) + self.assertEqual(result_s, {}) + def test_check_dates_optional(self): + self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") + self.mock_job.member = "fc2" + self.mock_job.chunk = 1 + self.mock_job.split = 1 + result = JobList._check_dates(self.relationships_dates_optional, self.mock_job) + expected_output = { + "DATES_TO": "20020201?", + "MEMBERS_TO": "fc2?", + "CHUNKS_TO": "ALL?", + "SPLITS_TO": "1?" + } + self.assertEqual(result, expected_output) + def test_check_dates(self): + # Call the function to get the result + self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") + self.mock_job.member = "fc2" + self.mock_job.chunk = 1 + self.mock_job.split = 1 + result = JobList._check_dates(self.relationships_dates, self.mock_job) + expected_output = { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + self.assertEqual(result, expected_output) + self.mock_job.date = datetime.strptime("20020202", "%Y%m%d") + result = JobList._check_dates(self.relationships_dates, self.mock_job) + self.assertEqual(result, {}) + def test_check_members(self): + # Call the function to get the result + self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") + self.mock_job.member = "fc2" + + result = JobList._check_members(self.relationships_members, self.mock_job) + expected_output = { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + self.assertEqual(result, expected_output) + self.mock_job.member = "fc3" + result = JobList._check_members(self.relationships_members, self.mock_job) + self.assertEqual(result, {}) + + def test_check_splits(self): + # Call the function to get the result + + self.mock_job.split = 1 + result = JobList._check_splits(self.relationships_splits, self.mock_job) + expected_output = { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + self.assertEqual(result, expected_output) + self.mock_job.split = 2 + result = JobList._check_splits(self.relationships_splits, self.mock_job) + self.assertEqual(result, {}) + def test_check_chunks(self): + # Call the function to get the result + + self.mock_job.chunk = 1 + result = JobList._check_chunks(self.relationships_chunks, self.mock_job) + expected_output = { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + self.assertEqual(result, expected_output) + self.mock_job.chunk = 2 + result = JobList._check_chunks(self.relationships_chunks, self.mock_job) + self.assertEqual(result, {}) + # test splits_from + self.mock_job.split = 5 + result = JobList._check_chunks(self.relationships_chunks2, self.mock_job) + expected_output2 = { + "SPLITS_TO": "2" + } + self.assertEqual(result, expected_output2) + self.mock_job.split = 1 + result = JobList._check_chunks(self.relationships_chunks2, self.mock_job) + self.assertEqual(result, {}) + + def test_check_general(self): + # Call the function to get the result + + self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") + self.mock_job.member = "fc2" + self.mock_job.chunk = 1 + self.mock_job.split = 1 + result = JobList._filter_current_job(self.mock_job,self.relationships_general) + expected_output = { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + self.assertEqual(result, expected_output) + + + def test_valid_parent(self): + # Call the function to get the result + + date_list = ["20020201"] + member_list = ["fc1", "fc2", "fc3"] + chunk_list = [1, 2, 3] + is_a_natural_relation = False + # Filter_to values + filter_ = { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + # PArent job values + self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") + self.mock_job.member = "fc2" + self.mock_job.chunk = 1 + self.mock_job.split = 1 + result = JobList._valid_parent(self.mock_job, date_list, member_list, chunk_list, is_a_natural_relation, filter_) + # it returns a tuple, the first element is the result, the second is the optional flag + self.assertEqual(result, (True,False)) + filter_ = { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1?" + } + result = JobList._valid_parent(self.mock_job, date_list, member_list, chunk_list, is_a_natural_relation, filter_) + self.assertEqual(result, (True,True)) + + +if __name__ == '__main__': + unittest.main()