From 6b606972a34a221ac30131e0eda815cce40edb46 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 22 Jun 2023 16:00:20 +0200 Subject: [PATCH 01/68] first step --- autosubmit/job/job_list.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index edd67d1c5..215350ced 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -70,6 +70,7 @@ class JobList(object): self._persistence_file = "job_list_" + expid self._job_list = list() self._base_job_list = list() + self.jobs_edges = dict() self._expid = expid self._config = config self.experiment_data = as_conf.experiment_data @@ -732,6 +733,18 @@ class JobList(object): return True, True return True, False return False,False + + def _add_edge_info(self,job,parent): + """ + Special relations to be check in the update_list method + :param job: Current job + :param parent: parent jobs to check + :return: + """ + if job.name not in self.jobs_edges: + self.jobs_edges[job.name] = [] + else: + self.jobs_edges[job.name].append(parent) @staticmethod def _manage_job_dependencies(dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys, dependencies, graph): @@ -784,8 +797,17 @@ class JobList(object): job.add_parent(parent) JobList._add_edge(graph, job, parent) # Could be more variables in the future - if optional: - job.add_edge_info(parent.name,special_variables={"optional":True}) + # todo + checkpoint = False + if optional and checkpoint: + JobList._add_edge_info(job,parent) + job.add_edge_info(parent.name,special_variables={"optional":True,"checkpoint":True}) + if optional and not checkpoint: + #JobList._add_edge_info(job) + job.add_edge_info(parent.name, special_variables={"optional": True}) + if not optional and checkpoint: + JobList._add_edge_info(job,parent) + job.add_edge_info(parent.name, special_variables={"checkpoint": True}) JobList.handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member, member_list, dependency.section, graph, other_parents) -- GitLab From 07c0f010ff4965b3643df1499dfca5b8e2f2aa93 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 26 Jun 2023 11:51:20 +0200 Subject: [PATCH 02/68] adding checkpoints --- autosubmit/job/job.py | 22 ++++++++++++++++++++- autosubmit/job/job_common.py | 33 +++++++++++++++++++++++++++++--- autosubmit/job/job_list.py | 14 +++++++++----- autosubmit/platforms/platform.py | 22 +++++++++++++++++++++ 4 files changed, 82 insertions(+), 9 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 8f80f23d3..e1919b778 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -216,7 +216,9 @@ class Job(object): self.total_jobs = None self.max_waiting_jobs = None self.exclusive = "" - + self._check_point = "" + # internal + self.current_checkpoint_step = 0 @property @autosubmit_parameter(name='tasktype') def section(self): @@ -248,6 +250,23 @@ class Job(object): self._fail_count = value @property + @autosubmit_parameter(name='checkpoint') + def checkpoint(self): + """Generates a checkpoint step for this job based on job.type""" + return self._checkpoint + @checkpoint.setter + def checkpoint(self): + """Generates a checkpoint step for this job based on job.type""" + if self.type == Type.PYTHON: + self._checkpoint = "checkpoint()" + elif self.type == Type.R: + self._checkpoint = "checkpoint()" + else: # bash + self._checkpoint = "as_checkpoint" + + def get_checkpoint_files(self): + """Downloads checkpoint files from remote host. If they aren't already in local.""" + self.platform.get_checkpoint_files(self) @autosubmit_parameter(name='sdate') def sdate(self): """Current start date.""" @@ -1342,6 +1361,7 @@ class Job(object): return parameters def update_job_parameters(self,as_conf, parameters): + parameters["AS_CHECKPOINT"] = self.checkpoint parameters['JOBNAME'] = self.name parameters['FAIL_COUNT'] = str(self.fail_count) parameters['SDATE'] = self.sdate diff --git a/autosubmit/job/job_common.py b/autosubmit/job/job_common.py index 4d05d985c..042c6e330 100644 --- a/autosubmit/job/job_common.py +++ b/autosubmit/job/job_common.py @@ -128,6 +128,16 @@ class StatisticsSnippetBash: job_name_ptrn='%CURRENT_LOGDIR%/%JOBNAME%' echo $(date +%s) > ${job_name_ptrn}_STAT + ################### + # AS CHECKPOINT FUNCTION + ################### + # Creates a new checkpoint file upton call based on the current numbers of calls to the function + + AS_CHECKPOINT_CALLS=0 + function as_checkpoint { + AS_CHECKPOINT_CALLS=$((AS_CHECKPOINT_CALLS+1)) + touch ${job_name_ptrn}_CHECKPOINT_${AS_CHECKPOINT_CALLS} + } ################### # Autosubmit job ################### @@ -190,11 +200,19 @@ class StatisticsSnippetPython: stat_file = open(job_name_ptrn + '_STAT', 'w') stat_file.write('{0:.0f}\\n'.format(time.time())) stat_file.close() - - + + ################### + # Autosubmit Checkpoint + ################### + # Creates a new checkpoint file upton call based on the current numbers of calls to the function + AS_CHECKPOINT_CALLS = 0 + def as_checkpoint(): + AS_CHECKPOINT_CALLS = AS_CHECKPOINT_CALLS + 1 + open(job_name_ptrn + '_CHECKPOINT_' + str(AS_CHECKPOINT_CALLS), 'w').close() ################### # Autosubmit job ################### + """) @@ -254,7 +272,16 @@ class StatisticsSnippetR: fileConn<-file(paste(job_name_ptrn,"_STAT", sep = ''),"w") writeLines(toString(trunc(as.numeric(Sys.time()))), fileConn) close(fileConn) - + ################### + # Autosubmit Checkpoint + ################### + # Creates a new checkpoint file upton call based on the current numbers of calls to the function + AS_CHECKPOINT_CALLS = 0 + as_checkpoint <- function() { + AS_CHECKPOINT_CALLS <<- AS_CHECKPOINT_CALLS + 1 + fileConn<-file(paste(job_name_ptrn,"_CHECKPOINT_",AS_CHECKPOINT_CALLS, sep = ''),"w") + close(fileConn) + } ################### # Autosubmit job ################### diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 215350ced..43449ab6c 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -797,11 +797,13 @@ class JobList(object): job.add_parent(parent) JobList._add_edge(graph, job, parent) # Could be more variables in the future - # todo - checkpoint = False + # todo, default to TRUE for testing propouses + checkpoint = "!r" + #checkpoint = "!r1" + #checkpoint = "!r1,2,3" if optional and checkpoint: JobList._add_edge_info(job,parent) - job.add_edge_info(parent.name,special_variables={"optional":True,"checkpoint":True}) + job.add_edge_info(parent.name,special_variables={"optional":True,"checkpoint":checkpoint}) if optional and not checkpoint: #JobList._add_edge_info(job) job.add_edge_info(parent.name, special_variables={"optional": True}) @@ -1908,6 +1910,10 @@ class JobList(object): def parameters(self, value): self._parameters = value + def check_checkpoint(self, job, parent): + """ Check if a checkpoint step exists for this edge""" + return job.get_checkpoint_files(parent.name) + def update_list(self, as_conf, store_change=True, fromSetStatus=False, submitter=None, first_time=False): # type: (AutosubmitConfig, bool, bool, object, bool) -> bool """ @@ -2001,8 +2007,6 @@ class JobList(object): Log.debug( "Resetting sync job: {0} status to: WAITING for parents completion...".format( job.name)) - - Log.debug('Updating WAITING jobs') if not fromSetStatus: all_parents_completed = [] diff --git a/autosubmit/platforms/platform.py b/autosubmit/platforms/platform.py index 1f23cc6fc..e5c5a80d8 100644 --- a/autosubmit/platforms/platform.py +++ b/autosubmit/platforms/platform.py @@ -512,6 +512,28 @@ class Platform(object): (job_out_filename, job_err_filename) = remote_logs self.get_files([job_out_filename, job_err_filename], False, 'LOG_{0}'.format(exp_id)) + def get_checkpoint_files(self,job): + """ + Get all the checkpoint files + + :param job_name: name of the job + :type job_name: str + """ + from pathlib import Path + local_checkpoint_path = Path(f"{self.get_files_path()}/CHECKPOINT_{job.current_checkpoint_step}") + if Path(local_checkpoint_path).exists(): + return True + while self.check_file_exists(f'{job.name}_CHECKPOINT_{job.current_checkpoint_step}'): + # check if it exists locally + if not self.check_file_exists(f'{job.name}_CHECKPOINT_{job.current_checkpoint_step}', False): + if self.get_file('{0}_CHECKPOINT'.format(job.name), must_exist=False): + return True + else: + return False + else: + return False + + self.get_files(['{0}_checkpoint'.format(job_name), '{0}_checkpoint.json'.format(job_name)], False) def get_completed_files(self, job_name, retries=0, recovery=False, wrapper_failed=False): """ Get the COMPLETED file of the given job -- GitLab From 1d197bb8333e0f5a6e02b9a16c9d76bd8ea8f077 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 28 Jun 2023 12:00:55 +0200 Subject: [PATCH 03/68] almost done --- autosubmit/job/job.py | 4 +- autosubmit/job/job_list.py | 126 ++++++++++++++++++++++--------- autosubmit/platforms/platform.py | 39 +++++----- test/unit/test_dependencies.py | 42 ++++++++++- 4 files changed, 156 insertions(+), 55 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index e1919b778..9aa86cd29 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -219,6 +219,7 @@ class Job(object): self._check_point = "" # internal self.current_checkpoint_step = 0 + @property @autosubmit_parameter(name='tasktype') def section(self): @@ -584,6 +585,7 @@ class Job(object): :type value: HPCPlatform """ self._partition = value + @property def children(self): """ @@ -1514,7 +1516,7 @@ class Job(object): template_file.close() else: if self.type == Type.BASH: - template = 'sleep 5' + template = '%AS_CHECKPOINT%;sleep 320;%AS_CHECKPOINT%;sleep 320' elif self.type == Type.PYTHON2: template = 'time.sleep(5)' + "\n" elif self.type == Type.PYTHON3 or self.type == Type.PYTHON: diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 43449ab6c..acd47c0a3 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -275,8 +275,7 @@ class JobList(object): raise AutosubmitCritical("Some section jobs of the wrapper:{0} are not in the current job_list defined in jobs.conf".format(wrapper_section),7014,str(e)) - @staticmethod - def _add_dependencies(date_list, member_list, chunk_list, dic_jobs, graph, option="DEPENDENCIES"): + def _add_dependencies(self,date_list, member_list, chunk_list, dic_jobs, graph, option="DEPENDENCIES"): jobs_data = dic_jobs._jobs_data.get("JOBS",{}) for job_section in jobs_data.keys(): Log.debug("Adding dependencies for {0} jobs".format(job_section)) @@ -295,7 +294,7 @@ class JobList(object): dependencies_keys[dependency] = {} if dependencies_keys is None: dependencies_keys = {} - dependencies = JobList._manage_dependencies(dependencies_keys, dic_jobs, job_section) + dependencies = self._manage_dependencies(dependencies_keys, dic_jobs, job_section) for job in dic_jobs.get_jobs(job_section): num_jobs = 1 @@ -303,7 +302,7 @@ class JobList(object): num_jobs = len(job) for i in range(num_jobs): _job = job[i] if num_jobs > 1 else job - JobList._manage_job_dependencies(dic_jobs, _job, date_list, member_list, chunk_list, dependencies_keys, + self._manage_job_dependencies(dic_jobs, _job, date_list, member_list, chunk_list, dependencies_keys, dependencies, graph) pass @@ -423,24 +422,48 @@ class JobList(object): else: return False + @staticmethod + def _parse_checkpoint(data): + checkpoint = {"STATUS": None, "FROM_STEP": None} + data = data.lower() + if data[0] == "r": + checkpoint["STATUS"] = Status.RUNNING + if len(data) > 1: + checkpoint["FROM_STEP"] = data[1:] + else: + checkpoint["FROM_STEP"] = "1" + elif data[0] == "f": + checkpoint["STATUS"] = Status.FAILED + if len(data) > 1: + checkpoint["FROM_STEP"] = data[1:] + else: + checkpoint["FROM_STEP"] = "1" + elif data[0] == "q": + checkpoint["STATUS"] = Status.QUEUING + elif data[0] == "s": + checkpoint["STATUS"] = Status.SUBMITTED + return checkpoint @staticmethod - def _check_relationship(relationships,level_to_check,value_to_check): + def _check_relationship(relationships, level_to_check, value_to_check): """ Check if the current_job_value is included in the filter_value - :param relationship: current filter level to check. + :param relationships: current filter level to check. :param level_to_check: can be a date, member, chunk or split. :param value_to_check: Can be None, a date, a member, a chunk or a split. :return: """ filters = [] - for filter_range,filter_data in relationships.get(level_to_check,{}).items(): + for filter_range, filter_data in relationships.get(level_to_check, {}).items(): if not value_to_check or str(filter_range).upper() in "ALL" or str(value_to_check).upper() in str(filter_range).upper(): if filter_data: if "?" in filter_range: filter_data["OPTIONAL"] = True else: filter_data["OPTIONAL"] = relationships["OPTIONAL"] + if "!" in filter_range: + filter_data["CHECKPOINT"] = "!"+filter_range.split("!")[1] + #JobList._parse_checkpoint(filter_range.split("!")[1]) filters.append(filter_data) # Normalize the filter return if len(filters) == 0: @@ -486,22 +509,22 @@ class JobList(object): # Will enter, go recursivily to the similar methods and in the end it will do: # Will enter members_from, and obtain [{DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", CHUNKS_FROM{...}] if "MEMBERS_FROM" in filter: - filters_to_apply_m = JobList._check_members({"MEMBERS_FROM": (filter.pop("MEMBERS_FROM")),"OPTIONAL":optional}, current_job) + filters_to_apply_m = JobList._check_members({"MEMBERS_FROM": (filter.pop("MEMBERS_FROM")), "OPTIONAL":optional}, current_job) if len(filters_to_apply_m) > 0: filters_to_apply[i].update(filters_to_apply_m) # Will enter chunks_from, and obtain [{DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"] if "CHUNKS_FROM" in filter: - filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM")),"OPTIONAL":optional}, current_job) + filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM")), "OPTIONAL":optional}, current_job) if len(filters_to_apply_c) > 0 and len(filters_to_apply_c[0]) > 0: filters_to_apply[i].update(filters_to_apply_c) - #IGNORED + # IGNORED if "SPLITS_FROM" in filter: filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_s) > 0: filters_to_apply[i].update(filters_to_apply_s) # Unify filters from all filters_from where the current job is included to have a single SET of filters_to if optional: - for i,filter in enumerate(filters_to_apply): + for i in range(0, len(filters_to_apply)): filters_to_apply[i]["OPTIONAL"] = True filters_to_apply = JobList._unify_to_filters(filters_to_apply) # {DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"} @@ -517,19 +540,19 @@ class JobList(object): """ filters_to_apply = JobList._check_relationship(relationships, "MEMBERS_FROM", current_job.member) optional = False - for i,filter in enumerate(filters_to_apply): - optional = filter.pop("OPTIONAL", False) - if "CHUNKS_FROM" in filter: - filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM")),"OPTIONAL":optional}, current_job) + for i, filter_ in enumerate(filters_to_apply): + optional = filter_.pop("OPTIONAL", False) + if "CHUNKS_FROM" in filter_: + filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter_.pop("CHUNKS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_c) > 0: filters_to_apply[i].update(filters_to_apply_c) - if "SPLITS_FROM" in filter: - filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) + if "SPLITS_FROM" in filter_: + filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter_.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_s) > 0: filters_to_apply[i].update(filters_to_apply_s) if optional: - for i,filter in enumerate(filters_to_apply): + for i in range(0, len(filters_to_apply) > 0): filters_to_apply[i]["OPTIONAL"] = True filters_to_apply = JobList._unify_to_filters(filters_to_apply) return filters_to_apply @@ -584,11 +607,22 @@ class JobList(object): if aux: aux = aux.split(",") for element in aux: - if element.lower().strip("?") in ["natural","none"] and len(unified_filter[filter_type]) > 0: + # element is SECTION(alphanumeric) then ? or ! can figure and then ! or ? can figure + # Get only the first alphanumeric part + parsed_element = re.findall(r"[\w']+", element)[0].lower() + # Get the rest + data = element[len(parsed_element):] + if parsed_element in ["natural", "none"] and len(unified_filter[filter_type]) > 0: continue else: - if filter_to.get("OPTIONAL",False) and element[-1] != "?": - element += "?" + if filter_to.get("OPTIONAL", False) or "?" in data: + if "?" not in element: + element += "?" + if "!" in data: + element = parsed_element+data + elif filter_to.get("CHECKPOINT", None): + element = parsed_element+filter_to.get("CHECKPOINT", None) + unified_filter[filter_type].add(element) @staticmethod def _normalize_to_filters(filter_to,filter_type): @@ -631,8 +665,7 @@ class JobList(object): @staticmethod def _filter_current_job(current_job,relationships): - ''' - This function will filter the current job based on the relationships given + ''' This function will filter the current job based on the relationships given :param current_job: Current job to filter :param relationships: Relationships to apply :return: dict() with the filters to apply, or empty dict() if no filters to apply @@ -658,6 +691,8 @@ class JobList(object): if relationships is not None and len(relationships) > 0: if "OPTIONAL" not in relationships: relationships["OPTIONAL"] = False + if "CHECKPOINT" not in relationships: + relationships["CHECKPOINT"] = None # Look for a starting point, this can be if else becasue they're exclusive as a DATE_FROM can't be in a MEMBER_FROM and so on if "DATES_FROM" in relationships: filters_to_apply = JobList._check_dates(relationships, current_job) @@ -669,6 +704,7 @@ class JobList(object): filters_to_apply = JobList._check_splits(relationships, current_job) else: relationships.pop("OPTIONAL", None) + relationships.pop("CHECKPOINT", None) relationships.pop("CHUNKS_FROM", None) relationships.pop("MEMBERS_FROM", None) relationships.pop("DATES_FROM", None) @@ -745,8 +781,8 @@ class JobList(object): self.jobs_edges[job.name] = [] else: self.jobs_edges[job.name].append(parent) - @staticmethod - def _manage_job_dependencies(dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys, dependencies, + + def _manage_job_dependencies(self,dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys, dependencies, graph): ''' Manage the dependencies of a job @@ -795,20 +831,19 @@ class JobList(object): # If the parent is valid, add it to the graph if valid: job.add_parent(parent) - JobList._add_edge(graph, job, parent) + self._add_edge(graph, job, parent) # Could be more variables in the future # todo, default to TRUE for testing propouses - checkpoint = "!r" - #checkpoint = "!r1" - #checkpoint = "!r1,2,3" + # Do parse checkpoint + checkpoint= {"status":Status.RUNNING,"from_step":2} if optional and checkpoint: - JobList._add_edge_info(job,parent) + self._add_edge_info(job,parent) job.add_edge_info(parent.name,special_variables={"optional":True,"checkpoint":checkpoint}) if optional and not checkpoint: #JobList._add_edge_info(job) job.add_edge_info(parent.name, special_variables={"optional": True}) if not optional and checkpoint: - JobList._add_edge_info(job,parent) + self._add_edge_info(job,parent) job.add_edge_info(parent.name, special_variables={"checkpoint": True}) JobList.handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member, member_list, dependency.section, graph, other_parents) @@ -1914,6 +1949,18 @@ class JobList(object): """ Check if a checkpoint step exists for this edge""" return job.get_checkpoint_files(parent.name) + def check_checkpoint_parent_status(self): + """ + Check if all parents of a job have the correct status for checkpointing + :return: jobs that fullfill the special conditions """ + jobs_to_check = [] + for job, parent_to_check in self.jobs_edges.keys(): + checkpoint_info = job.edge_info.get(parent_to_check.name, {}).get("checkpoint", None) + if checkpoint_info: + if job.get_checkpoint_files(checkpoint_info["from_step"]): + if parent_to_check.status != checkpoint_info["status"]: + jobs_to_check.append(job) + return jobs_to_check def update_list(self, as_conf, store_change=True, fromSetStatus=False, submitter=None, first_time=False): # type: (AutosubmitConfig, bool, bool, object, bool) -> bool """ @@ -1936,7 +1983,7 @@ class JobList(object): write_log_status = False if not first_time: for job in self.get_failed(): - if self.jobs_data[job.section].get("RETRIALS",None) is None: + if self.jobs_data[job.section].get("RETRIALS", None) is None: retrials = int(as_conf.get_retrials()) else: retrials = int(job.retrials) @@ -1950,7 +1997,7 @@ class JobList(object): else: aux_job_delay = int(job.delay_retrials) - if self.jobs_data[job.section].get("DELAY_RETRY_TIME",None) or aux_job_delay <= 0: + if self.jobs_data[job.section].get("DELAY_RETRY_TIME", None) or aux_job_delay <= 0: delay_retry_time = str(as_conf.get_delay_retry_time()) else: delay_retry_time = job.retry_delay @@ -1958,7 +2005,7 @@ class JobList(object): retry_delay = job.fail_count * int(delay_retry_time[:-1]) + int(delay_retry_time[:-1]) elif "*" in delay_retry_time: retry_delay = int(delay_retry_time[1:]) - for retrial_amount in range(0,job.fail_count): + for retrial_amount in range(0, job.fail_count): retry_delay += retry_delay * 10 else: retry_delay = int(delay_retry_time) @@ -1985,6 +2032,17 @@ class JobList(object): job.status = Status.FAILED job.packed = False save = True + # Check checkpoint jobs, the status can be Ready, Running, Queuing + for job in self.check_checkpoint_parent_status(): + # Check if all jobs fullfill the conditions to a job be ready + tmp = [parent for parent in job.parents if parent.status == Status.COMPLETED or parent in self.jobs_edges[job] ] + if len(tmp) == len(job.parents): + job.status = Status.READY + job.id = None + job.packed = False + job.wrapper_type = None + save = True + Log.debug(f"Special condition fullfilled for job {job.name}") # if waiting jobs has all parents completed change its State to READY for job in self.get_completed(): if job.synchronize is not None and len(str(job.synchronize)) > 0: diff --git a/autosubmit/platforms/platform.py b/autosubmit/platforms/platform.py index e5c5a80d8..1689aec93 100644 --- a/autosubmit/platforms/platform.py +++ b/autosubmit/platforms/platform.py @@ -1,5 +1,6 @@ import locale import os +from pathlib import Path import traceback from autosubmit.job.job_common import Status @@ -512,28 +513,19 @@ class Platform(object): (job_out_filename, job_err_filename) = remote_logs self.get_files([job_out_filename, job_err_filename], False, 'LOG_{0}'.format(exp_id)) - def get_checkpoint_files(self,job): + def get_checkpoint_files(self, job, step_to_check): """ - Get all the checkpoint files - - :param job_name: name of the job - :type job_name: str + Get all the checkpoint files of the given job """ - from pathlib import Path - local_checkpoint_path = Path(f"{self.get_files_path()}/CHECKPOINT_{job.current_checkpoint_step}") - if Path(local_checkpoint_path).exists(): - return True - while self.check_file_exists(f'{job.name}_CHECKPOINT_{job.current_checkpoint_step}'): - # check if it exists locally - if not self.check_file_exists(f'{job.name}_CHECKPOINT_{job.current_checkpoint_step}', False): - if self.get_file('{0}_CHECKPOINT'.format(job.name), must_exist=False): - return True - else: - return False - else: - return False - self.get_files(['{0}_checkpoint'.format(job_name), '{0}_checkpoint.json'.format(job_name)], False) + if step_to_check >= job.current_checkpoint_step: + return True + local_checkpoint_path = Path(f"{self.get_files_path()}/CHECKPOINT_{step_to_check}") + while self.check_file_exists(local_checkpoint_path): + job.current_checkpoint_step += 1 + self.remove_checkpoint_file(local_checkpoint_path) + else: + return False def get_completed_files(self, job_name, retries=0, recovery=False, wrapper_failed=False): """ Get the COMPLETED file of the given job @@ -604,6 +596,15 @@ class Platform(object): Log.debug('{0} been removed', filename) return True return False + def remove_checkpoint_file(self, filename): + """ + Removes *CHECKPOINT* files from remote + + :param job_name: name of job to check + :return: True if successful, False otherwise + """ + if self.check_file_exists(filename): + self.delete_file(filename) def check_file_exists(self, src, wrapper_failed=False): return True diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index a08a4a73d..0af18081b 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -11,6 +11,7 @@ class TestJobList(unittest.TestCase): # Define common test case inputs here self.relationships_dates = { "OPTIONAL": False, + "CHECKPOINT": None, "DATES_FROM": { "20020201": { "MEMBERS_FROM": { @@ -34,6 +35,7 @@ class TestJobList(unittest.TestCase): self.relationships_members = { "OPTIONAL": False, + "CHECKPOINT": None, "MEMBERS_FROM": { "fc2": { "SPLITS_FROM": { @@ -49,6 +51,7 @@ class TestJobList(unittest.TestCase): } self.relationships_chunks = { "OPTIONAL": False, + "CHECKPOINT": None, "CHUNKS_FROM": { "1": { "DATES_TO": "20020201", @@ -60,6 +63,7 @@ class TestJobList(unittest.TestCase): } self.relationships_chunks2 = { "OPTIONAL": False, + "CHECKPOINT": None, "CHUNKS_FROM": { "1": { "DATES_TO": "20020201", @@ -77,9 +81,9 @@ class TestJobList(unittest.TestCase): } } - self.relationships_splits = { "OPTIONAL": False, + "CHECKPOINT": None, "SPLITS_FROM": { "1": { "DATES_TO": "20020201", @@ -108,6 +112,42 @@ class TestJobList(unittest.TestCase): self.mock_job.member = None self.mock_job.chunk = None self.mock_job.split = None + + def test_parse_checkpoint(self): + data = "r2" + correct = {"FROM_STEP": '2', "STATUS":Status.RUNNING} + result = JobList._parse_checkpoint(data) + self.assertEqual(result, correct) + data = "r" + correct = {"FROM_STEP": '1', "STATUS":Status.RUNNING} + result = JobList._parse_checkpoint(data) + self.assertEqual(result, correct) + data = "f2" + correct = {"FROM_STEP": '2', "STATUS":Status.FAILED} + result = JobList._parse_checkpoint(data) + self.assertEqual(result, correct) + data = "f" + correct = {"FROM_STEP": '1', "STATUS":Status.FAILED} + result = JobList._parse_checkpoint(data) + self.assertEqual(result, correct) + data = "s" + correct = {"FROM_STEP": None, "STATUS":Status.SUBMITTED} + result = JobList._parse_checkpoint(data) + self.assertEqual(result, correct) + data = "s2" + correct = {"FROM_STEP": None, "STATUS":Status.SUBMITTED} + result = JobList._parse_checkpoint(data) + self.assertEqual(result, correct) + data = "q" + correct = {"FROM_STEP": None, "STATUS":Status.QUEUING} + result = JobList._parse_checkpoint(data) + self.assertEqual(result, correct) + data = "q2" + correct = {"FROM_STEP": None, "STATUS":Status.QUEUING} + result = JobList._parse_checkpoint(data) + self.assertEqual(result, correct) + + def test_simple_dependency(self): result_d = JobList._check_dates({}, self.mock_job) result_m = JobList._check_members({}, self.mock_job) -- GitLab From 1d5805dda7678ebb1ffa4e768a6a7ad60df5fbf4 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 3 Jul 2023 12:06:35 +0200 Subject: [PATCH 04/68] start_conditions fix checkpoint_files --- autosubmit/job/job.py | 14 +++++++++----- autosubmit/job/job_list.py | 25 +++++++++++++++---------- autosubmit/platforms/platform.py | 23 ++++++++++++++++------- 3 files changed, 40 insertions(+), 22 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 9aa86cd29..55093af66 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -186,6 +186,7 @@ class Job(object): self.x11 = False self._local_logs = ('', '') self._remote_logs = ('', '') + self._checkpoint = None self.script_name = self.name + ".cmd" self.status = status self.prev_status = status @@ -216,7 +217,6 @@ class Job(object): self.total_jobs = None self.max_waiting_jobs = None self.exclusive = "" - self._check_point = "" # internal self.current_checkpoint_step = 0 @@ -265,9 +265,13 @@ class Job(object): else: # bash self._checkpoint = "as_checkpoint" - def get_checkpoint_files(self): - """Downloads checkpoint files from remote host. If they aren't already in local.""" - self.platform.get_checkpoint_files(self) + def get_checkpoint_files(self,steps): + """ + Downloads checkpoint files from remote host. If they aren't already in local. + :param steps: list of steps to download + :return: the max step downloaded + """ + return self.platform.get_checkpoint_files(self,steps) @autosubmit_parameter(name='sdate') def sdate(self): """Current start date.""" @@ -706,7 +710,7 @@ class Job(object): """ self.children.add(new_child) - def add_edge_info(self,parent_name, special_variables): + def add_edge_info(self,parent_name, special_variables={}): """ Adds edge information to the job diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index acd47c0a3..aefe961f0 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -770,7 +770,7 @@ class JobList(object): return True, False return False,False - def _add_edge_info(self,job,parent): + def _add_edge_info(self,job,special_variables): """ Special relations to be check in the update_list method :param job: Current job @@ -778,9 +778,12 @@ class JobList(object): :return: """ if job.name not in self.jobs_edges: - self.jobs_edges[job.name] = [] + self.jobs_edges[job] = special_variables.get("FROMSTEP", 0) else: - self.jobs_edges[job.name].append(parent) + if special_variables.get("FROMSTEP", 0) > self.jobs_edges[job]: + self.jobs_edges[job] = special_variables.get("FROMSTEP", 0) + + def _manage_job_dependencies(self,dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys, dependencies, graph): @@ -837,7 +840,7 @@ class JobList(object): # Do parse checkpoint checkpoint= {"status":Status.RUNNING,"from_step":2} if optional and checkpoint: - self._add_edge_info(job,parent) + self._add_edge_info(job,special_variables={"optional":True,"checkpoint":checkpoint}) job.add_edge_info(parent.name,special_variables={"optional":True,"checkpoint":checkpoint}) if optional and not checkpoint: #JobList._add_edge_info(job) @@ -1954,12 +1957,14 @@ class JobList(object): Check if all parents of a job have the correct status for checkpointing :return: jobs that fullfill the special conditions """ jobs_to_check = [] - for job, parent_to_check in self.jobs_edges.keys(): - checkpoint_info = job.edge_info.get(parent_to_check.name, {}).get("checkpoint", None) - if checkpoint_info: - if job.get_checkpoint_files(checkpoint_info["from_step"]): - if parent_to_check.status != checkpoint_info["status"]: - jobs_to_check.append(job) + for job, checkpoint_step in self.jobs_edges.items(): + if checkpoint_step > 0: + max_step = job.get_checkpoint_files(checkpoint_step) + else: + max_step = None + for parent in parent_to_check: #if checkpoint_info: + if parent.status != checkpoint_info["status"]: + jobs_to_check.append(job) return jobs_to_check def update_list(self, as_conf, store_change=True, fromSetStatus=False, submitter=None, first_time=False): # type: (AutosubmitConfig, bool, bool, object, bool) -> bool diff --git a/autosubmit/platforms/platform.py b/autosubmit/platforms/platform.py index 1689aec93..ba355ebb3 100644 --- a/autosubmit/platforms/platform.py +++ b/autosubmit/platforms/platform.py @@ -513,19 +513,28 @@ class Platform(object): (job_out_filename, job_err_filename) = remote_logs self.get_files([job_out_filename, job_err_filename], False, 'LOG_{0}'.format(exp_id)) - def get_checkpoint_files(self, job, step_to_check): + def get_checkpoint_files(self, job, max_step): """ - Get all the checkpoint files of the given job + Get all the checkpoint files of a job + :param job: Get the checkpoint files + :type job: Job + :param max_step: max step possible + :type max_step: int """ - if step_to_check >= job.current_checkpoint_step: - return True - local_checkpoint_path = Path(f"{self.get_files_path()}/CHECKPOINT_{step_to_check}") - while self.check_file_exists(local_checkpoint_path): + if job.current_checkpoint_step >= max_step: + return job.current_checkpoint_step + local_checkpoint_path = Path(f"{self.get_files_path()}/CHECKPOINT_{job.current_checkpoint_step}") + self.get_file(local_checkpoint_path, False, ignore_log=True) + while self.check_file_exists(local_checkpoint_path) and job.current_checkpoint_step <= max_step: + self.remove_checkpoint_file(local_checkpoint_path) job.current_checkpoint_step += 1 + local_checkpoint_path = Path(f"{self.get_files_path()}/CHECKPOINT_{job.current_checkpoint_step}") + self.get_file(local_checkpoint_path, False, ignore_log=True) self.remove_checkpoint_file(local_checkpoint_path) else: - return False + self.remove_checkpoint_file(local_checkpoint_path) + return job.current_checkpoint_step def get_completed_files(self, job_name, retries=0, recovery=False, wrapper_failed=False): """ Get the COMPLETED file of the given job -- GitLab From 60274b4bc2007353064eb49c37ec1a191f0dea28 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 4 Jul 2023 17:24:27 +0200 Subject: [PATCH 05/68] added step in [] add STATUS and FROM_STEP syntax Changed ? meaning --- autosubmit/job/job.py | 5 +- autosubmit/job/job_list.py | 168 +++++++++++++++++-------------- autosubmit/platforms/platform.py | 14 +-- 3 files changed, 103 insertions(+), 84 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 55093af66..d426df57e 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -710,7 +710,7 @@ class Job(object): """ self.children.add(new_child) - def add_edge_info(self,parent_name, special_variables={}): + def add_edge_info(self, parent_name, special_variables): """ Adds edge information to the job @@ -722,8 +722,9 @@ class Job(object): if parent_name not in self.edge_info: self.edge_info[parent_name] = special_variables else: + #TODO self.edge_info[parent_name].update(special_variables) - pass + def delete_parent(self, parent): """ Remove a parent from the job diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index aefe961f0..198002cb8 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -24,6 +24,7 @@ import pickle import traceback import math import copy +from collections import defaultdict from time import localtime, strftime, mktime from shutil import move from autosubmit.job.job import Job @@ -55,7 +56,6 @@ def threaded(fn): return thread return wrapper - class JobList(object): """ Class to manage the list of jobs to be run by autosubmit @@ -70,7 +70,10 @@ class JobList(object): self._persistence_file = "job_list_" + expid self._job_list = list() self._base_job_list = list() - self.jobs_edges = dict() + # convert job_edges_structure function to lambda + self.jobs_edges = {} + #"now str" + self._expid = expid self._config = config self.experiment_data = as_conf.experiment_data @@ -444,26 +447,78 @@ class JobList(object): checkpoint["STATUS"] = Status.SUBMITTED return checkpoint + @staticmethod + def _parse_filter_to_check(value_to_check): + """ + Parse the filter to check and return the value to check. + Selection process: + value_to_check can be: + a range: [0:], [:N], [0:N], [:-1], [0:N:M] ... + a value: N + a list of values : 0,2,4,5,7,10 ... + a range with step: [0::M], [::2], [0::3], [::3] ... + :param value_to_check: value to check. + :return: parsed value to check. + """ + # regex + value_to_check = str(value_to_check).upper() + if value_to_check is None: + return None + elif value_to_check == "ALL": + return "ALL" + elif value_to_check == "NONE": + return None + elif value_to_check == 1: + # range + if value_to_check[0] == ":": + # [:N] + return slice(None, int(value_to_check[1:])) + elif value_to_check[-1] == ":": + # [N:] + return slice(int(value_to_check[:-1]), None) + else: + # [N:M] + return slice(int(value_to_check.split(":")[0]), int(value_to_check.split(":")[1])) + elif value_to_check.count(":") == 2: +# range with step + if value_to_check[0] == ":": + # [::M] + return slice(None, None, int(value_to_check[2:])) + elif value_to_check[-1] == ":": + # [N::] + return slice(int(value_to_check[:-2]), None, None) + else: + # [N::M] + return slice(int(value_to_check.split(":")[0]), None, int(value_to_check.split(":")[2])) + elif "," in value_to_check: + # list + return value_to_check.split(",") + else: + # value + return value_to_check + + + + @staticmethod def _check_relationship(relationships, level_to_check, value_to_check): """ Check if the current_job_value is included in the filter_value :param relationships: current filter level to check. - :param level_to_check: can be a date, member, chunk or split. + :param level_to_check: Can be date_from, member_from, chunk_from, split_from. :param value_to_check: Can be None, a date, a member, a chunk or a split. :return: """ filters = [] - for filter_range, filter_data in relationships.get(level_to_check, {}).items(): - if not value_to_check or str(filter_range).upper() in "ALL" or str(value_to_check).upper() in str(filter_range).upper(): - if filter_data: - if "?" in filter_range: - filter_data["OPTIONAL"] = True - else: - filter_data["OPTIONAL"] = relationships["OPTIONAL"] - if "!" in filter_range: - filter_data["CHECKPOINT"] = "!"+filter_range.split("!")[1] - #JobList._parse_checkpoint(filter_range.split("!")[1]) + relationship = relationships.get(level_to_check, {}) + status = relationship.pop("STATUS", relationships.get("STATUS", None)) + from_step = relationship.pop("FROM_STEP", relationships.get("FROM_STEP", None)) + for filter_range, filter_data in relationship.items(): + if not value_to_check or str(value_to_check).upper() in str(JobList._parse_filter_to_check(filter_range)).upper(): + if not filter_data.get("STATUS", None): + filter_data["STATUS"] = status + if not filter_data.get("FROM_STEP", None): + filter_data["FROM_STEP"] = from_step filters.append(filter_data) # Normalize the filter return if len(filters) == 0: @@ -478,7 +533,7 @@ class JobList(object): :param current_job: Current job to check. :return: filters_to_apply """ - optional = False + filters_to_apply = JobList._check_relationship(relationships, "DATES_FROM", date2str(current_job.date)) # there could be multiple filters that apply... per example # Current task date is 20020201, and member is fc2 @@ -503,29 +558,25 @@ class JobList(object): # [{MEMBERS_FROM{..},CHUNKS_FROM{...}},{MEMBERS_FROM{..},SPLITS_FROM{...}}] for i,filter in enumerate(filters_to_apply): # {MEMBERS_FROM{..},CHUNKS_FROM{...}} I want too look ALL filters not only one, but I want to go recursivily until get the _TO filter - optional = filter.pop("OPTIONAL", False) # This is not an if_else, because the current level ( dates ) could have two different filters. # Second case commented: ( date_from 20020201 ) # Will enter, go recursivily to the similar methods and in the end it will do: # Will enter members_from, and obtain [{DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", CHUNKS_FROM{...}] if "MEMBERS_FROM" in filter: - filters_to_apply_m = JobList._check_members({"MEMBERS_FROM": (filter.pop("MEMBERS_FROM")), "OPTIONAL":optional}, current_job) + filters_to_apply_m = JobList._check_members({"MEMBERS_FROM": (filter.pop("MEMBERS_FROM"))}, current_job) if len(filters_to_apply_m) > 0: filters_to_apply[i].update(filters_to_apply_m) # Will enter chunks_from, and obtain [{DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"] if "CHUNKS_FROM" in filter: - filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM")), "OPTIONAL":optional}, current_job) + filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM"))}, current_job) if len(filters_to_apply_c) > 0 and len(filters_to_apply_c[0]) > 0: filters_to_apply[i].update(filters_to_apply_c) # IGNORED if "SPLITS_FROM" in filter: - filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) + filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM"))}, current_job) if len(filters_to_apply_s) > 0: filters_to_apply[i].update(filters_to_apply_s) # Unify filters from all filters_from where the current job is included to have a single SET of filters_to - if optional: - for i in range(0, len(filters_to_apply)): - filters_to_apply[i]["OPTIONAL"] = True filters_to_apply = JobList._unify_to_filters(filters_to_apply) # {DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"} return filters_to_apply @@ -539,21 +590,15 @@ class JobList(object): :return: filters_to_apply """ filters_to_apply = JobList._check_relationship(relationships, "MEMBERS_FROM", current_job.member) - optional = False for i, filter_ in enumerate(filters_to_apply): - optional = filter_.pop("OPTIONAL", False) if "CHUNKS_FROM" in filter_: - filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter_.pop("CHUNKS_FROM")),"OPTIONAL":optional}, current_job) + filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter_.pop("CHUNKS_FROM"))}, current_job) if len(filters_to_apply_c) > 0: filters_to_apply[i].update(filters_to_apply_c) - if "SPLITS_FROM" in filter_: - filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter_.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) + filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter_.pop("SPLITS_FROM"))}, current_job) if len(filters_to_apply_s) > 0: filters_to_apply[i].update(filters_to_apply_s) - if optional: - for i in range(0, len(filters_to_apply) > 0): - filters_to_apply[i]["OPTIONAL"] = True filters_to_apply = JobList._unify_to_filters(filters_to_apply) return filters_to_apply @@ -565,17 +610,13 @@ class JobList(object): :param current_job: Current job to check. :return: filters_to_apply """ - optional = False + filters_to_apply = JobList._check_relationship(relationships, "CHUNKS_FROM", current_job.chunk) for i,filter in enumerate(filters_to_apply): - optional = filter.pop("OPTIONAL", False) if "SPLITS_FROM" in filter: - filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) + filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM"))}, current_job) if len(filters_to_apply_s) > 0: filters_to_apply[i].update(filters_to_apply_s) - if optional: - for i,filter in enumerate(filters_to_apply): - filters_to_apply[i]["OPTIONAL"] = True filters_to_apply = JobList._unify_to_filters(filters_to_apply) return filters_to_apply @@ -607,7 +648,6 @@ class JobList(object): if aux: aux = aux.split(",") for element in aux: - # element is SECTION(alphanumeric) then ? or ! can figure and then ! or ? can figure # Get only the first alphanumeric part parsed_element = re.findall(r"[\w']+", element)[0].lower() # Get the rest @@ -615,14 +655,8 @@ class JobList(object): if parsed_element in ["natural", "none"] and len(unified_filter[filter_type]) > 0: continue else: - if filter_to.get("OPTIONAL", False) or "?" in data: - if "?" not in element: - element += "?" - if "!" in data: - element = parsed_element+data - elif filter_to.get("CHECKPOINT", None): - element = parsed_element+filter_to.get("CHECKPOINT", None) - + if "?" not in element: + element += data unified_filter[filter_type].add(element) @staticmethod def _normalize_to_filters(filter_to,filter_type): @@ -649,18 +683,20 @@ class JobList(object): """ unified_filter = {"DATES_TO": set(), "MEMBERS_TO": set(), "CHUNKS_TO": set(), "SPLITS_TO": set()} for filter_to in filter_to_apply: + if "STATUS" not in unified_filter and filter_to.get("STATUS", None): + unified_filter["STATUS"] = filter_to["STATUS"] + if "FROM_STEP" not in unified_filter and filter_to.get("FROM_STEP", None): + unified_filter["FROM_STEP"] = filter_to["FROM_STEP"] if len(filter_to) > 0: JobList._unify_to_filter(unified_filter,filter_to,"DATES_TO") JobList._unify_to_filter(unified_filter,filter_to,"MEMBERS_TO") JobList._unify_to_filter(unified_filter,filter_to,"CHUNKS_TO") JobList._unify_to_filter(unified_filter,filter_to,"SPLITS_TO") - filter_to.pop("OPTIONAL", None) JobList._normalize_to_filters(unified_filter,"DATES_TO") JobList._normalize_to_filters(unified_filter,"MEMBERS_TO") JobList._normalize_to_filters(unified_filter,"CHUNKS_TO") JobList._normalize_to_filters(unified_filter,"SPLITS_TO") - return unified_filter @staticmethod @@ -689,10 +725,6 @@ class JobList(object): filters_to_apply = {} # Check if filter_from-filter_to relationship is set if relationships is not None and len(relationships) > 0: - if "OPTIONAL" not in relationships: - relationships["OPTIONAL"] = False - if "CHECKPOINT" not in relationships: - relationships["CHECKPOINT"] = None # Look for a starting point, this can be if else becasue they're exclusive as a DATE_FROM can't be in a MEMBER_FROM and so on if "DATES_FROM" in relationships: filters_to_apply = JobList._check_dates(relationships, current_job) @@ -764,24 +796,22 @@ class JobList(object): valid_chunks = JobList._apply_filter(parent.chunk, chunks_to, associative_list["chunks"], "chunks") valid_splits = JobList._apply_filter(parent.split, splits_to, associative_list["splits"], "splits") if valid_dates and valid_members and valid_chunks and valid_splits: - for value in [dates_to, members_to, chunks_to, splits_to]: - if "?" in value: - return True, True - return True, False - return False,False + return True + return False - def _add_edge_info(self,job,special_variables): + def _add_edge_info(self,job,parent,special_status): """ Special relations to be check in the update_list method :param job: Current job :param parent: parent jobs to check :return: """ - if job.name not in self.jobs_edges: - self.jobs_edges[job] = special_variables.get("FROMSTEP", 0) + if job not in self.jobs_edges: + self.jobs_edges[job] = {} + if special_status not in self.jobs_edges[job]: + self.jobs_edges[job][special_status] = [] else: - if special_variables.get("FROMSTEP", 0) > self.jobs_edges[job]: - self.jobs_edges[job] = special_variables.get("FROMSTEP", 0) + self.jobs_edges[job][special_status].append(parent) @@ -830,24 +860,15 @@ class JobList(object): else: natural_relationship = False # Check if the current parent is a valid parent based on the dependencies set on expdef.conf - valid,optional = JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship,filters_to_apply) # If the parent is valid, add it to the graph - if valid: + if JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship,filters_to_apply): job.add_parent(parent) self._add_edge(graph, job, parent) # Could be more variables in the future - # todo, default to TRUE for testing propouses # Do parse checkpoint - checkpoint= {"status":Status.RUNNING,"from_step":2} - if optional and checkpoint: - self._add_edge_info(job,special_variables={"optional":True,"checkpoint":checkpoint}) - job.add_edge_info(parent.name,special_variables={"optional":True,"checkpoint":checkpoint}) - if optional and not checkpoint: - #JobList._add_edge_info(job) - job.add_edge_info(parent.name, special_variables={"optional": True}) - if not optional and checkpoint: - self._add_edge_info(job,parent) - job.add_edge_info(parent.name, special_variables={"checkpoint": True}) + if filters_to_apply.get("STATUS",None): + self._add_edge_info(job, parent, filters_to_apply["STATUS"]) + job.add_edge_info(parent.name,{filters_to_apply["STATUS"],filters_to_apply["FROM_STEP"]}) JobList.handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member, member_list, dependency.section, graph, other_parents) @@ -1957,6 +1978,7 @@ class JobList(object): Check if all parents of a job have the correct status for checkpointing :return: jobs that fullfill the special conditions """ jobs_to_check = [] + todo for job, checkpoint_step in self.jobs_edges.items(): if checkpoint_step > 0: max_step = job.get_checkpoint_files(checkpoint_step) diff --git a/autosubmit/platforms/platform.py b/autosubmit/platforms/platform.py index ba355ebb3..84012944b 100644 --- a/autosubmit/platforms/platform.py +++ b/autosubmit/platforms/platform.py @@ -524,16 +524,12 @@ class Platform(object): if job.current_checkpoint_step >= max_step: return job.current_checkpoint_step - local_checkpoint_path = Path(f"{self.get_files_path()}/CHECKPOINT_{job.current_checkpoint_step}") - self.get_file(local_checkpoint_path, False, ignore_log=True) - while self.check_file_exists(local_checkpoint_path) and job.current_checkpoint_step <= max_step: - self.remove_checkpoint_file(local_checkpoint_path) + remote_checkpoint_path = f"{self.get_files_path()}/CHECKPOINT_" + self.get_file(remote_checkpoint_path+job.current_checkpoint_step, False, ignore_log=True) + while self.check_file_exists(remote_checkpoint_path+job.current_checkpoint_step) and job.current_checkpoint_step < max_step: + self.remove_checkpoint_file(remote_checkpoint_path+job.current_checkpoint_step) job.current_checkpoint_step += 1 - local_checkpoint_path = Path(f"{self.get_files_path()}/CHECKPOINT_{job.current_checkpoint_step}") - self.get_file(local_checkpoint_path, False, ignore_log=True) - self.remove_checkpoint_file(local_checkpoint_path) - else: - self.remove_checkpoint_file(local_checkpoint_path) + self.get_file(remote_checkpoint_path+job.current_checkpoint_step, False, ignore_log=True) return job.current_checkpoint_step def get_completed_files(self, job_name, retries=0, recovery=False, wrapper_failed=False): """ -- GitLab From 4afa1ea813c02cb409a708e01d6b7b855285b29a Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 5 Jul 2023 16:52:10 +0200 Subject: [PATCH 06/68] all working, tODO clean old conditional code remake tests dependencies --- autosubmit/job/job.py | 18 ++++---- autosubmit/job/job_list.py | 71 ++++++++++++++++++++------------ autosubmit/platforms/platform.py | 18 ++++---- 3 files changed, 62 insertions(+), 45 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index d426df57e..8a24ebc76 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -219,6 +219,7 @@ class Job(object): self.exclusive = "" # internal self.current_checkpoint_step = 0 + self.max_checkpoint_step = 0 @property @autosubmit_parameter(name='tasktype') @@ -265,13 +266,13 @@ class Job(object): else: # bash self._checkpoint = "as_checkpoint" - def get_checkpoint_files(self,steps): + def get_checkpoint_files(self): """ Downloads checkpoint files from remote host. If they aren't already in local. :param steps: list of steps to download :return: the max step downloaded """ - return self.platform.get_checkpoint_files(self,steps) + return self.platform.get_checkpoint_files(self) @autosubmit_parameter(name='sdate') def sdate(self): """Current start date.""" @@ -710,20 +711,19 @@ class Job(object): """ self.children.add(new_child) - def add_edge_info(self, parent_name, special_variables): + def add_edge_info(self, parent, special_variables): """ Adds edge information to the job - :param parent_name: parent name - :type parent_name: str + :param parent: parent job + :type parent: Job :param special_variables: special variables :type special_variables: dict """ - if parent_name not in self.edge_info: - self.edge_info[parent_name] = special_variables + if special_variables["STATUS"] not in self.edge_info: + self.edge_info[special_variables["STATUS"]] = [(parent, special_variables.get("FROM_STEP", 0))] else: - #TODO - self.edge_info[parent_name].update(special_variables) + self.edge_info[special_variables["STATUS"]].append((parent, special_variables.get("FROM_STEP", 0))) def delete_parent(self, parent): """ diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 198002cb8..2f9f8d7b9 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -70,10 +70,7 @@ class JobList(object): self._persistence_file = "job_list_" + expid self._job_list = list() self._base_job_list = list() - # convert job_edges_structure function to lambda self.jobs_edges = {} - #"now str" - self._expid = expid self._config = config self.experiment_data = as_conf.experiment_data @@ -799,19 +796,19 @@ class JobList(object): return True return False - def _add_edge_info(self,job,parent,special_status): + def _add_edge_info(self,job,special_status): """ Special relations to be check in the update_list method :param job: Current job :param parent: parent jobs to check :return: """ - if job not in self.jobs_edges: - self.jobs_edges[job] = {} - if special_status not in self.jobs_edges[job]: - self.jobs_edges[job][special_status] = [] - else: - self.jobs_edges[job][special_status].append(parent) + if special_status not in self.jobs_edges: + self.jobs_edges[special_status] = set() + self.jobs_edges[special_status].add(job) + if "ALL" not in self.jobs_edges: + self.jobs_edges["ALL"] = set() + self.jobs_edges["ALL"].add(job) @@ -832,6 +829,7 @@ class JobList(object): parsed_date_list = [] for dat in date_list: parsed_date_list.append(date2str(dat)) + special_conditions = dict() for key in dependencies_keys: dependency = dependencies.get(key,None) if dependency is None: @@ -850,6 +848,12 @@ class JobList(object): all_parents = list(set(other_parents + parents_jobs)) # Get dates_to, members_to, chunks_to of the deepest level of the relationship. filters_to_apply = JobList._filter_current_job(job,copy.deepcopy(dependency.relationships)) + if "?" in [filters_to_apply.get("SPLITS_TO",""),filters_to_apply.get("DATES_TO",""),filters_to_apply.get("MEMBERS_TO",""),filters_to_apply.get("CHUNKS_TO","")]: + only_marked_status = True + else: + only_marked_status = False + special_conditions["STATUS"] = filters_to_apply.pop("STATUS", None) + special_conditions["FROM_STEP"] = filters_to_apply.pop("FROM_STEP", None) for parent in all_parents: # If splits is not None, the job is a list of jobs if parent.name == job.name: @@ -861,14 +865,25 @@ class JobList(object): natural_relationship = False # Check if the current parent is a valid parent based on the dependencies set on expdef.conf # If the parent is valid, add it to the graph + if JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship,filters_to_apply): job.add_parent(parent) self._add_edge(graph, job, parent) # Could be more variables in the future # Do parse checkpoint - if filters_to_apply.get("STATUS",None): - self._add_edge_info(job, parent, filters_to_apply["STATUS"]) - job.add_edge_info(parent.name,{filters_to_apply["STATUS"],filters_to_apply["FROM_STEP"]}) + if special_conditions.get("STATUS",None): + if only_marked_status: + if str(job.split)+"?" in filters_to_apply.get("SPLITS_TO","") or str(job.chunk)+"?" in filters_to_apply.get("CHUNKS_TO","") or str(job.member)+"?" in filters_to_apply.get("MEMBERS_TO","") or str(job.date)+"?" in filters_to_apply.get("DATES_TO",""): + selected = False + else: + selected = True + else: + selected = True + if selected: + job.max_checkpoint_step = int(special_conditions.get("FROM_STEP", 0)) if int(special_conditions.get("FROM_STEP", + 0)) > job.max_checkpoint_step else job.max_checkpoint_step + self._add_edge_info(job, special_conditions["STATUS"]) + job.add_edge_info(parent,special_conditions) JobList.handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member, member_list, dependency.section, graph, other_parents) @@ -1973,20 +1988,24 @@ class JobList(object): """ Check if a checkpoint step exists for this edge""" return job.get_checkpoint_files(parent.name) - def check_checkpoint_parent_status(self): + def check_special_status(self): """ Check if all parents of a job have the correct status for checkpointing :return: jobs that fullfill the special conditions """ jobs_to_check = [] - todo - for job, checkpoint_step in self.jobs_edges.items(): - if checkpoint_step > 0: - max_step = job.get_checkpoint_files(checkpoint_step) - else: - max_step = None - for parent in parent_to_check: #if checkpoint_info: - if parent.status != checkpoint_info["status"]: - jobs_to_check.append(job) + for status, sorted_job_list in self.jobs_edges.items(): + if status == "ALL": + continue + for job in sorted_job_list: + if status in ["RUNNING", "FAILED"]: + if job.platform.connected: # This will be true only when used under setstatus/run + job.get_checkpoint_files() + for parent in job.edge_info[status]: + if parent[0].status == Status.WAITING: + if status in ["RUNNING", "FAILED"] and int(parent[1]) >= job.current_checkpoint_step: + continue + else: + jobs_to_check.append(parent[0]) return jobs_to_check def update_list(self, as_conf, store_change=True, fromSetStatus=False, submitter=None, first_time=False): # type: (AutosubmitConfig, bool, bool, object, bool) -> bool @@ -2059,10 +2078,10 @@ class JobList(object): job.status = Status.FAILED job.packed = False save = True - # Check checkpoint jobs, the status can be Ready, Running, Queuing - for job in self.check_checkpoint_parent_status(): + # Check checkpoint jobs, the status can be Any + for job in self.check_special_status(): # Check if all jobs fullfill the conditions to a job be ready - tmp = [parent for parent in job.parents if parent.status == Status.COMPLETED or parent in self.jobs_edges[job] ] + tmp = [parent for parent in job.parents if parent.status == Status.COMPLETED or parent in self.jobs_edges["ALL"] ] if len(tmp) == len(job.parents): job.status = Status.READY job.id = None diff --git a/autosubmit/platforms/platform.py b/autosubmit/platforms/platform.py index 84012944b..059609cc6 100644 --- a/autosubmit/platforms/platform.py +++ b/autosubmit/platforms/platform.py @@ -513,7 +513,7 @@ class Platform(object): (job_out_filename, job_err_filename) = remote_logs self.get_files([job_out_filename, job_err_filename], False, 'LOG_{0}'.format(exp_id)) - def get_checkpoint_files(self, job, max_step): + def get_checkpoint_files(self, job): """ Get all the checkpoint files of a job :param job: Get the checkpoint files @@ -522,15 +522,13 @@ class Platform(object): :type max_step: int """ - if job.current_checkpoint_step >= max_step: - return job.current_checkpoint_step - remote_checkpoint_path = f"{self.get_files_path()}/CHECKPOINT_" - self.get_file(remote_checkpoint_path+job.current_checkpoint_step, False, ignore_log=True) - while self.check_file_exists(remote_checkpoint_path+job.current_checkpoint_step) and job.current_checkpoint_step < max_step: - self.remove_checkpoint_file(remote_checkpoint_path+job.current_checkpoint_step) - job.current_checkpoint_step += 1 - self.get_file(remote_checkpoint_path+job.current_checkpoint_step, False, ignore_log=True) - return job.current_checkpoint_step + if job.current_checkpoint_step < job.max_checkpoint_step: + remote_checkpoint_path = f"{self.get_files_path()}/CHECKPOINT_" + self.get_file(remote_checkpoint_path+str(job.current_checkpoint_step), False, ignore_log=True) + while self.check_file_exists(remote_checkpoint_path+str(job.current_checkpoint_step)) and job.current_checkpoint_step < job.max_checkpoint_step: + self.remove_checkpoint_file(remote_checkpoint_path+str(job.current_checkpoint_step)) + job.current_checkpoint_step += 1 + self.get_file(remote_checkpoint_path+str(job.current_checkpoint_step), False, ignore_log=True) def get_completed_files(self, job_name, retries=0, recovery=False, wrapper_failed=False): """ Get the COMPLETED file of the given job -- GitLab From 1bc6b0c6d967c18478666df5027531481d5c589d Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 5 Jul 2023 16:54:41 +0200 Subject: [PATCH 07/68] all working, tODO clean old conditional code remake tests dependencies --- autosubmit/job/job_list.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 2f9f8d7b9..0794d7dfb 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -874,9 +874,9 @@ class JobList(object): if special_conditions.get("STATUS",None): if only_marked_status: if str(job.split)+"?" in filters_to_apply.get("SPLITS_TO","") or str(job.chunk)+"?" in filters_to_apply.get("CHUNKS_TO","") or str(job.member)+"?" in filters_to_apply.get("MEMBERS_TO","") or str(job.date)+"?" in filters_to_apply.get("DATES_TO",""): - selected = False - else: selected = True + else: + selected = False else: selected = True if selected: -- GitLab From 3f5809333e93b4d212704bf1c24960491e743008 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 10 Jul 2023 08:34:38 +0200 Subject: [PATCH 08/68] refined some issues --- autosubmit/job/job.py | 6 +- autosubmit/job/job_list.py | 539 +++++++++++++++++++--------------- autosubmit/monitor/monitor.py | 59 +++- 3 files changed, 366 insertions(+), 238 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 8a24ebc76..a584364fb 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -721,9 +721,9 @@ class Job(object): :type special_variables: dict """ if special_variables["STATUS"] not in self.edge_info: - self.edge_info[special_variables["STATUS"]] = [(parent, special_variables.get("FROM_STEP", 0))] - else: - self.edge_info[special_variables["STATUS"]].append((parent, special_variables.get("FROM_STEP", 0))) + self.edge_info[special_variables["STATUS"]] = {} + + self.edge_info[special_variables["STATUS"]][parent.name] = (parent,special_variables.get("FROM_STEP", 0)) def delete_parent(self, parent): """ diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 0794d7dfb..71b128daa 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -14,37 +14,36 @@ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# You should have received a copy of the GNU General Public License -# along with Autosubmit. If not, see . -import collections import copy -import re +import datetime +import math import os import pickle +# You should have received a copy of the GNU General Public License +# along with Autosubmit. If not, see . +import re import traceback -import math -import copy -from collections import defaultdict -from time import localtime, strftime, mktime +from bscearth.utils.date import date2str, parse_date +from networkx import DiGraph from shutil import move +from threading import Thread +from time import localtime, strftime, mktime +from typing import List, Dict + +import autosubmit.database.db_structure as DbStructure +from autosubmit.helpers.data_transfer import JobRow from autosubmit.job.job import Job -from autosubmit.job.job_package_persistence import JobPackagePersistence +from autosubmit.job.job_common import Status, bcolors from autosubmit.job.job_dict import DicJobs +from autosubmit.job.job_package_persistence import JobPackagePersistence from autosubmit.job.job_packages import JobPackageThread from autosubmit.job.job_utils import Dependency -from autosubmit.job.job_common import Status, bcolors -from bscearth.utils.date import date2str, parse_date -import autosubmit.database.db_structure as DbStructure -import datetime -from networkx import DiGraph from autosubmit.job.job_utils import transitive_reduction -from log.log import AutosubmitCritical, AutosubmitError, Log -from threading import Thread from autosubmitconfigparser.config.basicconfig import BasicConfig from autosubmitconfigparser.config.configcommon import AutosubmitConfig -from autosubmit.helpers.data_transfer import JobRow -from typing import List, Dict -import log.fd_show +from log.log import AutosubmitCritical, AutosubmitError, Log + + # Log.get_logger("Log.Autosubmit") @@ -54,15 +53,17 @@ def threaded(fn): thread.name = "data_processing" thread.start() return thread + return wrapper + class JobList(object): """ Class to manage the list of jobs to be run by autosubmit """ - def __init__(self, expid, config, parser_factory, job_list_persistence,as_conf): + def __init__(self, expid, config, parser_factory, job_list_persistence, as_conf): self._persistence_path = os.path.join( config.LOCAL_ROOT_DIR, expid, "pkl") self._update_file = "updated_list_" + expid + ".txt" @@ -93,6 +94,7 @@ class JobList(object): self._run_members = None self.jobs_to_run_first = list() self.rerun_job_list = list() + @property def expid(self): """ @@ -127,46 +129,56 @@ class JobList(object): @run_members.setter def run_members(self, value): - if value is not None and len(str(value)) > 0 : + if value is not None and len(str(value)) > 0: self._run_members = value - self._base_job_list = [job for job in self._job_list] + self._base_job_list = [job for job in self._job_list] found_member = False processed_job_list = [] - for job in self._job_list: # We are assuming that the jobs are sorted in topological order (which is the default) - if (job.member is None and found_member is False) or job.member in self._run_members or job.status not in [Status.WAITING, Status.READY]: + for job in self._job_list: # We are assuming that the jobs are sorted in topological order (which is the default) + if ( + job.member is None and found_member is False) or job.member in self._run_members or job.status not in [ + Status.WAITING, Status.READY]: processed_job_list.append(job) if job.member is not None and len(str(job.member)) > 0: found_member = True - self._job_list = processed_job_list + self._job_list = processed_job_list # Old implementation that also considered children of the members. # self._job_list = [job for job in old_job_list if len( # job.parents) == 0 or len(set(old_job_list_names).intersection(set([jobp.name for jobp in job.parents]))) == len(job.parents)] - def create_dictionary(self, date_list, member_list, num_chunks, chunk_ini, date_format, default_retrials, wrapper_jobs): + def create_dictionary(self, date_list, member_list, num_chunks, chunk_ini, date_format, default_retrials, + wrapper_jobs): chunk_list = list(range(chunk_ini, num_chunks + 1)) jobs_parser = self._get_jobs_parser() dic_jobs = DicJobs(self, date_list, member_list, - chunk_list, date_format, default_retrials,jobs_data={},experiment_data=self.experiment_data) + chunk_list, date_format, default_retrials, jobs_data={}, + experiment_data=self.experiment_data) self._dic_jobs = dic_jobs for wrapper_section in wrapper_jobs: if str(wrapper_jobs[wrapper_section]).lower() != 'none': - self._ordered_jobs_by_date_member[wrapper_section] = self._create_sorted_dict_jobs(wrapper_jobs[wrapper_section]) + self._ordered_jobs_by_date_member[wrapper_section] = self._create_sorted_dict_jobs( + wrapper_jobs[wrapper_section]) else: self._ordered_jobs_by_date_member[wrapper_section] = {} pass + def _delete_edgeless_jobs(self): jobs_to_delete = [] # indices to delete for i, job in enumerate(self._job_list): if job.dependencies is not None: - if ( ( len(job.dependencies) > 0 and not job.has_parents()) and not job.has_children()) and job.delete_when_edgeless in ["true",True,1]: + if (( + len(job.dependencies) > 0 and not job.has_parents()) and not job.has_children()) and job.delete_when_edgeless in [ + "true", True, 1]: jobs_to_delete.append(job) # delete jobs by indices for i in jobs_to_delete: self._job_list.remove(i) + def generate(self, date_list, member_list, num_chunks, chunk_ini, parameters, date_format, default_retrials, - default_job_type, wrapper_type=None, wrapper_jobs=dict(), new=True, notransitive=False, update_structure=False, run_only_members=[],show_log=True,jobs_data={},as_conf=""): + default_job_type, wrapper_type=None, wrapper_jobs=dict(), new=True, notransitive=False, + update_structure=False, run_only_members=[], show_log=True, jobs_data={}, as_conf=""): """ Creates all jobs needed for the current workflow @@ -204,8 +216,8 @@ class JobList(object): chunk_list = list(range(chunk_ini, num_chunks + 1)) self._chunk_list = chunk_list - - dic_jobs = DicJobs(self,date_list, member_list,chunk_list, date_format, default_retrials,jobs_data,experiment_data=self.experiment_data) + dic_jobs = DicJobs(self, date_list, member_list, chunk_list, date_format, default_retrials, jobs_data, + experiment_data=self.experiment_data) self._dic_jobs = dic_jobs priority = 0 if show_log: @@ -221,15 +233,15 @@ class JobList(object): except Exception as e: pass Log.info("Deleting previous pkl due being incompatible with current AS version") - if os.path.exists(os.path.join(self._persistence_path, self._persistence_file+".pkl")): - os.remove(os.path.join(self._persistence_path, self._persistence_file+".pkl")) - if os.path.exists(os.path.join(self._persistence_path, self._persistence_file+"_backup.pkl")): - os.remove(os.path.join(self._persistence_path, self._persistence_file+"_backup.pkl")) + if os.path.exists(os.path.join(self._persistence_path, self._persistence_file + ".pkl")): + os.remove(os.path.join(self._persistence_path, self._persistence_file + ".pkl")) + if os.path.exists(os.path.join(self._persistence_path, self._persistence_file + "_backup.pkl")): + os.remove(os.path.join(self._persistence_path, self._persistence_file + "_backup.pkl")) - self._create_jobs(dic_jobs, priority,default_job_type, jobs_data) + self._create_jobs(dic_jobs, priority, default_job_type, jobs_data) if show_log: Log.info("Adding dependencies...") - self._add_dependencies(date_list, member_list,chunk_list, dic_jobs, self.graph) + self._add_dependencies(date_list, member_list, chunk_list, dic_jobs, self.graph) if show_log: Log.info("Removing redundant dependencies...") @@ -237,7 +249,7 @@ class JobList(object): new, notransitive, update_structure=update_structure) for job in self._job_list: job.parameters = parameters - job_data = jobs_data.get(job.name,"none") + job_data = jobs_data.get(job.name, "none") try: if job_data != "none": job.wrapper_type = job_data[12] @@ -254,7 +266,9 @@ class JobList(object): str(run_only_members))) old_job_list = [job for job in self._job_list] self._job_list = [ - job for job in old_job_list if job.member is None or job.member in run_only_members or job.status not in [Status.WAITING, Status.READY]] + job for job in old_job_list if + job.member is None or job.member in run_only_members or job.status not in [Status.WAITING, + Status.READY]] for job in self._job_list: for jobp in job.parents: if jobp in self._job_list: @@ -268,22 +282,24 @@ class JobList(object): for wrapper_section in wrapper_jobs: try: if wrapper_jobs[wrapper_section] is not None and len(str(wrapper_jobs[wrapper_section])) > 0: - self._ordered_jobs_by_date_member[wrapper_section] = self._create_sorted_dict_jobs(wrapper_jobs[wrapper_section]) + self._ordered_jobs_by_date_member[wrapper_section] = self._create_sorted_dict_jobs( + wrapper_jobs[wrapper_section]) else: self._ordered_jobs_by_date_member[wrapper_section] = {} except BaseException as e: - raise AutosubmitCritical("Some section jobs of the wrapper:{0} are not in the current job_list defined in jobs.conf".format(wrapper_section),7014,str(e)) - + raise AutosubmitCritical( + "Some section jobs of the wrapper:{0} are not in the current job_list defined in jobs.conf".format( + wrapper_section), 7014, str(e)) - def _add_dependencies(self,date_list, member_list, chunk_list, dic_jobs, graph, option="DEPENDENCIES"): - jobs_data = dic_jobs._jobs_data.get("JOBS",{}) + def _add_dependencies(self, date_list, member_list, chunk_list, dic_jobs, graph, option="DEPENDENCIES"): + jobs_data = dic_jobs._jobs_data.get("JOBS", {}) for job_section in jobs_data.keys(): Log.debug("Adding dependencies for {0} jobs".format(job_section)) # If it does not have dependencies, do nothing if not (job_section, option): continue - dependencies_keys = jobs_data[job_section].get(option,{}) + dependencies_keys = jobs_data[job_section].get(option, {}) if type(dependencies_keys) is str: if "," in dependencies_keys: dependencies_list = dependencies_keys.split(",") @@ -303,10 +319,9 @@ class JobList(object): for i in range(num_jobs): _job = job[i] if num_jobs > 1 else job self._manage_job_dependencies(dic_jobs, _job, date_list, member_list, chunk_list, dependencies_keys, - dependencies, graph) + dependencies, graph) pass - @staticmethod def _manage_dependencies(dependencies_keys, dic_jobs, job_section): parameters = dic_jobs._jobs_data["JOBS"] @@ -334,7 +349,7 @@ class JobList(object): distance = int(key_split[1]) if '[' in section: - #Todo check what is this because we never enter this + # Todo check what is this because we never enter this try: section_name = section[0:section.find("[")] splits_section = int( @@ -344,13 +359,14 @@ class JobList(object): section = section_name except Exception as e: pass - if parameters.get(section,None) is None: + if parameters.get(section, None) is None: Log.printlog("WARNING: SECTION {0} is not defined in jobs.conf".format(section)) continue - #raise AutosubmitCritical("Section:{0} doesn't exists.".format(section),7014) + # raise AutosubmitCritical("Section:{0} doesn't exists.".format(section),7014) dependency_running_type = str(parameters[section].get('RUNNING', 'once')).lower() delay = int(parameters[section].get('DELAY', -1)) - dependency = Dependency(section, distance, dependency_running_type, sign, delay, splits,relationships=dependencies_keys[key]) + dependency = Dependency(section, distance, dependency_running_type, sign, delay, splits, + relationships=dependencies_keys[key]) dependencies[key] = dependency return dependencies @@ -371,13 +387,14 @@ class JobList(object): return splits @staticmethod - def _apply_filter(parent_value,filter_value,associative_list,filter_type="dates"): + def _apply_filter(parent_value, filter_value, associative_list, filter_type="dates"): """ Check if the current_job_value is included in the filter_value :param parent_value: :param filter_value: filter :param associative_list: dates, members, chunks. - :param is_chunk: True if the filter_value is a chunk. + :param filter_type: dates, members, chunks. + :return: boolean """ to_filter = [] @@ -408,11 +425,11 @@ class JobList(object): start = start_end[0].strip("[]") end = start_end[1].strip("[]") del start_end - if filter_type not in ["chunks", "splits"]: # chunk directly + if filter_type not in ["chunks", "splits"]: # chunk directly for value in range(int(start), int(end) + 1): to_filter.append(value) - else: # index - for value in range(int(start+1), int(end) + 1): + else: # index + for value in range(int(start + 1), int(end) + 1): to_filter.append(value) else: to_filter.append(filter_value) @@ -477,7 +494,7 @@ class JobList(object): # [N:M] return slice(int(value_to_check.split(":")[0]), int(value_to_check.split(":")[1])) elif value_to_check.count(":") == 2: -# range with step + # range with step if value_to_check[0] == ":": # [::M] return slice(None, None, int(value_to_check[2:])) @@ -494,9 +511,6 @@ class JobList(object): # value return value_to_check - - - @staticmethod def _check_relationship(relationships, level_to_check, value_to_check): """ @@ -511,7 +525,8 @@ class JobList(object): status = relationship.pop("STATUS", relationships.get("STATUS", None)) from_step = relationship.pop("FROM_STEP", relationships.get("FROM_STEP", None)) for filter_range, filter_data in relationship.items(): - if not value_to_check or str(value_to_check).upper() in str(JobList._parse_filter_to_check(filter_range)).upper(): + if not value_to_check or str(value_to_check).upper() in str( + JobList._parse_filter_to_check(filter_range)).upper(): if not filter_data.get("STATUS", None): filter_data["STATUS"] = status if not filter_data.get("FROM_STEP", None): @@ -530,30 +545,30 @@ class JobList(object): :param current_job: Current job to check. :return: filters_to_apply """ - + filters_to_apply = JobList._check_relationship(relationships, "DATES_FROM", date2str(current_job.date)) # there could be multiple filters that apply... per example # Current task date is 20020201, and member is fc2 # Dummy example, not specially usefull in a real case - #DATES_FROM: - #all: - #MEMBERS_FROM: - #ALL: ... - #CHUNKS_FROM: - #ALL: ... - #20020201: - #MEMBERS_FROM: - #fc2: - #DATES_TO: "20020201" - #MEMBERS_TO: "fc2" - #CHUNKS_TO: "ALL" - #SPLITS_FROM: - #ALL: - #SPLITS_TO: "1" + # DATES_FROM: + # all: + # MEMBERS_FROM: + # ALL: ... + # CHUNKS_FROM: + # ALL: ... + # 20020201: + # MEMBERS_FROM: + # fc2: + # DATES_TO: "20020201" + # MEMBERS_TO: "fc2" + # CHUNKS_TO: "ALL" + # SPLITS_FROM: + # ALL: + # SPLITS_TO: "1" # this "for" iterates for ALL and fc2 as current task is selected in both filters # The dict in this step is: # [{MEMBERS_FROM{..},CHUNKS_FROM{...}},{MEMBERS_FROM{..},SPLITS_FROM{...}}] - for i,filter in enumerate(filters_to_apply): + for i, filter in enumerate(filters_to_apply): # {MEMBERS_FROM{..},CHUNKS_FROM{...}} I want too look ALL filters not only one, but I want to go recursivily until get the _TO filter # This is not an if_else, because the current level ( dates ) could have two different filters. # Second case commented: ( date_from 20020201 ) @@ -607,9 +622,9 @@ class JobList(object): :param current_job: Current job to check. :return: filters_to_apply """ - + filters_to_apply = JobList._check_relationship(relationships, "CHUNKS_FROM", current_job.chunk) - for i,filter in enumerate(filters_to_apply): + for i, filter in enumerate(filters_to_apply): if "SPLITS_FROM" in filter: filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM"))}, current_job) if len(filters_to_apply_s) > 0: @@ -632,7 +647,7 @@ class JobList(object): return filters_to_apply @staticmethod - def _unify_to_filter(unified_filter,filter_to,filter_type): + def _unify_to_filter(unified_filter, filter_to, filter_type): """ Unify filter_to filters into a single dictionary :param unified_filter: Single dictionary with all filters_to @@ -655,8 +670,9 @@ class JobList(object): if "?" not in element: element += data unified_filter[filter_type].add(element) + @staticmethod - def _normalize_to_filters(filter_to,filter_type): + def _normalize_to_filters(filter_to, filter_type): """ Normalize filter_to filters to a single string or "all" :param filter_to: Unified filter_to dictionary @@ -685,19 +701,19 @@ class JobList(object): if "FROM_STEP" not in unified_filter and filter_to.get("FROM_STEP", None): unified_filter["FROM_STEP"] = filter_to["FROM_STEP"] if len(filter_to) > 0: - JobList._unify_to_filter(unified_filter,filter_to,"DATES_TO") - JobList._unify_to_filter(unified_filter,filter_to,"MEMBERS_TO") - JobList._unify_to_filter(unified_filter,filter_to,"CHUNKS_TO") - JobList._unify_to_filter(unified_filter,filter_to,"SPLITS_TO") - - JobList._normalize_to_filters(unified_filter,"DATES_TO") - JobList._normalize_to_filters(unified_filter,"MEMBERS_TO") - JobList._normalize_to_filters(unified_filter,"CHUNKS_TO") - JobList._normalize_to_filters(unified_filter,"SPLITS_TO") + JobList._unify_to_filter(unified_filter, filter_to, "DATES_TO") + JobList._unify_to_filter(unified_filter, filter_to, "MEMBERS_TO") + JobList._unify_to_filter(unified_filter, filter_to, "CHUNKS_TO") + JobList._unify_to_filter(unified_filter, filter_to, "SPLITS_TO") + + JobList._normalize_to_filters(unified_filter, "DATES_TO") + JobList._normalize_to_filters(unified_filter, "MEMBERS_TO") + JobList._normalize_to_filters(unified_filter, "CHUNKS_TO") + JobList._normalize_to_filters(unified_filter, "SPLITS_TO") return unified_filter @staticmethod - def _filter_current_job(current_job,relationships): + def _filter_current_job(current_job, relationships): ''' This function will filter the current job based on the relationships given :param current_job: Current job to filter :param relationships: Relationships to apply @@ -741,10 +757,8 @@ class JobList(object): filters_to_apply = relationships return filters_to_apply - - @staticmethod - def _valid_parent(parent,member_list,date_list,chunk_list,is_a_natural_relation,filter_): + def _valid_parent(parent, member_list, date_list, chunk_list, is_a_natural_relation, filter_): ''' Check if the parent is valid for the current job :param parent: job to check @@ -752,17 +766,16 @@ class JobList(object): :param date_list: list of dates :param chunk_list: list of chunks :param is_a_natural_relation: if the relation is natural or not - :param filters_to_apply: filters to apply :return: True if the parent is valid, False otherwise ''' - #check if current_parent is listed on dependency.relationships + # check if current_parent is listed on dependency.relationships associative_list = {} associative_list["dates"] = date_list associative_list["members"] = member_list associative_list["chunks"] = chunk_list if parent.splits is not None: - associative_list["splits"] = [ str(split) for split in range(1,int(parent.splits)+1) ] + associative_list["splits"] = [str(split) for split in range(1, int(parent.splits) + 1)] else: associative_list["splits"] = None dates_to = str(filter_.get("DATES_TO", "natural")).lower() @@ -788,15 +801,15 @@ class JobList(object): associative_list["splits"] = [parent.split] if parent.split is not None else parent.splits parsed_parent_date = date2str(parent.date) if parent.date is not None else None # Apply all filters to look if this parent is an appropriated candidate for the current_job - valid_dates = JobList._apply_filter(parsed_parent_date, dates_to, associative_list["dates"], "dates") + valid_dates = JobList._apply_filter(parsed_parent_date, dates_to, associative_list["dates"], "dates") valid_members = JobList._apply_filter(parent.member, members_to, associative_list["members"], "members") - valid_chunks = JobList._apply_filter(parent.chunk, chunks_to, associative_list["chunks"], "chunks") - valid_splits = JobList._apply_filter(parent.split, splits_to, associative_list["splits"], "splits") + valid_chunks = JobList._apply_filter(parent.chunk, chunks_to, associative_list["chunks"], "chunks") + valid_splits = JobList._apply_filter(parent.split, splits_to, associative_list["splits"], "splits") if valid_dates and valid_members and valid_chunks and valid_splits: return True return False - def _add_edge_info(self,job,special_status): + def _add_edge_info(self, job, special_status): """ Special relations to be check in the update_list method :param job: Current job @@ -810,9 +823,8 @@ class JobList(object): self.jobs_edges["ALL"] = set() self.jobs_edges["ALL"].add(job) - - - def _manage_job_dependencies(self,dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys, dependencies, + def _manage_job_dependencies(self, dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys, + dependencies, graph): ''' Manage the dependencies of a job @@ -831,9 +843,10 @@ class JobList(object): parsed_date_list.append(date2str(dat)) special_conditions = dict() for key in dependencies_keys: - dependency = dependencies.get(key,None) + dependency = dependencies.get(key, None) if dependency is None: - Log.printlog("WARNING: SECTION {0} is not defined in jobs.conf. Dependency skipped".format(key),Log.WARNING) + Log.printlog("WARNING: SECTION {0} is not defined in jobs.conf. Dependency skipped".format(key), + Log.WARNING) continue skip, (chunk, member, date) = JobList._calculate_dependency_metadata(job.chunk, chunk_list, job.member, member_list, @@ -847,8 +860,10 @@ class JobList(object): natural_jobs = dic_jobs.get_jobs(dependency.section, date, member, chunk) all_parents = list(set(other_parents + parents_jobs)) # Get dates_to, members_to, chunks_to of the deepest level of the relationship. - filters_to_apply = JobList._filter_current_job(job,copy.deepcopy(dependency.relationships)) - if "?" in [filters_to_apply.get("SPLITS_TO",""),filters_to_apply.get("DATES_TO",""),filters_to_apply.get("MEMBERS_TO",""),filters_to_apply.get("CHUNKS_TO","")]: + filters_to_apply = JobList._filter_current_job(job, copy.deepcopy(dependency.relationships)) + if "?" in filters_to_apply.get("SPLITS_TO", "") or "?" in filters_to_apply.get("DATES_TO", + "") or "?" in filters_to_apply.get( + "MEMBERS_TO", "") or "?" in filters_to_apply.get("CHUNKS_TO", ""): only_marked_status = True else: only_marked_status = False @@ -859,31 +874,37 @@ class JobList(object): if parent.name == job.name: continue # Check if it is a natural relation. The only difference is that a chunk can depend on a chunks <= than the current chunk - if parent in natural_jobs and (job.chunk is None or parent.chunk is None or parent.chunk <= job.chunk ): + if parent in natural_jobs and (job.chunk is None or parent.chunk is None or parent.chunk <= job.chunk): natural_relationship = True else: natural_relationship = False # Check if the current parent is a valid parent based on the dependencies set on expdef.conf # If the parent is valid, add it to the graph - if JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship,filters_to_apply): + if JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship, + filters_to_apply): job.add_parent(parent) self._add_edge(graph, job, parent) # Could be more variables in the future # Do parse checkpoint - if special_conditions.get("STATUS",None): + if special_conditions.get("STATUS", None): if only_marked_status: - if str(job.split)+"?" in filters_to_apply.get("SPLITS_TO","") or str(job.chunk)+"?" in filters_to_apply.get("CHUNKS_TO","") or str(job.member)+"?" in filters_to_apply.get("MEMBERS_TO","") or str(job.date)+"?" in filters_to_apply.get("DATES_TO",""): + if str(job.split) + "?" in filters_to_apply.get("SPLITS_TO", "") or str( + job.chunk) + "?" in filters_to_apply.get("CHUNKS_TO", "") or str( + job.member) + "?" in filters_to_apply.get("MEMBERS_TO", "") or str( + job.date) + "?" in filters_to_apply.get("DATES_TO", ""): selected = True else: selected = False else: selected = True if selected: - job.max_checkpoint_step = int(special_conditions.get("FROM_STEP", 0)) if int(special_conditions.get("FROM_STEP", - 0)) > job.max_checkpoint_step else job.max_checkpoint_step + if special_conditions.get("FROM_STEP", None): + job.max_checkpoint_step = int(special_conditions.get("FROM_STEP", 0)) if int( + special_conditions.get("FROM_STEP", + 0)) > job.max_checkpoint_step else job.max_checkpoint_step self._add_edge_info(job, special_conditions["STATUS"]) - job.add_edge_info(parent,special_conditions) + job.add_edge_info(parent, special_conditions) JobList.handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member, member_list, dependency.section, graph, other_parents) @@ -939,7 +960,7 @@ class JobList(object): @staticmethod def handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member, member_list, - section_name, graph,visited_parents): + section_name, graph, visited_parents): if job.wait and job.frequency > 1: if job.chunk is not None and len(str(job.chunk)) > 0: max_distance = (chunk_list.index(chunk) + 1) % job.frequency @@ -982,9 +1003,10 @@ class JobList(object): parent = parents[i] if isinstance(parents, list) else parents graph.add_edge(parent.name, job.name) pass + @staticmethod def _create_jobs(dic_jobs, priority, default_job_type, jobs_data=dict()): - for section in dic_jobs._jobs_data.get("JOBS",{}).keys(): + for section in dic_jobs._jobs_data.get("JOBS", {}).keys(): Log.debug("Creating {0} jobs".format(section)) dic_jobs.read_section(section, priority, default_job_type, jobs_data) priority += 1 @@ -1001,7 +1023,10 @@ class JobList(object): :return: Sorted Dictionary of List that represents the jobs included in the wrapping process. \n :rtype: Dictionary Key: date, Value: (Dictionary Key: Member, Value: List of jobs that belong to the date, member, and are ordered by chunk number if it is a chunk job otherwise num_chunks from JOB TYPE (section) """ + # Dictionary Key: date, Value: (Dictionary Key: Member, Value: List) + job = None + dict_jobs = dict() for date in self._date_list: dict_jobs[date] = dict() @@ -1009,7 +1034,6 @@ class JobList(object): dict_jobs[date][member] = list() num_chunks = len(self._chunk_list) - sections_running_type_map = dict() if wrapper_jobs is not None and len(str(wrapper_jobs)) > 0: if type(wrapper_jobs) is not list: @@ -1019,13 +1043,12 @@ class JobList(object): char = " " wrapper_jobs = wrapper_jobs.split(char) - for section in wrapper_jobs: # RUNNING = once, as default. This value comes from jobs_.yml try: sections_running_type_map[section] = str(self.jobs_data[section].get("RUNNING", 'once')) except BaseException as e: - raise AutosubmitCritical("Key {0} doesn't exists.".format(section),7014,str(e)) + raise AutosubmitCritical("Key {0} doesn't exists.".format(section), 7014, str(e)) # Select only relevant jobs, those belonging to the sections defined in the wrapper @@ -1043,20 +1066,19 @@ class JobList(object): for member in self._member_list: # Filter list of fake jobs according to date and member, result not sorted at this point sorted_jobs_list = list(filter(lambda job: job.name.split("_")[1] == str_date and - job.name.split("_")[2] == member, filtered_jobs_fake_date_member)) - #sorted_jobs_list = [job for job in filtered_jobs_fake_date_member if job.name.split("_")[1] == str_date and + job.name.split("_")[2] == member, + filtered_jobs_fake_date_member)) + # sorted_jobs_list = [job for job in filtered_jobs_fake_date_member if job.name.split("_")[1] == str_date and # job.name.split("_")[2] == member] - #There can be no jobs for this member when select chunk/member is enabled + # There can be no jobs for this member when select chunk/member is enabled if not sorted_jobs_list or len(sorted_jobs_list) == 0: continue - previous_job = sorted_jobs_list[0] # get RUNNING for this section section_running_type = sections_running_type_map[previous_job.section] - jobs_to_sort = [previous_job] previous_section_running_type = None # Index starts at 1 because 0 has been taken in a previous step @@ -1069,14 +1091,16 @@ class JobList(object): previous_section_running_type = section_running_type section_running_type = sections_running_type_map[job.section] # Test if RUNNING is different between sections, or if we have reached the last item in sorted_jobs_list - if (previous_section_running_type is not None and previous_section_running_type != section_running_type) \ + if ( + previous_section_running_type is not None and previous_section_running_type != section_running_type) \ or index == len(sorted_jobs_list): # Sorting by date, member, chunk number if it is a chunk job otherwise num_chunks from JOB TYPE (section) # Important to note that the only differentiating factor would be chunk OR num_chunks jobs_to_sort = sorted(jobs_to_sort, key=lambda k: (k.name.split('_')[1], (k.name.split('_')[2]), (int(k.name.split('_')[3]) - if len(k.name.split('_')) == 5 else num_chunks + 1))) + if len(k.name.split( + '_')) == 5 else num_chunks + 1))) # Bringing back original job if identified for idx in range(0, len(jobs_to_sort)): @@ -1089,7 +1113,7 @@ class JobList(object): # By adding to the result at this step, only those with the same RUNNING have been added. dict_jobs[date][member] += jobs_to_sort jobs_to_sort = [] - if len(sorted_jobs_list) > 1 : + if len(sorted_jobs_list) > 1: jobs_to_sort.append(job) previous_job = job @@ -1121,7 +1145,7 @@ class JobList(object): fake_job = copy.deepcopy(job) # Use previous values to modify name of fake job fake_job.name = fake_job.name.split('_', 1)[0] + "_" + self._get_date(date) + "_" \ - + member + "_" + fake_job.name.split("_", 1)[1] + + member + "_" + fake_job.name.split("_", 1)[1] # Filling list of fake jobs, only difference is the name filtered_jobs_fake_date_member.append(fake_job) # Mapping fake jobs to original ones @@ -1211,7 +1235,8 @@ class JobList(object): def copy_ordered_jobs_by_date_member(self): pass - def get_ordered_jobs_by_date_member(self,section): + + def get_ordered_jobs_by_date_member(self, section): """ Get the dictionary of jobs ordered according to wrapper's expression divided by date and member @@ -1250,7 +1275,8 @@ class JobList(object): :return: completed jobs :rtype: list """ - uncompleted_jobs = [job for job in self._job_list if (platform is None or job.platform.name == platform.name) and + uncompleted_jobs = [job for job in self._job_list if + (platform is None or job.platform.name == platform.name) and job.status != Status.COMPLETED] if wrapper: @@ -1362,7 +1388,8 @@ class JobList(object): :rtype: list """ unsubmitted = [job for job in self._job_list if (platform is None or job.platform.name == platform.name) and - (job.status != Status.SUBMITTED and job.status != Status.QUEUING and job.status == Status.RUNNING and job.status == Status.COMPLETED)] + ( + job.status != Status.SUBMITTED and job.status != Status.QUEUING and job.status == Status.RUNNING and job.status == Status.COMPLETED)] if wrapper: return [job for job in unsubmitted if job.packed is False] @@ -1386,7 +1413,7 @@ class JobList(object): else: return all_jobs - def get_job_names(self,lower_case=False): + def get_job_names(self, lower_case=False): """ Returns a list of all job names :param: lower_case: if true, returns lower case job names @@ -1407,21 +1434,24 @@ class JobList(object): def update_two_step_jobs(self): prev_jobs_to_run_first = self.jobs_to_run_first if len(self.jobs_to_run_first) > 0: - self.jobs_to_run_first = [ job for job in self.jobs_to_run_first if job.status != Status.COMPLETED ] + self.jobs_to_run_first = [job for job in self.jobs_to_run_first if job.status != Status.COMPLETED] keep_running = False for job in self.jobs_to_run_first: - running_parents = [parent for parent in job.parents if parent.status != Status.WAITING and parent.status != Status.FAILED ] #job is parent of itself + running_parents = [parent for parent in job.parents if + parent.status != Status.WAITING and parent.status != Status.FAILED] # job is parent of itself if len(running_parents) == len(job.parents): keep_running = True if len(self.jobs_to_run_first) > 0 and keep_running is False: - raise AutosubmitCritical("No more jobs to run first, there were still pending jobs but they're unable to run without their parents or there are failed jobs.",7014) + raise AutosubmitCritical( + "No more jobs to run first, there were still pending jobs but they're unable to run without their parents or there are failed jobs.", + 7014) - def parse_jobs_by_filter(self, unparsed_jobs,two_step_start = True): + def parse_jobs_by_filter(self, unparsed_jobs, two_step_start=True): jobs_to_run_first = list() - select_jobs_by_name = "" #job_name - select_all_jobs_by_section = "" # all + select_jobs_by_name = "" # job_name + select_all_jobs_by_section = "" # all filter_jobs_by_section = "" # Select, chunk / member - if "&" in unparsed_jobs: # If there are explicit jobs add them + if "&" in unparsed_jobs: # If there are explicit jobs add them jobs_to_check = unparsed_jobs.split("&") select_jobs_by_name = jobs_to_check[0] unparsed_jobs = jobs_to_check[1] @@ -1438,20 +1468,26 @@ class JobList(object): filter_jobs_by_section = aux[1] if two_step_start: try: - self.jobs_to_run_first = self.get_job_related(select_jobs_by_name=select_jobs_by_name,select_all_jobs_by_section=select_all_jobs_by_section,filter_jobs_by_section=filter_jobs_by_section) + self.jobs_to_run_first = self.get_job_related(select_jobs_by_name=select_jobs_by_name, + select_all_jobs_by_section=select_all_jobs_by_section, + filter_jobs_by_section=filter_jobs_by_section) except Exception as e: - raise AutosubmitCritical("Check the {0} format.\nFirst filter is optional ends with '&'.\nSecond filter ends with ';'.\nThird filter must contain '['. ".format(unparsed_jobs)) + raise AutosubmitCritical( + "Check the {0} format.\nFirst filter is optional ends with '&'.\nSecond filter ends with ';'.\nThird filter must contain '['. ".format( + unparsed_jobs)) else: try: self.rerun_job_list = self.get_job_related(select_jobs_by_name=select_jobs_by_name, - select_all_jobs_by_section=select_all_jobs_by_section, - filter_jobs_by_section=filter_jobs_by_section,two_step_start=two_step_start) + select_all_jobs_by_section=select_all_jobs_by_section, + filter_jobs_by_section=filter_jobs_by_section, + two_step_start=two_step_start) except Exception as e: raise AutosubmitCritical( "Check the {0} format.\nFirst filter is optional ends with '&'.\nSecond filter ends with ';'.\nThird filter must contain '['. ".format( unparsed_jobs)) - def get_job_related(self, select_jobs_by_name="",select_all_jobs_by_section="",filter_jobs_by_section="",two_step_start=True): + def get_job_related(self, select_jobs_by_name="", select_all_jobs_by_section="", filter_jobs_by_section="", + two_step_start=True): """ :param two_step_start: :param select_jobs_by_name: job name @@ -1465,25 +1501,29 @@ class JobList(object): jobs_date = [] # First Filter {select job by name} if select_jobs_by_name != "": - jobs_by_name = [ job for job in self._job_list if re.search("(^|[^0-9a-z_])"+job.name.lower()+"([^a-z0-9_]|$)",select_jobs_by_name.lower()) is not None ] - jobs_by_name_no_expid = [job for job in self._job_list if - re.search("(^|[^0-9a-z_])" + job.name.lower()[5:] + "([^a-z0-9_]|$)", + jobs_by_name = [job for job in self._job_list if + re.search("(^|[^0-9a-z_])" + job.name.lower() + "([^a-z0-9_]|$)", select_jobs_by_name.lower()) is not None] + jobs_by_name_no_expid = [job for job in self._job_list if + re.search("(^|[^0-9a-z_])" + job.name.lower()[5:] + "([^a-z0-9_]|$)", + select_jobs_by_name.lower()) is not None] ultimate_jobs_list.extend(jobs_by_name) ultimate_jobs_list.extend(jobs_by_name_no_expid) # Second Filter { select all } if select_all_jobs_by_section != "": - all_jobs_by_section = [ job for job in self._job_list if re.search("(^|[^0-9a-z_])"+job.section.upper()+"([^a-z0-9_]|$)",select_all_jobs_by_section.upper()) is not None ] + all_jobs_by_section = [job for job in self._job_list if + re.search("(^|[^0-9a-z_])" + job.section.upper() + "([^a-z0-9_]|$)", + select_all_jobs_by_section.upper()) is not None] ultimate_jobs_list.extend(all_jobs_by_section) # Third Filter N section { date , member? , chunk?} # Section[date[member][chunk]] # filter_jobs_by_section="SIM[20[C:000][M:1]],DA[20 21[M:000 001][C:1]]" if filter_jobs_by_section != "": - section_name="" - section_dates="" - section_chunks="" - section_members="" + section_name = "" + section_dates = "" + section_chunks = "" + section_members = "" jobs_final = list() for complete_filter_by_section in filter_jobs_by_section.split(','): section_list = complete_filter_by_section.split('[') @@ -1499,20 +1539,27 @@ class JobList(object): elif 'm' in section_list[3].lower(): section_members = section_list[3].strip('mM:[]') - if section_name != "": jobs_filtered = [job for job in self._job_list if - re.search("(^|[^0-9a-z_])" + job.section.upper() + "([^a-z0-9_]|$)", - section_name.upper()) is not None] + re.search("(^|[^0-9a-z_])" + job.section.upper() + "([^a-z0-9_]|$)", + section_name.upper()) is not None] if section_dates != "": - jobs_date = [ job for job in jobs_filtered if re.search("(^|[^0-9a-z_])" + date2str(job.date, job.date_format) + "([^a-z0-9_]|$)", section_dates.lower()) is not None or job.date is None ] + jobs_date = [job for job in jobs_filtered if + re.search("(^|[^0-9a-z_])" + date2str(job.date, job.date_format) + "([^a-z0-9_]|$)", + section_dates.lower()) is not None or job.date is None] if section_chunks != "" or section_members != "": - jobs_final = [job for job in jobs_date if ( section_chunks == "" or re.search("(^|[^0-9a-z_])" + str(job.chunk) + "([^a-z0-9_]|$)",section_chunks) is not None ) and ( section_members == "" or re.search("(^|[^0-9a-z_])" + str(job.member) + "([^a-z0-9_]|$)",section_members.lower()) is not None ) ] + jobs_final = [job for job in jobs_date if ( + section_chunks == "" or re.search("(^|[^0-9a-z_])" + str(job.chunk) + "([^a-z0-9_]|$)", + section_chunks) is not None) and ( + section_members == "" or re.search( + "(^|[^0-9a-z_])" + str(job.member) + "([^a-z0-9_]|$)", + section_members.lower()) is not None)] ultimate_jobs_list.extend(jobs_final) # Duplicates out ultimate_jobs_list = list(set(ultimate_jobs_list)) - Log.debug("List of jobs filtered by TWO_STEP_START parameter:\n{0}".format([job.name for job in ultimate_jobs_list])) + Log.debug( + "List of jobs filtered by TWO_STEP_START parameter:\n{0}".format([job.name for job in ultimate_jobs_list])) return ultimate_jobs_list def get_logs(self): @@ -1550,7 +1597,8 @@ class JobList(object): :return: ready jobs :rtype: list """ - ready = [job for job in self._job_list if ( platform is None or platform == "" or job.platform.name == platform.name ) and + ready = [job for job in self._job_list if + (platform is None or platform == "" or job.platform.name == platform.name) and job.status == Status.READY and job.hold is hold] if wrapper: @@ -1570,6 +1618,7 @@ class JobList(object): prepared = [job for job in self._job_list if (platform is None or job.platform.name == platform.name) and job.status == Status.PREPARED] return prepared + def get_delayed(self, platform=None): """ Returns a list of delayed jobs @@ -1580,8 +1629,9 @@ class JobList(object): :rtype: list """ delayed = [job for job in self._job_list if (platform is None or job.platform.name == platform.name) and - job.status == Status.DELAYED] + job.status == Status.DELAYED] return delayed + def get_skipped(self, platform=None): """ Returns a list of skipped jobs @@ -1622,7 +1672,7 @@ class JobList(object): """ waiting_jobs = [job for job in self._job_list if ( - job.platform.type == platform_type and job.status == Status.WAITING)] + job.platform.type == platform_type and job.status == Status.WAITING)] return waiting_jobs def get_held_jobs(self, platform=None): @@ -1734,13 +1784,15 @@ class JobList(object): """ active = self.get_in_queue(platform) + self.get_ready( - platform=platform, hold=True) + self.get_ready(platform=platform, hold=False) + self.get_delayed(platform=platform) + platform=platform, hold=True) + self.get_ready(platform=platform, hold=False) + self.get_delayed( + platform=platform) tmp = [job for job in active if job.hold and not (job.status == - Status.SUBMITTED or job.status == Status.READY or job.status == Status.DELAYED) ] + Status.SUBMITTED or job.status == Status.READY or job.status == Status.DELAYED)] if len(tmp) == len(active): # IF only held jobs left without dependencies satisfied if len(tmp) != 0 and len(active) != 0: raise AutosubmitCritical( - "Only Held Jobs active. Exiting Autosubmit (TIP: This can happen if suspended or/and Failed jobs are found on the workflow)", 7066) + "Only Held Jobs active. Exiting Autosubmit (TIP: This can happen if suspended or/and Failed jobs are found on the workflow)", + 7066) active = [] return active @@ -1756,6 +1808,7 @@ class JobList(object): for job in self._job_list: if job.name == name: return job + def get_jobs_by_section(self, section_list): """ Returns the job that its name matches parameter section @@ -1786,7 +1839,7 @@ class JobList(object): def get_in_ready_grouped_id(self, platform): jobs = [] [jobs.append(job) for job in jobs if ( - platform is None or job.platform.name is platform.name)] + platform is None or job.platform.name is platform.name)] jobs_by_id = dict() for job in jobs: @@ -1889,15 +1942,15 @@ class JobList(object): try: self._persistence.save(self._persistence_path, - self._persistence_file, self._job_list if self.run_members is None or job_list is None else job_list) + self._persistence_file, + self._job_list if self.run_members is None or job_list is None else job_list) pass except BaseException as e: - raise AutosubmitError(str(e),6040,"Failure while saving the job_list") + raise AutosubmitError(str(e), 6040, "Failure while saving the job_list") except AutosubmitError as e: raise except BaseException as e: - raise AutosubmitError(str(e),6040,"Unknown failure while saving the job_list") - + raise AutosubmitError(str(e), 6040, "Unknown failure while saving the job_list") def backup_save(self): """ @@ -1911,8 +1964,8 @@ class JobList(object): exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, self.expid) tmp_path = os.path.join(exp_path, BasicConfig.LOCAL_TMP_DIR) aslogs_path = os.path.join(tmp_path, BasicConfig.LOCAL_ASLOG_DIR) - Log.reset_status_file(os.path.join(aslogs_path,"jobs_active_status.log"),"status") - Log.reset_status_file(os.path.join(aslogs_path,"jobs_failed_status.log"),"status_failed") + Log.reset_status_file(os.path.join(aslogs_path, "jobs_active_status.log"), "status") + Log.reset_status_file(os.path.join(aslogs_path, "jobs_failed_status.log"), "status_failed") job_list = self.get_completed()[-5:] + self.get_in_queue() failed_job_list = self.get_failed() if len(job_list) > 0: @@ -1920,7 +1973,7 @@ class JobList(object): "Job Id", "Job Status", "Job Platform", "Job Queue") if len(failed_job_list) > 0: Log.status_failed("\n{0:<35}{1:<15}{2:<15}{3:<20}{4:<15}", "Job Name", - "Job Id", "Job Status", "Job Platform", "Job Queue") + "Job Id", "Job Status", "Job Platform", "Job Queue") for job in job_list: if len(job.queue) > 0 and str(job.platform.queue).lower() != "none": queue = job.queue @@ -1961,8 +2014,10 @@ class JobList(object): "_" + output_date)) def get_skippable_jobs(self, jobs_in_wrapper): - job_list_skip = [job for job in self.get_job_list() if job.skippable == "true" and (job.status == Status.QUEUING or job.status == - Status.RUNNING or job.status == Status.COMPLETED or job.status == Status.READY) and jobs_in_wrapper.find(job.section) == -1] + job_list_skip = [job for job in self.get_job_list() if + job.skippable == "true" and (job.status == Status.QUEUING or job.status == + Status.RUNNING or job.status == Status.COMPLETED or job.status == Status.READY) and jobs_in_wrapper.find( + job.section) == -1] skip_by_section = dict() for job in job_list_skip: if job.section not in skip_by_section: @@ -1998,15 +2053,16 @@ class JobList(object): continue for job in sorted_job_list: if status in ["RUNNING", "FAILED"]: - if job.platform.connected: # This will be true only when used under setstatus/run + if job.platform.connected: # This will be true only when used under setstatus/run job.get_checkpoint_files() - for parent in job.edge_info[status]: + for parent in job.edge_info[status].values(): if parent[0].status == Status.WAITING: - if status in ["RUNNING", "FAILED"] and int(parent[1]) >= job.current_checkpoint_step: + if status in ["RUNNING", "FAILED"] and parent[1] and int(parent[1]) >= job.current_checkpoint_step: continue else: jobs_to_check.append(parent[0]) return jobs_to_check + def update_list(self, as_conf, store_change=True, fromSetStatus=False, submitter=None, first_time=False): # type: (AutosubmitConfig, bool, bool, object, bool) -> bool """ @@ -2081,7 +2137,8 @@ class JobList(object): # Check checkpoint jobs, the status can be Any for job in self.check_special_status(): # Check if all jobs fullfill the conditions to a job be ready - tmp = [parent for parent in job.parents if parent.status == Status.COMPLETED or parent in self.jobs_edges["ALL"] ] + tmp = [parent for parent in job.parents if + parent.status == Status.COMPLETED or parent in self.jobs_edges["ALL"]] if len(tmp) == len(job.parents): job.status = Status.READY job.id = None @@ -2118,9 +2175,12 @@ class JobList(object): if datetime.datetime.now() >= job.delay_end: job.status = Status.READY for job in self.get_waiting(): - tmp = [parent for parent in job.parents if parent.status == Status.COMPLETED or parent.status == Status.SKIPPED] - tmp2 = [parent for parent in job.parents if parent.status == Status.COMPLETED or parent.status == Status.SKIPPED or parent.status == Status.FAILED] - tmp3 = [parent for parent in job.parents if parent.status == Status.SKIPPED or parent.status == Status.FAILED] + tmp = [parent for parent in job.parents if + parent.status == Status.COMPLETED or parent.status == Status.SKIPPED] + tmp2 = [parent for parent in job.parents if + parent.status == Status.COMPLETED or parent.status == Status.SKIPPED or parent.status == Status.FAILED] + tmp3 = [parent for parent in job.parents if + parent.status == Status.SKIPPED or parent.status == Status.FAILED] failed_ones = [parent for parent in job.parents if parent.status == Status.FAILED] if job.parents is None or len(tmp) == len(job.parents): job.status = Status.READY @@ -2138,14 +2198,15 @@ class JobList(object): if parent.name in job.edge_info and job.edge_info[parent.name].get('optional', False): weak_dependencies_failure = True elif parent.section in job.dependencies: - if parent.status not in [Status.COMPLETED,Status.SKIPPED]: + if parent.status not in [Status.COMPLETED, Status.SKIPPED]: strong_dependencies_failure = True break if not strong_dependencies_failure and weak_dependencies_failure: job.status = Status.READY job.hold = False Log.debug( - "Setting job: {0} status to: READY (conditional jobs are completed/failed)...".format(job.name)) + "Setting job: {0} status to: READY (conditional jobs are completed/failed)...".format( + job.name)) break if as_conf.get_remote_dependencies() == "true": all_parents_completed.append(job.name) @@ -2173,23 +2234,26 @@ class JobList(object): job.hold = False save = True Log.debug( - "A job in prepared status has all parent completed, job: {0} status set to: READY ...".format(job.name)) + "A job in prepared status has all parent completed, job: {0} status set to: READY ...".format( + job.name)) Log.debug('Updating WAITING jobs eligible for be prepared') # Setup job name should be a variable for job in self.get_waiting_remote_dependencies('slurm'): if job.name not in all_parents_completed: tmp = [parent for parent in job.parents if ( - (parent.status == Status.SKIPPED or parent.status == Status.COMPLETED or parent.status == Status.QUEUING or parent.status == Status.RUNNING) and "setup" not in parent.name.lower())] + ( + parent.status == Status.SKIPPED or parent.status == Status.COMPLETED or parent.status == Status.QUEUING or parent.status == Status.RUNNING) and "setup" not in parent.name.lower())] if len(tmp) == len(job.parents): job.status = Status.PREPARED job.hold = True Log.debug( - "Setting job: {0} status to: Prepared for be held (all parents queuing, running or completed)...".format(job.name)) + "Setting job: {0} status to: Prepared for be held (all parents queuing, running or completed)...".format( + job.name)) Log.debug('Updating Held jobs') if self.job_package_map: held_jobs = [job for job in self.get_held_jobs() if ( - job.id not in list(self.job_package_map.keys()))] + job.id not in list(self.job_package_map.keys()))] held_jobs += [wrapper_job for wrapper_job in list(self.job_package_map.values()) if wrapper_job.status == Status.HELD] else: @@ -2208,7 +2272,7 @@ class JobList(object): job.hold = hold_wrapper if not job.hold: for inner_job in job.job_list: - inner_job.hold = False + inner_job.hold = False Log.debug( "Setting job: {0} status to: Queuing (all parents completed)...".format( job.name)) @@ -2216,7 +2280,7 @@ class JobList(object): tmp = [ parent for parent in job.parents if parent.status == Status.COMPLETED] if len(tmp) == len(job.parents): - job.hold = False + job.hold = False Log.debug( "Setting job: {0} status to: Queuing (all parents completed)...".format( job.name)) @@ -2248,7 +2312,7 @@ class JobList(object): for related_job in jobs_to_skip[section]: if members.index(job.member) < members.index( related_job.member) and job.chunk == related_job.chunk and jobdate == date2str( - related_job.date, related_job.date_format): + related_job.date, related_job.date_format): try: if job.status == Status.QUEUING: job.platform.send_command(job.platform.cancel_cmd + " " + str(job.id), @@ -2257,7 +2321,7 @@ class JobList(object): pass # job_id finished already job.status = Status.SKIPPED save = True - #save = True + # save = True self.update_two_step_jobs() Log.debug('Update finished') return save @@ -2306,7 +2370,8 @@ class JobList(object): if m_time_job_conf: if m_time_job_conf > m_time_db: Log.info( - "File jobs_{0}.yml has been modified since the last time the structure persistence was saved.".format(self.expid)) + "File jobs_{0}.yml has been modified since the last time the structure persistence was saved.".format( + self.expid)) structure_valid = False else: Log.info( @@ -2329,7 +2394,7 @@ class JobList(object): if structure_valid is False: # Structure does not exist, or it is not be updated, attempt to create it. Log.info("Updating structure persistence...") - self.graph = transitive_reduction(self.graph) # add threads for large experiments? todo + self.graph = transitive_reduction(self.graph) # add threads for large experiments? todo if self.graph: for job in self._job_list: children_to_remove = [ @@ -2365,7 +2430,8 @@ class JobList(object): out = False return out - def save_wrappers(self,packages_to_save,failed_packages,as_conf,packages_persistence,hold=False,inspect=False): + def save_wrappers(self, packages_to_save, failed_packages, as_conf, packages_persistence, hold=False, + inspect=False): for package in packages_to_save: if package.jobs[0].id not in failed_packages: if hasattr(package, "name"): @@ -2380,6 +2446,7 @@ class JobList(object): # Saving only when it is a real multi job package packages_persistence.save( package.name, package.jobs, package._expid, inspect) + def check_scripts(self, as_conf): """ When we have created the scripts, all parameters should have been substituted. @@ -2437,7 +2504,7 @@ class JobList(object): self._job_list.remove(job) - def rerun(self, job_list_unparsed,as_conf, monitor=False): + def rerun(self, job_list_unparsed, as_conf, monitor=False): """ Updates job list to rerun the jobs specified by a job list :param job_list_unparsed: list of jobs to rerun @@ -2448,7 +2515,7 @@ class JobList(object): :type monitor: bool """ - self.parse_jobs_by_filter(job_list_unparsed,two_step_start=False) + self.parse_jobs_by_filter(job_list_unparsed, two_step_start=False) member_list = set() chunk_list = set() date_list = set() @@ -2477,8 +2544,8 @@ class JobList(object): for job_section in job_sections: Log.debug( "Reading rerun dependencies for {0} jobs".format(job_section)) - if as_conf.jobs_data[job_section].get('DEPENDENCIES',None) is not None: - dependencies_keys = as_conf.jobs_data[job_section].get('DEPENDENCIES',{}) + if as_conf.jobs_data[job_section].get('DEPENDENCIES', None) is not None: + dependencies_keys = as_conf.jobs_data[job_section].get('DEPENDENCIES', {}) if type(dependencies_keys) is str: dependencies_keys = dependencies_keys.upper().split() if dependencies_keys is None: @@ -2487,11 +2554,16 @@ class JobList(object): for job in self.get_jobs_by_section(job_section): for key in dependencies_keys: dependency = dependencies[key] - skip, (chunk, member, date) = JobList._calculate_dependency_metadata(job.chunk, self._chunk_list, job.member, self._member_list, job.date, self._date_list, dependency) + skip, (chunk, member, date) = JobList._calculate_dependency_metadata(job.chunk, + self._chunk_list, + job.member, + self._member_list, + job.date, self._date_list, + dependency) if skip: continue section_name = dependencies[key].section - for parent in self._dic_jobs.get_jobs(section_name, job.date, job.member,job.chunk): + for parent in self._dic_jobs.get_jobs(section_name, job.date, job.member, job.chunk): if not monitor: parent.status = Status.WAITING Log.debug("Parent: " + parent.name) @@ -2535,10 +2607,11 @@ class JobList(object): allJobs = self.get_all() if existingList is None else existingList # Header result = (bcolors.BOLD if nocolor is False else '') + \ - "## String representation of Job List [" + str(len(allJobs)) + "] " + "## String representation of Job List [" + str(len(allJobs)) + "] " if statusChange is not None and len(str(statusChange)) > 0: result += "with " + (bcolors.OKGREEN if nocolor is False else '') + str(len(list(statusChange.keys())) - ) + " Change(s) ##" + (bcolors.ENDC + bcolors.ENDC if nocolor is False else '') + ) + " Change(s) ##" + ( + bcolors.ENDC + bcolors.ENDC if nocolor is False else '') else: result += " ## " @@ -2549,7 +2622,7 @@ class JobList(object): if len(job.parents) == 0: roots.append(job) visited = list() - #print(root) + # print(root) # root exists for root in roots: if root is not None and len(str(root)) > 0: @@ -2610,17 +2683,17 @@ class JobList(object): prefix += "| " # Prefix + Job Name result = "\n" + prefix + \ - (bcolors.BOLD + bcolors.CODE_TO_COLOR[job.status] if nocolor is False else '') + \ - job.name + \ - (bcolors.ENDC + bcolors.ENDC if nocolor is False else '') + (bcolors.BOLD + bcolors.CODE_TO_COLOR[job.status] if nocolor is False else '') + \ + job.name + \ + (bcolors.ENDC + bcolors.ENDC if nocolor is False else '') if len(job._children) > 0: level += 1 children = job._children total_children = len(job._children) # Writes children number and status if color are not being showed result += " ~ [" + str(total_children) + (" children] " if total_children > 1 else " child] ") + \ - ("[" + Status.VALUE_TO_KEY[job.status] + - "] " if nocolor is True else "") + ("[" + Status.VALUE_TO_KEY[job.status] + + "] " if nocolor is True else "") if statusChange is not None and len(str(statusChange)) > 0: # Writes change if performed result += (bcolors.BOLD + @@ -2639,7 +2712,7 @@ class JobList(object): "] " if nocolor is True else "") return result - + @staticmethod def retrieve_packages(BasicConfig, expid, current_jobs=None): """ @@ -2704,7 +2777,8 @@ class JobList(object): return job_to_package, package_to_jobs, package_to_package_id, package_to_symbol @staticmethod - def retrieve_times(status_code, name, tmp_path, make_exception=False, job_times=None, seconds=False, job_data_collection=None): + def retrieve_times(status_code, name, tmp_path, make_exception=False, job_times=None, seconds=False, + job_data_collection=None): """ Retrieve job timestamps from database. :param job_data_collection: @@ -2765,10 +2839,13 @@ class JobList(object): if status_code in [Status.SUSPENDED]: t_submit = t_start = t_finish = 0 - return JobRow(job_data.job_name, int(queue_time), int(running_time), status, energy, JobList.ts_to_datetime(t_submit), JobList.ts_to_datetime(t_start), JobList.ts_to_datetime(t_finish), job_data.ncpus, job_data.run_id) + return JobRow(job_data.job_name, int(queue_time), int(running_time), status, energy, + JobList.ts_to_datetime(t_submit), JobList.ts_to_datetime(t_start), + JobList.ts_to_datetime(t_finish), job_data.ncpus, job_data.run_id) # Using standard procedure - if status_code in [Status.RUNNING, Status.SUBMITTED, Status.QUEUING, Status.FAILED] or make_exception is True: + if status_code in [Status.RUNNING, Status.SUBMITTED, Status.QUEUING, + Status.FAILED] or make_exception is True: # COMPLETED adds too much overhead so these values are now stored in a database and retrieved separately submit_time, start_time, finish_time, status = JobList._job_running_check( status_code, name, tmp_path) @@ -2781,7 +2858,7 @@ class JobList(object): Status.FAILED] else 0 else: queuing_for_min = ( - datetime.datetime.now() - submit_time) + datetime.datetime.now() - submit_time) running_for_min = datetime.datetime.now() - datetime.datetime.now() submit_time = mktime(submit_time.timetuple()) start_time = 0 @@ -2815,9 +2892,9 @@ class JobList(object): return seconds_queued = seconds_queued * \ - (-1) if seconds_queued < 0 else seconds_queued + (-1) if seconds_queued < 0 else seconds_queued seconds_running = seconds_running * \ - (-1) if seconds_running < 0 else seconds_running + (-1) if seconds_running < 0 else seconds_running if seconds is False: queue_time = math.ceil( seconds_queued / 60) if seconds_queued > 0 else 0 @@ -2827,17 +2904,17 @@ class JobList(object): queue_time = seconds_queued running_time = seconds_running - return JobRow(name, - int(queue_time), - int(running_time), - status, - energy, - JobList.ts_to_datetime(submit_time), - JobList.ts_to_datetime(start_time), - JobList.ts_to_datetime(finish_time), - 0, - 0) - + return JobRow(name, + int(queue_time), + int(running_time), + status, + energy, + JobList.ts_to_datetime(submit_time), + JobList.ts_to_datetime(start_time), + JobList.ts_to_datetime(finish_time), + 0, + 0) + @staticmethod def _job_running_check(status_code, name, tmp_path): """ @@ -2908,7 +2985,7 @@ class JobList(object): if len(values) > 3 and current_status != status_from_job and current_status != "NA": current_status = "SUSPICIOUS" return submit_time, start_time, finish_time, current_status - + @staticmethod def ts_to_datetime(timestamp): if timestamp and timestamp > 0: @@ -2916,4 +2993,4 @@ class JobList(object): # timestamp).strftime('%Y-%m-%d %H:%M:%S')) return datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S') else: - return None \ No newline at end of file + return None diff --git a/autosubmit/monitor/monitor.py b/autosubmit/monitor/monitor.py index 8b8bffc55..53293ef11 100644 --- a/autosubmit/monitor/monitor.py +++ b/autosubmit/monitor/monitor.py @@ -233,6 +233,48 @@ class Monitor: Log.debug('Graph definition finalized') return graph + def _check_final_status(self, job, child): + # order of self._table + # child.edge_info is a tuple, I want to get first element of each tuple with a lambda + label = None + if len(child.edge_info) > 0: + if job in child.edge_info.get("FAILED",{}): + color = self._table.get(Status.FAILED,None) + label = child.edge_info["FAILED"].get(job.name,0)[1] + elif job.name in child.edge_info.get("RUNNING",{}): + color = self._table.get(Status.RUNNING,None) + label = child.edge_info["RUNNING"].get(job.name,0)[1] + elif job.name in child.edge_info.get("QUEUING",{}): + color = self._table.get(Status.QUEUING,None) + elif job.name in child.edge_info.get("HELD",{}): + color = self._table.get(Status.HELD,None) + elif job.name in child.edge_info.get("DELAYED",{}): + color = self._table.get(Status.DELAYED,None) + elif job.name in child.edge_info.get("UNKNOWN",{}): + color = self._table.get(Status.UNKNOWN,None) + elif job.name in child.edge_info.get("SUSPENDED",{}): + color = self._table.get(Status.SUSPENDED,None) + elif job.name in child.edge_info.get("SKIPPED",{}): + color = self._table.get(Status.SKIPPED,None) + elif job.name in child.edge_info.get("WAITING",{}): + color = self._table.get(Status.WAITING,None) + elif job.name in child.edge_info.get("READY",{}): + color = self._table.get(Status.READY,None) + elif job.name in child.edge_info.get("SUBMITTED",{}): + color = self._table.get(Status.SUBMITTED,None) + else: + color = self._table.get(Status.COMPLETED,None) + if label and label == 0: + label = None + return color,label + else: + return None, None + + + + + + def _add_children(self, job, exp, node_job, groups, hide_groups): if job in self.nodes_plotted: return @@ -241,20 +283,29 @@ class Monitor: for child in sorted(job.children, key=lambda k: k.name): node_child, skip = self._check_node_exists( exp, child, groups, hide_groups) + color, label = self._check_final_status(job, child) if len(node_child) == 0 and not skip: node_child = self._create_node(child, groups, hide_groups) if node_child: exp.add_node(node_child) - if job.name in child.edge_info and child.edge_info[job.name].get('optional', False): - exp.add_edge(pydotplus.Edge(node_job, node_child,style="dashed")) + if color: + # label = None doesn't disable label, instead it sets it to nothing and complain about invalid syntax + if label: + exp.add_edge(pydotplus.Edge(node_job, node_child,style="dashed",color=color,label=label)) + else: + exp.add_edge(pydotplus.Edge(node_job, node_child,style="dashed",color=color)) else: exp.add_edge(pydotplus.Edge(node_job, node_child)) else: skip = True elif not skip: node_child = node_child[0] - if job.name in child.edge_info and child.edge_info[job.name].get('optional', False): - exp.add_edge(pydotplus.Edge(node_job, node_child,style="dashed")) + if color: + # label = None doesn't disable label, instead it sets it to nothing and complain about invalid syntax + if label: + exp.add_edge(pydotplus.Edge(node_job, node_child, style="dashed", color=color, label=label)) + else: + exp.add_edge(pydotplus.Edge(node_job, node_child, style="dashed", color=color)) else: exp.add_edge(pydotplus.Edge(node_job, node_child)) skip = True -- GitLab From 8abb98c5a23f5a89dd6b685537fb8b2954fd20cb Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 10 Jul 2023 15:13:08 +0200 Subject: [PATCH 09/68] added some tests --- autosubmit/job/job_list.py | 168 +++++++++++++++++++++++---------- test/unit/test_dependencies.py | 64 ++++++++++--- 2 files changed, 168 insertions(+), 64 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 71b128daa..f29926ab4 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -439,94 +439,162 @@ class JobList(object): else: return False + + @staticmethod - def _parse_checkpoint(data): - checkpoint = {"STATUS": None, "FROM_STEP": None} - data = data.lower() - if data[0] == "r": - checkpoint["STATUS"] = Status.RUNNING - if len(data) > 1: - checkpoint["FROM_STEP"] = data[1:] - else: - checkpoint["FROM_STEP"] = "1" - elif data[0] == "f": - checkpoint["STATUS"] = Status.FAILED - if len(data) > 1: - checkpoint["FROM_STEP"] = data[1:] - else: - checkpoint["FROM_STEP"] = "1" - elif data[0] == "q": - checkpoint["STATUS"] = Status.QUEUING - elif data[0] == "s": - checkpoint["STATUS"] = Status.SUBMITTED - return checkpoint + def _parse_filters_to_check(list_of_values_to_check,value_list=[]): + final_values = [] + list_of_values_to_check = str(list_of_values_to_check).upper() + if list_of_values_to_check is None: + return None + elif list_of_values_to_check == "ALL": + return ["ALL"] + elif list_of_values_to_check == "NONE": + return None + elif list_of_values_to_check == "NATURAL": + return ["NATURAL"] + elif "," in list_of_values_to_check: + for value_to_check in list_of_values_to_check.split(","): + final_values.extend(JobList._parse_filter_to_check(value_to_check,value_list)) + else: + final_values = JobList._parse_filter_to_check(list_of_values_to_check,value_list) + return final_values + @staticmethod - def _parse_filter_to_check(value_to_check): + def _parse_filter_to_check(value_to_check,value_list=[]): """ Parse the filter to check and return the value to check. Selection process: value_to_check can be: a range: [0:], [:N], [0:N], [:-1], [0:N:M] ... - a value: N - a list of values : 0,2,4,5,7,10 ... + a value: N. a range with step: [0::M], [::2], [0::3], [::3] ... :param value_to_check: value to check. + :param value_list: list of values to check. Dates, members, chunks or splits. :return: parsed value to check. """ - # regex + value_to_check = str(value_to_check).upper() - if value_to_check is None: - return None - elif value_to_check == "ALL": - return "ALL" - elif value_to_check == "NONE": - return None - elif value_to_check == 1: + if value_to_check.count(":") == 1: # range - if value_to_check[0] == ":": + if value_to_check[1] == ":": # [:N] - return slice(None, int(value_to_check[1:])) - elif value_to_check[-1] == ":": + # Find N index in the list + start = None + end = value_to_check.split(":")[1].strip("[]") + # get index in the value_list + if len(value_list) > 0: + end = value_list.index(end) + else: + end = int(end) + elif value_to_check[-2] == ":": # [N:] - return slice(int(value_to_check[:-1]), None) + # Find N index in the list + start = value_to_check.split(":")[0].strip("[]") + end = None + # get index in the value_list + if len(value_list) > 0: + start = value_list.index(start) + else: + start = int(start) else: # [N:M] - return slice(int(value_to_check.split(":")[0]), int(value_to_check.split(":")[1])) + # Find N index in the list + start = value_to_check.split(":")[0].strip("[]") + end = value_to_check.split(":")[1].strip("[]") + step = None + # get index in the value_list + if len(value_list) > 0: + start = value_list.index(start) + end = value_list.index(end) + else: + start = int(start) + end = int(end) + if end is not None: + end+=1 + if len(value_list) > 0: + return value_list[slice(start, end)] + else: + return [ str(number_gen) for number_gen in range(start, end)] elif value_to_check.count(":") == 2: # range with step - if value_to_check[0] == ":": - # [::M] - return slice(None, None, int(value_to_check[2:])) - elif value_to_check[-1] == ":": - # [N::] - return slice(int(value_to_check[:-2]), None, None) + if value_to_check[-2] == ":" and value_to_check[-3] == ":": # [N::] + # Find N index in the list + start = value_to_check.split(":")[0].strip("[]") + end = None + step = None + # get index in the value_list + if len(value_list) > 0: + start = value_list.index(start) + else: + start = int(start) + elif value_to_check[1] == ":" and value_to_check[2] == ":": # [::S] + # Find N index in the list + start = None + end = None + step = value_to_check.split(":")[-1].strip("[]") + # get index in the value_list + step = int(step) + elif value_to_check[1] == ":" and value_to_check[-2] == ":": # [:M:] + # Find N index in the list + start = None + end = value_to_check.split(":")[1].strip("[]") + step = None + # get index in the value_list + if len(value_list) > 0: + end = value_list.index(end) + else: + end = int(end) + else: # [N:M:S] + # Find N index in the list + start = value_to_check.split(":")[0].strip("[]") + end = value_to_check.split(":")[1].strip("[]") + step = value_to_check.split(":")[2].strip("[]") + # get index in the value_list + if len(value_list) > 0: + start = value_list.index(start) + end = value_list.index(end) + else: + start = int(start) + end = int(end) + step = int(step) + if end is not None: + end+=1 + if len(value_list) > 0: + return value_list[slice(start, end, step)] else: - # [N::M] - return slice(int(value_to_check.split(":")[0]), None, int(value_to_check.split(":")[2])) - elif "," in value_to_check: - # list - return value_to_check.split(",") + return [str(number_gen) for number_gen in range(start, end, step)] else: # value - return value_to_check + return [value_to_check] + @staticmethod def _check_relationship(relationships, level_to_check, value_to_check): """ Check if the current_job_value is included in the filter_value :param relationships: current filter level to check. - :param level_to_check: Can be date_from, member_from, chunk_from, split_from. + :param level_to_check: Can be dates_from, members_from, chunks_from, splits_from. :param value_to_check: Can be None, a date, a member, a chunk or a split. :return: """ filters = [] + if level_to_check == "DATES_FROM": + values_list = self._date_list + elif level_to_check == "MEMBERS_FROM": + values_list = self._member_list + elif level_to_check == "CHUNKS_FROM": + values_list = self._chunk_list + else: + values_list = [] # need to obtain the MAX amount of splits set in the workflow + relationship = relationships.get(level_to_check, {}) status = relationship.pop("STATUS", relationships.get("STATUS", None)) from_step = relationship.pop("FROM_STEP", relationships.get("FROM_STEP", None)) for filter_range, filter_data in relationship.items(): if not value_to_check or str(value_to_check).upper() in str( - JobList._parse_filter_to_check(filter_range)).upper(): + JobList._parse_filters_to_check(filter_range)).upper(): if not filter_data.get("STATUS", None): filter_data["STATUS"] = status if not filter_data.get("FROM_STEP", None): diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 0af18081b..6b554f76a 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -1,17 +1,21 @@ -import unittest - import mock +import unittest from copy import deepcopy -from autosubmit.job.job_list import JobList +from datetime import datetime + from autosubmit.job.job import Job from autosubmit.job.job_common import Status -from datetime import datetime +from autosubmit.job.job_list import JobList + + class TestJobList(unittest.TestCase): def setUp(self): + self.date_list = ["20020201", "20020202", "20020203", "20020204", "20020205", "20020206", "20020207", "20020208", "20020209", "20020210"] + self.member_list = ["fc1", "fc2", "fc3", "fc4", "fc5", "fc6", "fc7", "fc8", "fc9", "fc10"] + self.chunk_list = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"] + self.split_list = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"] # Define common test case inputs here self.relationships_dates = { - "OPTIONAL": False, - "CHECKPOINT": None, "DATES_FROM": { "20020201": { "MEMBERS_FROM": { @@ -34,8 +38,6 @@ class TestJobList(unittest.TestCase): self.relationships_dates_optional["DATES_FROM"]["20020201"]["SPLITS_FROM"] = { "ALL": { "SPLITS_TO": "1?" } } self.relationships_members = { - "OPTIONAL": False, - "CHECKPOINT": None, "MEMBERS_FROM": { "fc2": { "SPLITS_FROM": { @@ -50,8 +52,6 @@ class TestJobList(unittest.TestCase): } } self.relationships_chunks = { - "OPTIONAL": False, - "CHECKPOINT": None, "CHUNKS_FROM": { "1": { "DATES_TO": "20020201", @@ -62,8 +62,6 @@ class TestJobList(unittest.TestCase): } } self.relationships_chunks2 = { - "OPTIONAL": False, - "CHECKPOINT": None, "CHUNKS_FROM": { "1": { "DATES_TO": "20020201", @@ -82,8 +80,6 @@ class TestJobList(unittest.TestCase): } self.relationships_splits = { - "OPTIONAL": False, - "CHECKPOINT": None, "SPLITS_FROM": { "1": { "DATES_TO": "20020201", @@ -170,6 +166,46 @@ class TestJobList(unittest.TestCase): "SPLITS_TO": "1?" } self.assertEqual(result, expected_output) + def test_parse_filters_to_check(self): + result = JobList._parse_filters_to_check("20020201,20020202,20020203",self.date_list) + expected_output = ["20020201","20020202","20020203"] + self.assertEqual(result, expected_output) + result = JobList._parse_filters_to_check("20020201,[20020203:20020205]",self.date_list) + + + def test_parse_filter_to_check(self): + # Call the function to get the result + # Value can have the following formats: + # a range: [0:], [:N], [0:N], [:-1], [0:N:M] ... + # a value: N + # a range with step: [0::M], [::2], [0::3], [::3] ... + result = JobList._parse_filter_to_check("20020201",self.date_list) + expected_output = ["20020201"] + self.assertEqual(result, expected_output) + result = JobList._parse_filter_to_check("[20020201:20020203]",self.date_list) + expected_output = ["20020201","20020202","20020203"] + self.assertEqual(result, expected_output) + result = JobList._parse_filter_to_check("[20020201:20020203:2]",self.date_list) + expected_output = ["20020201","20020203"] + self.assertEqual(result, expected_output) + result = JobList._parse_filter_to_check("[20020202:]",self.date_list) + expected_output = self.date_list[1:] + self.assertEqual(result, expected_output) + result = JobList._parse_filter_to_check("[:20020203]",self.date_list) + expected_output = self.date_list[:3] + self.assertEqual(result, expected_output) + result = JobList._parse_filter_to_check("[::2]",self.date_list) + expected_output = self.date_list[::2] + self.assertEqual(result, expected_output) + result = JobList._parse_filter_to_check("[20020203::]",self.date_list) + expected_output = self.date_list[2:] + self.assertEqual(result, expected_output) + result = JobList._parse_filter_to_check("[:20020203:]",self.date_list) + expected_output = self.date_list[:3] + self.assertEqual(result, expected_output) + + + def test_check_dates(self): # Call the function to get the result self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") -- GitLab From 280e4e78ba1b2937cb5a31dacf48237f27708c4d Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 12 Jul 2023 08:56:42 +0200 Subject: [PATCH 10/68] pipeline passing --- autosubmit/job/job.py | 43 ++++----- autosubmit/job/job_list.py | 63 +++++++------ test/unit/test_dependencies.py | 157 ++++++++++++++++----------------- test/unit/test_job.py | 15 ++-- 4 files changed, 137 insertions(+), 141 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index a584364fb..f622c9574 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -21,33 +21,33 @@ Main module for Autosubmit. Only contains an interface class to all functionality implemented on Autosubmit """ -import os -import re -import time -import json -import datetime -import textwrap from collections import OrderedDict -import copy +import copy +import datetime +import json import locale +import os +import re +import textwrap +import time +from bscearth.utils.date import date2str, parse_date, previous_day, chunk_end_date, chunk_start_date, Log, subs_dates +from functools import reduce +from threading import Thread +from time import sleep +from typing import List, Union -from autosubmitconfigparser.config.configcommon import AutosubmitConfig -from autosubmit.job.job_common import Status, Type, increase_wallclock_by_chunk +from autosubmit.helpers.parameters import autosubmit_parameter, autosubmit_parameters +from autosubmit.history.experiment_history import ExperimentHistory from autosubmit.job.job_common import StatisticsSnippetBash, StatisticsSnippetPython from autosubmit.job.job_common import StatisticsSnippetR, StatisticsSnippetEmpty +from autosubmit.job.job_common import Status, Type, increase_wallclock_by_chunk from autosubmit.job.job_utils import get_job_package_code -from autosubmitconfigparser.config.basicconfig import BasicConfig -from autosubmit.history.experiment_history import ExperimentHistory -from bscearth.utils.date import date2str, parse_date, previous_day, chunk_end_date, chunk_start_date, Log, subs_dates -from time import sleep -from threading import Thread from autosubmit.platforms.paramiko_submitter import ParamikoSubmitter -from log.log import Log, AutosubmitCritical, AutosubmitError -from typing import List, Union -from functools import reduce +from autosubmitconfigparser.config.basicconfig import BasicConfig +from autosubmitconfigparser.config.configcommon import AutosubmitConfig from autosubmitconfigparser.config.yamlparser import YAMLParserFactory -from autosubmit.helpers.parameters import autosubmit_parameter, autosubmit_parameters +from log.log import Log, AutosubmitCritical, AutosubmitError Log.get_logger("Autosubmit") @@ -256,6 +256,7 @@ class Job(object): def checkpoint(self): """Generates a checkpoint step for this job based on job.type""" return self._checkpoint + @checkpoint.setter def checkpoint(self): """Generates a checkpoint step for this job based on job.type""" @@ -268,11 +269,11 @@ class Job(object): def get_checkpoint_files(self): """ - Downloads checkpoint files from remote host. If they aren't already in local. - :param steps: list of steps to download - :return: the max step downloaded + Check if there is a file on the remote host that contains the checkpoint """ return self.platform.get_checkpoint_files(self) + + @property @autosubmit_parameter(name='sdate') def sdate(self): """Current start date.""" diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index f29926ab4..89d8238c9 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -569,9 +569,7 @@ class JobList(object): # value return [value_to_check] - - @staticmethod - def _check_relationship(relationships, level_to_check, value_to_check): + def _check_relationship(self, relationships, level_to_check, value_to_check): """ Check if the current_job_value is included in the filter_value :param relationships: current filter level to check. @@ -581,6 +579,10 @@ class JobList(object): """ filters = [] if level_to_check == "DATES_FROM": + try: + value_to_check = date2str(value_to_check, "%Y%m%d") + except: + pass values_list = self._date_list elif level_to_check == "MEMBERS_FROM": values_list = self._member_list @@ -593,8 +595,8 @@ class JobList(object): status = relationship.pop("STATUS", relationships.get("STATUS", None)) from_step = relationship.pop("FROM_STEP", relationships.get("FROM_STEP", None)) for filter_range, filter_data in relationship.items(): - if not value_to_check or str(value_to_check).upper() in str( - JobList._parse_filters_to_check(filter_range)).upper(): + if filter_range in ["ALL","NATURAL"] or ( not value_to_check or str(value_to_check).upper() in str( + JobList._parse_filters_to_check(filter_range)).upper()): if not filter_data.get("STATUS", None): filter_data["STATUS"] = status if not filter_data.get("FROM_STEP", None): @@ -605,8 +607,8 @@ class JobList(object): filters = [{}] return filters - @staticmethod - def _check_dates(relationships, current_job): + + def _check_dates(self, relationships, current_job): """ Check if the current_job_value is included in the filter_from and retrieve filter_to value :param relationships: Remaining filters to apply. @@ -614,7 +616,7 @@ class JobList(object): :return: filters_to_apply """ - filters_to_apply = JobList._check_relationship(relationships, "DATES_FROM", date2str(current_job.date)) + filters_to_apply = self._check_relationship(relationships, "DATES_FROM", date2str(current_job.date)) # there could be multiple filters that apply... per example # Current task date is 20020201, and member is fc2 # Dummy example, not specially usefull in a real case @@ -643,17 +645,17 @@ class JobList(object): # Will enter, go recursivily to the similar methods and in the end it will do: # Will enter members_from, and obtain [{DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", CHUNKS_FROM{...}] if "MEMBERS_FROM" in filter: - filters_to_apply_m = JobList._check_members({"MEMBERS_FROM": (filter.pop("MEMBERS_FROM"))}, current_job) + filters_to_apply_m = self._check_members({"MEMBERS_FROM": (filter.pop("MEMBERS_FROM"))}, current_job) if len(filters_to_apply_m) > 0: filters_to_apply[i].update(filters_to_apply_m) # Will enter chunks_from, and obtain [{DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"] if "CHUNKS_FROM" in filter: - filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM"))}, current_job) + filters_to_apply_c = self._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM"))}, current_job) if len(filters_to_apply_c) > 0 and len(filters_to_apply_c[0]) > 0: filters_to_apply[i].update(filters_to_apply_c) # IGNORED if "SPLITS_FROM" in filter: - filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM"))}, current_job) + filters_to_apply_s = self._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM"))}, current_job) if len(filters_to_apply_s) > 0: filters_to_apply[i].update(filters_to_apply_s) # Unify filters from all filters_from where the current job is included to have a single SET of filters_to @@ -661,29 +663,28 @@ class JobList(object): # {DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"} return filters_to_apply - @staticmethod - def _check_members(relationships, current_job): + + def _check_members(self,relationships, current_job): """ Check if the current_job_value is included in the filter_from and retrieve filter_to value :param relationships: Remaining filters to apply. :param current_job: Current job to check. :return: filters_to_apply """ - filters_to_apply = JobList._check_relationship(relationships, "MEMBERS_FROM", current_job.member) + filters_to_apply = self._check_relationship(relationships, "MEMBERS_FROM", current_job.member) for i, filter_ in enumerate(filters_to_apply): if "CHUNKS_FROM" in filter_: - filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter_.pop("CHUNKS_FROM"))}, current_job) + filters_to_apply_c = self._check_chunks({"CHUNKS_FROM": (filter_.pop("CHUNKS_FROM"))}, current_job) if len(filters_to_apply_c) > 0: filters_to_apply[i].update(filters_to_apply_c) if "SPLITS_FROM" in filter_: - filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter_.pop("SPLITS_FROM"))}, current_job) + filters_to_apply_s = self._check_splits({"SPLITS_FROM": (filter_.pop("SPLITS_FROM"))}, current_job) if len(filters_to_apply_s) > 0: filters_to_apply[i].update(filters_to_apply_s) filters_to_apply = JobList._unify_to_filters(filters_to_apply) return filters_to_apply - @staticmethod - def _check_chunks(relationships, current_job): + def _check_chunks(self,relationships, current_job): """ Check if the current_job_value is included in the filter_from and retrieve filter_to value :param relationships: Remaining filters to apply. @@ -691,17 +692,16 @@ class JobList(object): :return: filters_to_apply """ - filters_to_apply = JobList._check_relationship(relationships, "CHUNKS_FROM", current_job.chunk) + filters_to_apply = self._check_relationship(relationships, "CHUNKS_FROM", current_job.chunk) for i, filter in enumerate(filters_to_apply): if "SPLITS_FROM" in filter: - filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM"))}, current_job) + filters_to_apply_s = self._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM"))}, current_job) if len(filters_to_apply_s) > 0: filters_to_apply[i].update(filters_to_apply_s) filters_to_apply = JobList._unify_to_filters(filters_to_apply) return filters_to_apply - @staticmethod - def _check_splits(relationships, current_job): + def _check_splits(self,relationships, current_job): """ Check if the current_job_value is included in the filter_from and retrieve filter_to value :param relationships: Remaining filters to apply. @@ -709,7 +709,7 @@ class JobList(object): :return: filters_to_apply """ - filters_to_apply = JobList._check_relationship(relationships, "SPLITS_FROM", current_job.split) + filters_to_apply = self._check_relationship(relationships, "SPLITS_FROM", current_job.split) # No more FROM sections to check, unify _to FILTERS and return filters_to_apply = JobList._unify_to_filters(filters_to_apply) return filters_to_apply @@ -780,8 +780,7 @@ class JobList(object): JobList._normalize_to_filters(unified_filter, "SPLITS_TO") return unified_filter - @staticmethod - def _filter_current_job(current_job, relationships): + def _filter_current_job(self,current_job, relationships): ''' This function will filter the current job based on the relationships given :param current_job: Current job to filter :param relationships: Relationships to apply @@ -808,13 +807,13 @@ class JobList(object): if relationships is not None and len(relationships) > 0: # Look for a starting point, this can be if else becasue they're exclusive as a DATE_FROM can't be in a MEMBER_FROM and so on if "DATES_FROM" in relationships: - filters_to_apply = JobList._check_dates(relationships, current_job) + filters_to_apply = self._check_dates(relationships, current_job) elif "MEMBERS_FROM" in relationships: - filters_to_apply = JobList._check_members(relationships, current_job) + filters_to_apply = self._check_members(relationships, current_job) elif "CHUNKS_FROM" in relationships: - filters_to_apply = JobList._check_chunks(relationships, current_job) + filters_to_apply = self._check_chunks(relationships, current_job) elif "SPLITS_FROM" in relationships: - filters_to_apply = JobList._check_splits(relationships, current_job) + filters_to_apply = self._check_splits(relationships, current_job) else: relationships.pop("OPTIONAL", None) relationships.pop("CHECKPOINT", None) @@ -928,10 +927,8 @@ class JobList(object): natural_jobs = dic_jobs.get_jobs(dependency.section, date, member, chunk) all_parents = list(set(other_parents + parents_jobs)) # Get dates_to, members_to, chunks_to of the deepest level of the relationship. - filters_to_apply = JobList._filter_current_job(job, copy.deepcopy(dependency.relationships)) - if "?" in filters_to_apply.get("SPLITS_TO", "") or "?" in filters_to_apply.get("DATES_TO", - "") or "?" in filters_to_apply.get( - "MEMBERS_TO", "") or "?" in filters_to_apply.get("CHUNKS_TO", ""): + filters_to_apply = self._filter_current_job(job, copy.deepcopy(dependency.relationships)) + if "?" in filters_to_apply.get("SPLITS_TO", "") or "?" in filters_to_apply.get("DATES_TO","") or "?" in filters_to_apply.get("MEMBERS_TO", "") or "?" in filters_to_apply.get("CHUNKS_TO", ""): only_marked_status = True else: only_marked_status = False diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 6b554f76a..007aac369 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -1,4 +1,6 @@ +import inspect import mock +import tempfile import unittest from copy import deepcopy from datetime import datetime @@ -6,14 +8,50 @@ from datetime import datetime from autosubmit.job.job import Job from autosubmit.job.job_common import Status from autosubmit.job.job_list import JobList +from autosubmit.job.job_list_persistence import JobListPersistenceDb +from autosubmitconfigparser.config.yamlparser import YAMLParserFactory +class FakeBasicConfig: + def __init__(self): + pass + def props(self): + pr = {} + for name in dir(self): + value = getattr(self, name) + if not name.startswith('__') and not inspect.ismethod(value) and not inspect.isfunction(value): + pr[name] = value + return pr + DB_DIR = '/dummy/db/dir' + DB_FILE = '/dummy/db/file' + DB_PATH = '/dummy/db/path' + LOCAL_ROOT_DIR = '/dummy/local/root/dir' + LOCAL_TMP_DIR = '/dummy/local/temp/dir' + LOCAL_PROJ_DIR = '/dummy/local/proj/dir' + DEFAULT_PLATFORMS_CONF = '' + DEFAULT_JOBS_CONF = '' + class TestJobList(unittest.TestCase): def setUp(self): + self.experiment_id = 'random-id' + self.as_conf = mock.Mock() + self.as_conf.experiment_data = dict() + self.as_conf.experiment_data["JOBS"] = dict() + self.as_conf.jobs_data = self.as_conf.experiment_data["JOBS"] + self.as_conf.experiment_data["PLATFORMS"] = dict() + self.temp_directory = tempfile.mkdtemp() + self.JobList = JobList(self.experiment_id, FakeBasicConfig, YAMLParserFactory(), + JobListPersistenceDb(self.temp_directory, 'db'), self.as_conf) self.date_list = ["20020201", "20020202", "20020203", "20020204", "20020205", "20020206", "20020207", "20020208", "20020209", "20020210"] self.member_list = ["fc1", "fc2", "fc3", "fc4", "fc5", "fc6", "fc7", "fc8", "fc9", "fc10"] self.chunk_list = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"] self.split_list = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"] + self.JobList._date_list = self.date_list + self.JobList._member_list = self.member_list + self.JobList._chunk_list = self.chunk_list + self.JobList._split_list = self.split_list + + # Define common test case inputs here self.relationships_dates = { "DATES_FROM": { @@ -109,69 +147,30 @@ class TestJobList(unittest.TestCase): self.mock_job.chunk = None self.mock_job.split = None - def test_parse_checkpoint(self): - data = "r2" - correct = {"FROM_STEP": '2', "STATUS":Status.RUNNING} - result = JobList._parse_checkpoint(data) - self.assertEqual(result, correct) - data = "r" - correct = {"FROM_STEP": '1', "STATUS":Status.RUNNING} - result = JobList._parse_checkpoint(data) - self.assertEqual(result, correct) - data = "f2" - correct = {"FROM_STEP": '2', "STATUS":Status.FAILED} - result = JobList._parse_checkpoint(data) - self.assertEqual(result, correct) - data = "f" - correct = {"FROM_STEP": '1', "STATUS":Status.FAILED} - result = JobList._parse_checkpoint(data) - self.assertEqual(result, correct) - data = "s" - correct = {"FROM_STEP": None, "STATUS":Status.SUBMITTED} - result = JobList._parse_checkpoint(data) - self.assertEqual(result, correct) - data = "s2" - correct = {"FROM_STEP": None, "STATUS":Status.SUBMITTED} - result = JobList._parse_checkpoint(data) - self.assertEqual(result, correct) - data = "q" - correct = {"FROM_STEP": None, "STATUS":Status.QUEUING} - result = JobList._parse_checkpoint(data) - self.assertEqual(result, correct) - data = "q2" - correct = {"FROM_STEP": None, "STATUS":Status.QUEUING} - result = JobList._parse_checkpoint(data) - self.assertEqual(result, correct) - - def test_simple_dependency(self): - result_d = JobList._check_dates({}, self.mock_job) - result_m = JobList._check_members({}, self.mock_job) - result_c = JobList._check_chunks({}, self.mock_job) - result_s = JobList._check_splits({}, self.mock_job) + result_d = self.JobList._check_dates({}, self.mock_job) + result_m = self.JobList._check_members({}, self.mock_job) + result_c = self.JobList._check_chunks({}, self.mock_job) + result_s = self.JobList._check_splits({}, self.mock_job) self.assertEqual(result_d, {}) self.assertEqual(result_m, {}) self.assertEqual(result_c, {}) self.assertEqual(result_s, {}) - def test_check_dates_optional(self): - self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") - self.mock_job.member = "fc2" - self.mock_job.chunk = 1 - self.mock_job.split = 1 - result = JobList._check_dates(self.relationships_dates_optional, self.mock_job) - expected_output = { - "DATES_TO": "20020201?", - "MEMBERS_TO": "fc2?", - "CHUNKS_TO": "ALL?", - "SPLITS_TO": "1?" - } - self.assertEqual(result, expected_output) + def test_parse_filters_to_check(self): - result = JobList._parse_filters_to_check("20020201,20020202,20020203",self.date_list) + """Test the _parse_filters_to_check function""" + result = self.JobList._parse_filters_to_check("20020201,20020202,20020203",self.date_list) expected_output = ["20020201","20020202","20020203"] self.assertEqual(result, expected_output) - result = JobList._parse_filters_to_check("20020201,[20020203:20020205]",self.date_list) - + result = self.JobList._parse_filters_to_check("20020201,[20020203:20020205]",self.date_list) + expected_output = ["20020201","20020203","20020204","20020205"] + self.assertEqual(result, expected_output) + result = self.JobList._parse_filters_to_check("[20020201:20020203],[20020205:20020207]",self.date_list) + expected_output = ["20020201","20020202","20020203","20020205","20020206","20020207"] + self.assertEqual(result, expected_output) + result = self.JobList._parse_filters_to_check("20020201",self.date_list) + expected_output = ["20020201"] + self.assertEqual(result, expected_output) def test_parse_filter_to_check(self): # Call the function to get the result @@ -179,40 +178,38 @@ class TestJobList(unittest.TestCase): # a range: [0:], [:N], [0:N], [:-1], [0:N:M] ... # a value: N # a range with step: [0::M], [::2], [0::3], [::3] ... - result = JobList._parse_filter_to_check("20020201",self.date_list) + result = self.JobList._parse_filter_to_check("20020201",self.date_list) expected_output = ["20020201"] self.assertEqual(result, expected_output) - result = JobList._parse_filter_to_check("[20020201:20020203]",self.date_list) + result = self.JobList._parse_filter_to_check("[20020201:20020203]",self.date_list) expected_output = ["20020201","20020202","20020203"] self.assertEqual(result, expected_output) - result = JobList._parse_filter_to_check("[20020201:20020203:2]",self.date_list) + result = self.JobList._parse_filter_to_check("[20020201:20020203:2]",self.date_list) expected_output = ["20020201","20020203"] self.assertEqual(result, expected_output) - result = JobList._parse_filter_to_check("[20020202:]",self.date_list) + result = self.JobList._parse_filter_to_check("[20020202:]",self.date_list) expected_output = self.date_list[1:] self.assertEqual(result, expected_output) - result = JobList._parse_filter_to_check("[:20020203]",self.date_list) + result = self.JobList._parse_filter_to_check("[:20020203]",self.date_list) expected_output = self.date_list[:3] self.assertEqual(result, expected_output) - result = JobList._parse_filter_to_check("[::2]",self.date_list) + result = self.JobList._parse_filter_to_check("[::2]",self.date_list) expected_output = self.date_list[::2] self.assertEqual(result, expected_output) - result = JobList._parse_filter_to_check("[20020203::]",self.date_list) + result = self.JobList._parse_filter_to_check("[20020203::]",self.date_list) expected_output = self.date_list[2:] self.assertEqual(result, expected_output) - result = JobList._parse_filter_to_check("[:20020203:]",self.date_list) + result = self.JobList._parse_filter_to_check("[:20020203:]",self.date_list) expected_output = self.date_list[:3] self.assertEqual(result, expected_output) - - def test_check_dates(self): # Call the function to get the result self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") self.mock_job.member = "fc2" self.mock_job.chunk = 1 self.mock_job.split = 1 - result = JobList._check_dates(self.relationships_dates, self.mock_job) + result = self.JobList._check_dates(self.relationships_dates, self.mock_job) expected_output = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", @@ -221,14 +218,14 @@ class TestJobList(unittest.TestCase): } self.assertEqual(result, expected_output) self.mock_job.date = datetime.strptime("20020202", "%Y%m%d") - result = JobList._check_dates(self.relationships_dates, self.mock_job) + result = self.JobList._check_dates(self.relationships_dates, self.mock_job) self.assertEqual(result, {}) def test_check_members(self): # Call the function to get the result self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") self.mock_job.member = "fc2" - result = JobList._check_members(self.relationships_members, self.mock_job) + result = self.JobList._check_members(self.relationships_members, self.mock_job) expected_output = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", @@ -237,14 +234,14 @@ class TestJobList(unittest.TestCase): } self.assertEqual(result, expected_output) self.mock_job.member = "fc3" - result = JobList._check_members(self.relationships_members, self.mock_job) + result = self.JobList._check_members(self.relationships_members, self.mock_job) self.assertEqual(result, {}) def test_check_splits(self): # Call the function to get the result self.mock_job.split = 1 - result = JobList._check_splits(self.relationships_splits, self.mock_job) + result = self.JobList._check_splits(self.relationships_splits, self.mock_job) expected_output = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", @@ -253,13 +250,13 @@ class TestJobList(unittest.TestCase): } self.assertEqual(result, expected_output) self.mock_job.split = 2 - result = JobList._check_splits(self.relationships_splits, self.mock_job) + result = self.JobList._check_splits(self.relationships_splits, self.mock_job) self.assertEqual(result, {}) def test_check_chunks(self): # Call the function to get the result self.mock_job.chunk = 1 - result = JobList._check_chunks(self.relationships_chunks, self.mock_job) + result = self.JobList._check_chunks(self.relationships_chunks, self.mock_job) expected_output = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", @@ -268,17 +265,17 @@ class TestJobList(unittest.TestCase): } self.assertEqual(result, expected_output) self.mock_job.chunk = 2 - result = JobList._check_chunks(self.relationships_chunks, self.mock_job) + result = self.JobList._check_chunks(self.relationships_chunks, self.mock_job) self.assertEqual(result, {}) # test splits_from self.mock_job.split = 5 - result = JobList._check_chunks(self.relationships_chunks2, self.mock_job) + result = self.JobList._check_chunks(self.relationships_chunks2, self.mock_job) expected_output2 = { "SPLITS_TO": "2" } self.assertEqual(result, expected_output2) self.mock_job.split = 1 - result = JobList._check_chunks(self.relationships_chunks2, self.mock_job) + result = self.JobList._check_chunks(self.relationships_chunks2, self.mock_job) self.assertEqual(result, {}) def test_check_general(self): @@ -288,7 +285,7 @@ class TestJobList(unittest.TestCase): self.mock_job.member = "fc2" self.mock_job.chunk = 1 self.mock_job.split = 1 - result = JobList._filter_current_job(self.mock_job,self.relationships_general) + result = self.JobList._filter_current_job(self.mock_job,self.relationships_general) expected_output = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", @@ -317,17 +314,17 @@ class TestJobList(unittest.TestCase): self.mock_job.member = "fc2" self.mock_job.chunk = 1 self.mock_job.split = 1 - result = JobList._valid_parent(self.mock_job, date_list, member_list, chunk_list, is_a_natural_relation, filter_) + result = self.JobList._valid_parent(self.mock_job, date_list, member_list, chunk_list, is_a_natural_relation, filter_) # it returns a tuple, the first element is the result, the second is the optional flag - self.assertEqual(result, (True,False)) + self.assertEqual(result, True) filter_ = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", "CHUNKS_TO": "ALL", "SPLITS_TO": "1?" } - result = JobList._valid_parent(self.mock_job, date_list, member_list, chunk_list, is_a_natural_relation, filter_) - self.assertEqual(result, (True,True)) + result = self.JobList._valid_parent(self.mock_job, date_list, member_list, chunk_list, is_a_natural_relation, filter_) + self.assertEqual(result, True) if __name__ == '__main__': diff --git a/test/unit/test_job.py b/test/unit/test_job.py index caaf9c60a..f1bfbcbac 100644 --- a/test/unit/test_job.py +++ b/test/unit/test_job.py @@ -1,17 +1,19 @@ from unittest import TestCase + +import datetime +import inspect import os import sys -from autosubmitconfigparser.config.configcommon import AutosubmitConfig -from autosubmit.job.job_common import Status -from autosubmit.job.job import Job -from autosubmit.platforms.platform import Platform from mock import Mock, MagicMock from mock import patch -import datetime - # compatibility with both versions (2 & 3) from sys import version_info +from autosubmit.job.job import Job +from autosubmit.job.job_common import Status +from autosubmit.platforms.platform import Platform +from autosubmitconfigparser.config.configcommon import AutosubmitConfig + if version_info.major == 2: import builtins as builtins else: @@ -362,7 +364,6 @@ class TestJob(TestCase): self.job.date_format = test[1] self.assertEquals(test[2], self.job.sdate) -import inspect class FakeBasicConfig: def __init__(self): pass -- GitLab From db9161202401301212034ee05cc525bf902fceb6 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 12 Jul 2023 09:04:24 +0200 Subject: [PATCH 11/68] comment change --- autosubmit/job/job_list.py | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 89d8238c9..f15ae29e6 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -615,35 +615,9 @@ class JobList(object): :param current_job: Current job to check. :return: filters_to_apply """ - + # Check the test_dependencies.py to see how to use this function filters_to_apply = self._check_relationship(relationships, "DATES_FROM", date2str(current_job.date)) - # there could be multiple filters that apply... per example - # Current task date is 20020201, and member is fc2 - # Dummy example, not specially usefull in a real case - # DATES_FROM: - # all: - # MEMBERS_FROM: - # ALL: ... - # CHUNKS_FROM: - # ALL: ... - # 20020201: - # MEMBERS_FROM: - # fc2: - # DATES_TO: "20020201" - # MEMBERS_TO: "fc2" - # CHUNKS_TO: "ALL" - # SPLITS_FROM: - # ALL: - # SPLITS_TO: "1" - # this "for" iterates for ALL and fc2 as current task is selected in both filters - # The dict in this step is: - # [{MEMBERS_FROM{..},CHUNKS_FROM{...}},{MEMBERS_FROM{..},SPLITS_FROM{...}}] for i, filter in enumerate(filters_to_apply): - # {MEMBERS_FROM{..},CHUNKS_FROM{...}} I want too look ALL filters not only one, but I want to go recursivily until get the _TO filter - # This is not an if_else, because the current level ( dates ) could have two different filters. - # Second case commented: ( date_from 20020201 ) - # Will enter, go recursivily to the similar methods and in the end it will do: - # Will enter members_from, and obtain [{DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", CHUNKS_FROM{...}] if "MEMBERS_FROM" in filter: filters_to_apply_m = self._check_members({"MEMBERS_FROM": (filter.pop("MEMBERS_FROM"))}, current_job) if len(filters_to_apply_m) > 0: -- GitLab From 83a26a4102cd2fcb48b3be67ca92b7648dc95ad2 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 13 Jul 2023 08:54:12 +0200 Subject: [PATCH 12/68] apply_parent changed to use the same function than in the other place --- autosubmit/job/job_list.py | 40 ++++++---------------------------- test/unit/test_dependencies.py | 5 ++--- 2 files changed, 9 insertions(+), 36 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index f15ae29e6..2bbba5fb0 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -397,44 +397,18 @@ class JobList(object): :return: boolean """ - to_filter = [] - # strip special chars if any filter_value = filter_value.strip("?") - if str(parent_value).lower().find("none") != -1: + if "NONE" in str(parent_value).upper(): return True - if filter_value.lower().find("all") != -1: + to_filter = JobList._parse_filter_to_check(filter_value,associative_list) + if "ALL" in to_filter: return True - elif filter_value.lower().find("natural") != -1: + elif "NATURAL" in to_filter: if parent_value is None or parent_value in associative_list: return True - elif filter_value.lower().find("none") != -1: + elif "NONE" in to_filter: return False - elif filter_value.find(",") != -1: - aux_filter = filter_value.split(",") - if filter_type not in ["chunks", "splits"]: - for value in aux_filter: - if str(value).isdigit(): - to_filter.append(associative_list[int(value)]) - else: - to_filter.append(value) - else: - to_filter = aux_filter - del aux_filter - elif filter_value.find(":") != -1: - start_end = filter_value.split(":") - start = start_end[0].strip("[]") - end = start_end[1].strip("[]") - del start_end - if filter_type not in ["chunks", "splits"]: # chunk directly - for value in range(int(start), int(end) + 1): - to_filter.append(value) - else: # index - for value in range(int(start + 1), int(end) + 1): - to_filter.append(value) - else: - to_filter.append(filter_value) - - if str(parent_value).upper() in str(to_filter).upper(): + elif str(parent_value).upper() in to_filter: return True else: return False @@ -596,7 +570,7 @@ class JobList(object): from_step = relationship.pop("FROM_STEP", relationships.get("FROM_STEP", None)) for filter_range, filter_data in relationship.items(): if filter_range in ["ALL","NATURAL"] or ( not value_to_check or str(value_to_check).upper() in str( - JobList._parse_filters_to_check(filter_range)).upper()): + JobList._parse_filters_to_check(filter_range,values_list)).upper()): if not filter_data.get("STATUS", None): filter_data["STATUS"] = status if not filter_data.get("FROM_STEP", None): diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 007aac369..4b4c19f83 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -297,7 +297,6 @@ class TestJobList(unittest.TestCase): def test_valid_parent(self): # Call the function to get the result - date_list = ["20020201"] member_list = ["fc1", "fc2", "fc3"] chunk_list = [1, 2, 3] @@ -314,7 +313,7 @@ class TestJobList(unittest.TestCase): self.mock_job.member = "fc2" self.mock_job.chunk = 1 self.mock_job.split = 1 - result = self.JobList._valid_parent(self.mock_job, date_list, member_list, chunk_list, is_a_natural_relation, filter_) + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) # it returns a tuple, the first element is the result, the second is the optional flag self.assertEqual(result, True) filter_ = { @@ -323,7 +322,7 @@ class TestJobList(unittest.TestCase): "CHUNKS_TO": "ALL", "SPLITS_TO": "1?" } - result = self.JobList._valid_parent(self.mock_job, date_list, member_list, chunk_list, is_a_natural_relation, filter_) + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) self.assertEqual(result, True) -- GitLab From ccf4695375470cd50ceb1ffe4725274371997136 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 13 Jul 2023 09:01:20 +0200 Subject: [PATCH 13/68] added additional tests --- autosubmit/job/job_list.py | 15 ++++++++++++--- test/unit/test_dependencies.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 2bbba5fb0..dea212622 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -481,7 +481,10 @@ class JobList(object): # get index in the value_list if len(value_list) > 0: start = value_list.index(start) - end = value_list.index(end) + try: + end = value_list.index(end) + except ValueError: + end = len(value_list)-1 else: start = int(start) end = int(end) @@ -517,7 +520,10 @@ class JobList(object): step = None # get index in the value_list if len(value_list) > 0: - end = value_list.index(end) + try: + end = value_list.index(end) + except ValueError: + end = len(value_list)-1 else: end = int(end) else: # [N:M:S] @@ -528,7 +534,10 @@ class JobList(object): # get index in the value_list if len(value_list) > 0: start = value_list.index(start) - end = value_list.index(end) + try: + end = value_list.index(end) + except ValueError: + end = len(value_list)-1 else: start = int(start) end = int(end) diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 4b4c19f83..6ef7d5161 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -324,6 +324,34 @@ class TestJobList(unittest.TestCase): } result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) self.assertEqual(result, True) + filter_ = { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1?" + } + self.mock_job.split = 2 + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) + self.assertEqual(result, False) + filter_ = { + "DATES_TO": "[20020201:20020205]", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + self.mock_job.split = 1 + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) + self.assertEqual(result, True) + filter_ = { + "DATES_TO": "[20020201:20020205]", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + self.mock_job.date = datetime.strptime("20020206", "%Y%m%d") + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) + self.assertEqual(result, False) + if __name__ == '__main__': -- GitLab From 19d738dda0c418e3908baa2bae1c579095eade64 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 13 Jul 2023 12:50:03 +0200 Subject: [PATCH 14/68] added additional tests --- autosubmit/job/job.py | 7 +++++-- autosubmit/job/job_list.py | 27 ++++++++++++++++----------- test/unit/test_dependencies.py | 11 +++++++++++ 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index f622c9574..f2b2c44f7 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1369,7 +1369,8 @@ class Job(object): return parameters def update_job_parameters(self,as_conf, parameters): - parameters["AS_CHECKPOINT"] = self.checkpoint + if self.checkpoint: # To activate placeholder sustitution per in the template + parameters["AS_CHECKPOINT"] = self.checkpoint parameters['JOBNAME'] = self.name parameters['FAIL_COUNT'] = str(self.fail_count) parameters['SDATE'] = self.sdate @@ -1451,6 +1452,8 @@ class Job(object): parameters['EXPORT'] = self.export parameters['PROJECT_TYPE'] = as_conf.get_project_type() self.wchunkinc = as_conf.get_wchunkinc(self.section) + for key,value in as_conf.jobs_data.get(self.section,{}).items(): + parameters["CURRENT_"+key.upper()] = value return parameters def update_parameters(self, as_conf, parameters, @@ -1522,7 +1525,7 @@ class Job(object): template_file.close() else: if self.type == Type.BASH: - template = '%AS_CHECKPOINT%;sleep 320;%AS_CHECKPOINT%;sleep 320' + template = '%CURRENT_TESTNAME%;%AS_CHECKPOINT%;sleep 320;%AS_CHECKPOINT%;sleep 320' elif self.type == Type.PYTHON2: template = 'time.sleep(5)' + "\n" elif self.type == Type.PYTHON3 or self.type == Type.PYTHON: diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index dea212622..d78496883 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -401,14 +401,16 @@ class JobList(object): if "NONE" in str(parent_value).upper(): return True to_filter = JobList._parse_filter_to_check(filter_value,associative_list) - if "ALL" in to_filter: + if len(to_filter) == 0: + return False + elif "ALL".casefold() == str(to_filter[0]).casefold(): return True - elif "NATURAL" in to_filter: + elif "NATURAL" == str(to_filter[0]).casefold(): if parent_value is None or parent_value in associative_list: return True - elif "NONE" in to_filter: + elif "NONE" == str(to_filter[0]).casefold(): return False - elif str(parent_value).upper() in to_filter: + elif str(parent_value).casefold() in ( str(filter_).casefold() for filter_ in to_filter): return True else: return False @@ -448,8 +450,8 @@ class JobList(object): :param value_list: list of values to check. Dates, members, chunks or splits. :return: parsed value to check. """ - - value_to_check = str(value_to_check).upper() + if len(value_list) > 0 and type(value_list[0]) == str: # We dont want to cast split or chunk values + value_to_check = str(value_to_check).upper() if value_to_check.count(":") == 1: # range if value_to_check[1] == ":": @@ -563,16 +565,19 @@ class JobList(object): filters = [] if level_to_check == "DATES_FROM": try: - value_to_check = date2str(value_to_check, "%Y%m%d") + value_to_check = date2str(value_to_check, "%Y%m%d") # need to convert in some cases except: pass - values_list = self._date_list + try: + values_list = [date2str(date_, "%Y%m%d") for date_ in self._date_list] # need to convert in some cases + except: + values_list = self._date_list elif level_to_check == "MEMBERS_FROM": - values_list = self._member_list + values_list = self._member_list # Str list elif level_to_check == "CHUNKS_FROM": - values_list = self._chunk_list + values_list = self._chunk_list # int list else: - values_list = [] # need to obtain the MAX amount of splits set in the workflow + values_list = [] # splits, int list ( artificially generated later ) relationship = relationships.get(level_to_check, {}) status = relationship.pop("STATUS", relationships.get("STATUS", None)) diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 6ef7d5161..9f3fe855c 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -351,6 +351,17 @@ class TestJobList(unittest.TestCase): self.mock_job.date = datetime.strptime("20020206", "%Y%m%d") result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) self.assertEqual(result, False) + filter_ = { + "DATES_TO": "[20020201:20020205]", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "[2:4]", + "SPLITS_TO": "[1:5]" + } + self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") + self.mock_job.chunk = 2 + self.mock_job.split = 1 + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) + self.assertEqual(result, True) -- GitLab From ee49da23c2e5edbfe381e7569efcebe3273a3fdc Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 13 Jul 2023 15:08:00 +0200 Subject: [PATCH 15/68] fix --- test/unit/test_dependencies.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 9f3fe855c..a1d6fbe45 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -300,6 +300,7 @@ class TestJobList(unittest.TestCase): date_list = ["20020201"] member_list = ["fc1", "fc2", "fc3"] chunk_list = [1, 2, 3] + self.mock_job.splits = 10 is_a_natural_relation = False # Filter_to values filter_ = { @@ -331,6 +332,7 @@ class TestJobList(unittest.TestCase): "SPLITS_TO": "1?" } self.mock_job.split = 2 + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) self.assertEqual(result, False) filter_ = { -- GitLab From 055c43bcbedf10c3abb6dcfc02c05a83772acb25 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 19 Jul 2023 14:56:44 +0200 Subject: [PATCH 16/68] Some rework, may be finally ready to review --- autosubmit/job/job_list.py | 212 +++++++++++++++++---------------- test/unit/test_checkpoints.py | 148 +++++++++++++++++++++++ test/unit/test_dependencies.py | 124 +++++++++++++------ test/unit/test_job_list.py | 1 - 4 files changed, 346 insertions(+), 139 deletions(-) create mode 100644 test/unit/test_checkpoints.py diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index d78496883..90f2448e0 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -387,7 +387,7 @@ class JobList(object): return splits @staticmethod - def _apply_filter(parent_value, filter_value, associative_list, filter_type="dates"): + def _apply_filter(parent_value, filter_value, associative_list, level_to_check="DATES_FROM"): """ Check if the current_job_value is included in the filter_value :param parent_value: @@ -398,17 +398,19 @@ class JobList(object): :return: boolean """ filter_value = filter_value.strip("?") - if "NONE" in str(parent_value).upper(): + if "NONE".casefold() in str(parent_value).casefold(): return True - to_filter = JobList._parse_filter_to_check(filter_value,associative_list) - if len(to_filter) == 0: + to_filter = JobList._parse_filters_to_check(filter_value,associative_list,level_to_check) + if to_filter is None: + return False + elif len(to_filter) == 0: return False elif "ALL".casefold() == str(to_filter[0]).casefold(): return True - elif "NATURAL" == str(to_filter[0]).casefold(): + elif "NATURAL".casefold() == str(to_filter[0]).casefold(): if parent_value is None or parent_value in associative_list: return True - elif "NONE" == str(to_filter[0]).casefold(): + elif "NONE".casefold() == str(to_filter[0]).casefold(): return False elif str(parent_value).casefold() in ( str(filter_).casefold() for filter_ in to_filter): return True @@ -418,27 +420,27 @@ class JobList(object): @staticmethod - def _parse_filters_to_check(list_of_values_to_check,value_list=[]): + def _parse_filters_to_check(list_of_values_to_check,value_list=[],level_to_check="DATES_FROM"): final_values = [] list_of_values_to_check = str(list_of_values_to_check).upper() if list_of_values_to_check is None: return None - elif list_of_values_to_check == "ALL": + elif list_of_values_to_check.casefold() == "ALL".casefold() : return ["ALL"] - elif list_of_values_to_check == "NONE": - return None - elif list_of_values_to_check == "NATURAL": + elif list_of_values_to_check.casefold() == "NONE".casefold(): + return ["NONE"] + elif list_of_values_to_check.casefold() == "NATURAL".casefold(): return ["NATURAL"] elif "," in list_of_values_to_check: for value_to_check in list_of_values_to_check.split(","): - final_values.extend(JobList._parse_filter_to_check(value_to_check,value_list)) + final_values.extend(JobList._parse_filter_to_check(value_to_check,value_list,level_to_check)) else: - final_values = JobList._parse_filter_to_check(list_of_values_to_check,value_list) + final_values = JobList._parse_filter_to_check(list_of_values_to_check,value_list,level_to_check) return final_values @staticmethod - def _parse_filter_to_check(value_to_check,value_list=[]): + def _parse_filter_to_check(value_to_check,value_list=[],level_to_check="DATES_FROM"): """ Parse the filter to check and return the value to check. Selection process: @@ -450,8 +452,7 @@ class JobList(object): :param value_list: list of values to check. Dates, members, chunks or splits. :return: parsed value to check. """ - if len(value_list) > 0 and type(value_list[0]) == str: # We dont want to cast split or chunk values - value_to_check = str(value_to_check).upper() + step = 1 if value_to_check.count(":") == 1: # range if value_to_check[1] == ":": @@ -459,54 +460,32 @@ class JobList(object): # Find N index in the list start = None end = value_to_check.split(":")[1].strip("[]") - # get index in the value_list - if len(value_list) > 0: - end = value_list.index(end) - else: + if level_to_check in ["CHUNKS_FROM","SPLITS_FROM"]: end = int(end) elif value_to_check[-2] == ":": # [N:] # Find N index in the list start = value_to_check.split(":")[0].strip("[]") - end = None - # get index in the value_list - if len(value_list) > 0: - start = value_list.index(start) - else: + if level_to_check in ["CHUNKS_FROM","SPLITS_FROM"]: start = int(start) + end = None else: # [N:M] # Find N index in the list start = value_to_check.split(":")[0].strip("[]") end = value_to_check.split(":")[1].strip("[]") - step = None - # get index in the value_list - if len(value_list) > 0: - start = value_list.index(start) - try: - end = value_list.index(end) - except ValueError: - end = len(value_list)-1 - else: + step = 1 + if level_to_check in ["CHUNKS_FROM","SPLITS_FROM"]: start = int(start) end = int(end) - if end is not None: - end+=1 - if len(value_list) > 0: - return value_list[slice(start, end)] - else: - return [ str(number_gen) for number_gen in range(start, end)] elif value_to_check.count(":") == 2: # range with step if value_to_check[-2] == ":" and value_to_check[-3] == ":": # [N::] # Find N index in the list start = value_to_check.split(":")[0].strip("[]") end = None - step = None - # get index in the value_list - if len(value_list) > 0: - start = value_list.index(start) - else: + step = 1 + if level_to_check in ["CHUNKS_FROM","SPLITS_FROM"]: start = int(start) elif value_to_check[1] == ":" and value_to_check[2] == ":": # [::S] # Find N index in the list @@ -519,40 +498,37 @@ class JobList(object): # Find N index in the list start = None end = value_to_check.split(":")[1].strip("[]") - step = None - # get index in the value_list - if len(value_list) > 0: - try: - end = value_list.index(end) - except ValueError: - end = len(value_list)-1 - else: + if level_to_check in ["CHUNKS_FROM","SPLITS_FROM"]: end = int(end) + step = 1 else: # [N:M:S] # Find N index in the list start = value_to_check.split(":")[0].strip("[]") end = value_to_check.split(":")[1].strip("[]") step = value_to_check.split(":")[2].strip("[]") - # get index in the value_list - if len(value_list) > 0: - start = value_list.index(start) - try: - end = value_list.index(end) - except ValueError: - end = len(value_list)-1 - else: + step = int(step) + if level_to_check in ["CHUNKS_FROM","SPLITS_FROM"]: start = int(start) end = int(end) - step = int(step) - if end is not None: - end+=1 - if len(value_list) > 0: - return value_list[slice(start, end, step)] - else: - return [str(number_gen) for number_gen in range(start, end, step)] else: # value return [value_to_check] + ## values to return + if len(value_list) > 0: + if start is None: + start = value_list[0] + if end is None: + end = value_list[-1] + try: + return value_list[slice(value_list.index(start), value_list.index(end)+1, int(step))] + except ValueError: + return value_list[slice(0,len(value_list)-1,int(step))] + else: + if not start: + start = 0 + if end is None: + return [] + return [number_gen for number_gen in range(int(start), int(end)+1, int(step))] def _check_relationship(self, relationships, level_to_check, value_to_check): """ @@ -583,8 +559,8 @@ class JobList(object): status = relationship.pop("STATUS", relationships.get("STATUS", None)) from_step = relationship.pop("FROM_STEP", relationships.get("FROM_STEP", None)) for filter_range, filter_data in relationship.items(): - if filter_range in ["ALL","NATURAL"] or ( not value_to_check or str(value_to_check).upper() in str( - JobList._parse_filters_to_check(filter_range,values_list)).upper()): + if filter_range.casefold() in ["ALL".casefold(),"NATURAL".casefold()] or ( not value_to_check or str(value_to_check).upper() in str( + JobList._parse_filters_to_check(filter_range,values_list,level_to_check)).upper()): if not filter_data.get("STATUS", None): filter_data["STATUS"] = status if not filter_data.get("FROM_STEP", None): @@ -621,7 +597,7 @@ class JobList(object): if len(filters_to_apply_s) > 0: filters_to_apply[i].update(filters_to_apply_s) # Unify filters from all filters_from where the current job is included to have a single SET of filters_to - filters_to_apply = JobList._unify_to_filters(filters_to_apply) + filters_to_apply = self._unify_to_filters(filters_to_apply) # {DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"} return filters_to_apply @@ -643,7 +619,7 @@ class JobList(object): filters_to_apply_s = self._check_splits({"SPLITS_FROM": (filter_.pop("SPLITS_FROM"))}, current_job) if len(filters_to_apply_s) > 0: filters_to_apply[i].update(filters_to_apply_s) - filters_to_apply = JobList._unify_to_filters(filters_to_apply) + filters_to_apply = self._unify_to_filters(filters_to_apply) return filters_to_apply def _check_chunks(self,relationships, current_job): @@ -660,7 +636,7 @@ class JobList(object): filters_to_apply_s = self._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM"))}, current_job) if len(filters_to_apply_s) > 0: filters_to_apply[i].update(filters_to_apply_s) - filters_to_apply = JobList._unify_to_filters(filters_to_apply) + filters_to_apply = self._unify_to_filters(filters_to_apply) return filters_to_apply def _check_splits(self,relationships, current_job): @@ -673,11 +649,10 @@ class JobList(object): filters_to_apply = self._check_relationship(relationships, "SPLITS_FROM", current_job.split) # No more FROM sections to check, unify _to FILTERS and return - filters_to_apply = JobList._unify_to_filters(filters_to_apply) + filters_to_apply = self._unify_to_filters(filters_to_apply) return filters_to_apply - @staticmethod - def _unify_to_filter(unified_filter, filter_to, filter_type): + def _unify_to_filter(self,unified_filter, filter_to, filter_type): """ Unify filter_to filters into a single dictionary :param unified_filter: Single dictionary with all filters_to @@ -685,21 +660,49 @@ class JobList(object): :param filter_type: "DATES_TO", "MEMBERS_TO", "CHUNKS_TO", "SPLITS_TO" :return: unified_filter """ - if "all" not in unified_filter[filter_type]: + if filter_type == "DATES_TO": + value_list = self._date_list + level_to_check = "DATES_FROM" + elif filter_type == "MEMBERS_TO": + value_list = self._member_list + level_to_check = "MEMBERS_FROM" + elif filter_type == "CHUNKS_TO": + value_list = self._chunk_list + level_to_check = "CHUNKS_FROM" + elif filter_type == "SPLITS_TO": + value_list = self._split_list + level_to_check = "SPLITS_FROM" + if "all".casefold() not in unified_filter[filter_type].casefold(): aux = filter_to.pop(filter_type, None) if aux: aux = aux.split(",") for element in aux: - # Get only the first alphanumeric part - parsed_element = re.findall(r"[\w']+", element)[0].lower() - # Get the rest - data = element[len(parsed_element):] - if parsed_element in ["natural", "none"] and len(unified_filter[filter_type]) > 0: + if element == "": + continue + # Get only the first alphanumeric part and [:] chars + parsed_element = re.findall(r"([\[:\]a-zA-Z0-9]+)", element)[0].lower() + extra_data = element[len(parsed_element):] + parsed_element = JobList._parse_filter_to_check(parsed_element, value_list = value_list, level_to_check = filter_type) + # convert list to str + skip = False + if isinstance(parsed_element, list): + # check if any element is natural or none + for ele in parsed_element: + if ele.lower() in ["natural", "none"]: + skip = True + else: + if parsed_element.lower() in ["natural", "none"]: + skip = True + if skip and len(unified_filter[filter_type]) > 0: continue else: - if "?" not in element: - element += data - unified_filter[filter_type].add(element) + for ele in parsed_element: + if ele not in unified_filter[filter_type]: + if len(unified_filter[filter_type]) > 0 and unified_filter[filter_type][-1] == ",": + unified_filter[filter_type] += ele + extra_data + else: + unified_filter[filter_type] += "," + ele + extra_data + "," + return unified_filter @staticmethod def _normalize_to_filters(filter_to, filter_type): @@ -709,32 +712,35 @@ class JobList(object): :param filter_type: "DATES_TO", "MEMBERS_TO", "CHUNKS_TO", "SPLITS_TO" :return: """ - if len(filter_to[filter_type]) == 0: + if len(filter_to[filter_type]) == 0 or ("," in filter_to[filter_type] and len(filter_to[filter_type]) == 1): filter_to.pop(filter_type, None) - elif "all" in filter_to[filter_type]: + elif "all".casefold() in filter_to[filter_type]: filter_to[filter_type] = "all" else: - # transform to str separated by commas if multiple elements - filter_to[filter_type] = ",".join(filter_to[filter_type]) + # delete last comma + if "," in filter_to[filter_type][-1]: + filter_to[filter_type] = filter_to[filter_type][:-1] + # delete first comma + if "," in filter_to[filter_type][0]: + filter_to[filter_type] = filter_to[filter_type][1:] - @staticmethod - def _unify_to_filters(filter_to_apply): + def _unify_to_filters(self,filter_to_apply): """ Unify all filter_to filters into a single dictionary ( of current selection ) :param filter_to_apply: Filters to apply :return: Single dictionary with all filters_to """ - unified_filter = {"DATES_TO": set(), "MEMBERS_TO": set(), "CHUNKS_TO": set(), "SPLITS_TO": set()} + unified_filter = {"DATES_TO": "", "MEMBERS_TO": "", "CHUNKS_TO": "", "SPLITS_TO": ""} for filter_to in filter_to_apply: if "STATUS" not in unified_filter and filter_to.get("STATUS", None): unified_filter["STATUS"] = filter_to["STATUS"] if "FROM_STEP" not in unified_filter and filter_to.get("FROM_STEP", None): unified_filter["FROM_STEP"] = filter_to["FROM_STEP"] if len(filter_to) > 0: - JobList._unify_to_filter(unified_filter, filter_to, "DATES_TO") - JobList._unify_to_filter(unified_filter, filter_to, "MEMBERS_TO") - JobList._unify_to_filter(unified_filter, filter_to, "CHUNKS_TO") - JobList._unify_to_filter(unified_filter, filter_to, "SPLITS_TO") + self._unify_to_filter(unified_filter, filter_to, "DATES_TO") + self._unify_to_filter(unified_filter, filter_to, "MEMBERS_TO") + self._unify_to_filter(unified_filter, filter_to, "CHUNKS_TO") + self._unify_to_filter(unified_filter, filter_to, "SPLITS_TO") JobList._normalize_to_filters(unified_filter, "DATES_TO") JobList._normalize_to_filters(unified_filter, "MEMBERS_TO") @@ -777,8 +783,6 @@ class JobList(object): elif "SPLITS_FROM" in relationships: filters_to_apply = self._check_splits(relationships, current_job) else: - relationships.pop("OPTIONAL", None) - relationships.pop("CHECKPOINT", None) relationships.pop("CHUNKS_FROM", None) relationships.pop("MEMBERS_FROM", None) relationships.pop("DATES_FROM", None) @@ -2079,17 +2083,19 @@ class JobList(object): if status == "ALL": continue for job in sorted_job_list: + if job.status != Status.WAITING: + continue if status in ["RUNNING", "FAILED"]: if job.platform.connected: # This will be true only when used under setstatus/run job.get_checkpoint_files() for parent in job.edge_info[status].values(): - if parent[0].status == Status.WAITING: - if status in ["RUNNING", "FAILED"] and parent[1] and int(parent[1]) >= job.current_checkpoint_step: - continue - else: - jobs_to_check.append(parent[0]) + if status in ["RUNNING", "FAILED"] and parent[1] and int(parent[1]) >= job.current_checkpoint_step: + continue + else: + jobs_to_check.append(parent[0]) return jobs_to_check + def update_list(self, as_conf, store_change=True, fromSetStatus=False, submitter=None, first_time=False): # type: (AutosubmitConfig, bool, bool, object, bool) -> bool """ diff --git a/test/unit/test_checkpoints.py b/test/unit/test_checkpoints.py new file mode 100644 index 000000000..8772e3ae2 --- /dev/null +++ b/test/unit/test_checkpoints.py @@ -0,0 +1,148 @@ +from unittest import TestCase + +import inspect +import shutil +import tempfile +from mock import Mock +from random import randrange + +from autosubmit.job.job import Job +from autosubmit.job.job_common import Status +from autosubmit.job.job_list import JobList +from autosubmit.job.job_list_persistence import JobListPersistenceDb +from autosubmitconfigparser.config.yamlparser import YAMLParserFactory + + +class TestJobList(TestCase): + def setUp(self): + self.experiment_id = 'random-id' + self.as_conf = Mock() + self.as_conf.experiment_data = dict() + self.as_conf.experiment_data["JOBS"] = dict() + self.as_conf.jobs_data = self.as_conf.experiment_data["JOBS"] + self.as_conf.experiment_data["PLATFORMS"] = dict() + self.temp_directory = tempfile.mkdtemp() + self.job_list = JobList(self.experiment_id, FakeBasicConfig, YAMLParserFactory(), + JobListPersistenceDb(self.temp_directory, 'db'), self.as_conf) + + # creating jobs for self list + self.completed_job = self._createDummyJobWithStatus(Status.COMPLETED) + self.completed_job2 = self._createDummyJobWithStatus(Status.COMPLETED) + self.completed_job3 = self._createDummyJobWithStatus(Status.COMPLETED) + self.completed_job4 = self._createDummyJobWithStatus(Status.COMPLETED) + + self.submitted_job = self._createDummyJobWithStatus(Status.SUBMITTED) + self.submitted_job2 = self._createDummyJobWithStatus(Status.SUBMITTED) + self.submitted_job3 = self._createDummyJobWithStatus(Status.SUBMITTED) + + self.running_job = self._createDummyJobWithStatus(Status.RUNNING) + self.running_job2 = self._createDummyJobWithStatus(Status.RUNNING) + + self.queuing_job = self._createDummyJobWithStatus(Status.QUEUING) + + self.failed_job = self._createDummyJobWithStatus(Status.FAILED) + self.failed_job2 = self._createDummyJobWithStatus(Status.FAILED) + self.failed_job3 = self._createDummyJobWithStatus(Status.FAILED) + self.failed_job4 = self._createDummyJobWithStatus(Status.FAILED) + + self.ready_job = self._createDummyJobWithStatus(Status.READY) + self.ready_job2 = self._createDummyJobWithStatus(Status.READY) + self.ready_job3 = self._createDummyJobWithStatus(Status.READY) + + self.waiting_job = self._createDummyJobWithStatus(Status.WAITING) + self.waiting_job2 = self._createDummyJobWithStatus(Status.WAITING) + + self.unknown_job = self._createDummyJobWithStatus(Status.UNKNOWN) + + + self.job_list._job_list = [self.completed_job, self.completed_job2, self.completed_job3, self.completed_job4, + self.submitted_job, self.submitted_job2, self.submitted_job3, self.running_job, + self.running_job2, self.queuing_job, self.failed_job, self.failed_job2, + self.failed_job3, self.failed_job4, self.ready_job, self.ready_job2, + self.ready_job3, self.waiting_job, self.waiting_job2, self.unknown_job] + self.waiting_job.parents.add(self.ready_job) + self.waiting_job.parents.add(self.completed_job) + self.waiting_job.parents.add(self.failed_job) + self.waiting_job.parents.add(self.submitted_job) + self.waiting_job.parents.add(self.running_job) + self.waiting_job.parents.add(self.queuing_job) + + def tearDown(self) -> None: + shutil.rmtree(self.temp_directory) + + def test_add_edge_job(self): + special_variables = dict() + special_variables["STATUS"] = Status.COMPLETED + special_variables["FROM_STEP"] = 0 + for p in self.waiting_job.parents: + self.waiting_job.add_edge_info(p, special_variables) + for parent in self.waiting_job.parents: + self.assertEqual(self.waiting_job.edge_info[special_variables["STATUS"]][parent.name], + (parent, special_variables.get("FROM_STEP", 0))) + + + def test_add_edge_info_joblist(self): + special_conditions = dict() + special_conditions["STATUS"] = Status.COMPLETED + special_conditions["FROM_STEP"] = 0 + self.job_list._add_edge_info(self.waiting_job, special_conditions["STATUS"]) + self.assertEqual(len(self.job_list.jobs_edges.get(Status.COMPLETED,[])),1) + self.job_list._add_edge_info(self.waiting_job2, special_conditions["STATUS"]) + self.assertEqual(len(self.job_list.jobs_edges.get(Status.COMPLETED,[])),2) + + def test_check_special_status(self): + self.waiting_job.edge_info = dict() + + self.job_list.jobs_edges = dict() + # Adds edge info for waiting_job in the list + self.job_list._add_edge_info(self.waiting_job, Status.COMPLETED) + self.job_list._add_edge_info(self.waiting_job, Status.READY) + self.job_list._add_edge_info(self.waiting_job, Status.RUNNING) + self.job_list._add_edge_info(self.waiting_job, Status.SUBMITTED) + self.job_list._add_edge_info(self.waiting_job, Status.QUEUING) + self.job_list._add_edge_info(self.waiting_job, Status.FAILED) + # Adds edge info for waiting_job + special_variables = dict() + for p in self.waiting_job.parents: + special_variables["STATUS"] = p.status + special_variables["FROM_STEP"] = 0 + self.waiting_job.add_edge_info(p,special_variables) + # call to special status + jobs_to_check = self.job_list.check_special_status() + for job in jobs_to_check: + tmp = [parent for parent in job.parents if + parent.status == Status.COMPLETED or parent in self.jobs_edges["ALL"]] + assert len(tmp) == len(job.parents) + self.waiting_job.add_parent(self.waiting_job2) + for job in jobs_to_check: + tmp = [parent for parent in job.parents if + parent.status == Status.COMPLETED or parent in self.jobs_edges["ALL"]] + assert len(tmp) == len(job.parents) + + + + def _createDummyJobWithStatus(self, status): + job_name = str(randrange(999999, 999999999)) + job_id = randrange(1, 999) + job = Job(job_name, job_id, status, 0) + job.type = randrange(0, 2) + return job + +class FakeBasicConfig: + def __init__(self): + pass + def props(self): + pr = {} + for name in dir(self): + value = getattr(self, name) + if not name.startswith('__') and not inspect.ismethod(value) and not inspect.isfunction(value): + pr[name] = value + return pr + DB_DIR = '/dummy/db/dir' + DB_FILE = '/dummy/db/file' + DB_PATH = '/dummy/db/path' + LOCAL_ROOT_DIR = '/dummy/local/root/dir' + LOCAL_TMP_DIR = '/dummy/local/temp/dir' + LOCAL_PROJ_DIR = '/dummy/local/proj/dir' + DEFAULT_PLATFORMS_CONF = '' + DEFAULT_JOBS_CONF = '' diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index a1d6fbe45..085700ac6 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -44,8 +44,8 @@ class TestJobList(unittest.TestCase): JobListPersistenceDb(self.temp_directory, 'db'), self.as_conf) self.date_list = ["20020201", "20020202", "20020203", "20020204", "20020205", "20020206", "20020207", "20020208", "20020209", "20020210"] self.member_list = ["fc1", "fc2", "fc3", "fc4", "fc5", "fc6", "fc7", "fc8", "fc9", "fc10"] - self.chunk_list = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"] - self.split_list = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"] + self.chunk_list = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + self.split_list = [1, 2, 3, 4, 5] self.JobList._date_list = self.date_list self.JobList._member_list = self.member_list self.JobList._chunk_list = self.chunk_list @@ -58,9 +58,9 @@ class TestJobList(unittest.TestCase): "20020201": { "MEMBERS_FROM": { "fc2": { - "DATES_TO": "20020201", + "DATES_TO": "[20020201:20020202]*,20020203", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL" + "CHUNKS_TO": "all" } }, "SPLITS_FROM": { @@ -72,7 +72,7 @@ class TestJobList(unittest.TestCase): } } self.relationships_dates_optional = deepcopy(self.relationships_dates) - self.relationships_dates_optional["DATES_FROM"]["20020201"]["MEMBERS_FROM"] = { "fc2?": { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", "CHUNKS_TO": "ALL", "SPLITS_TO": "5" } } + self.relationships_dates_optional["DATES_FROM"]["20020201"]["MEMBERS_FROM"] = { "fc2?": { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", "CHUNKS_TO": "all", "SPLITS_TO": "5" } } self.relationships_dates_optional["DATES_FROM"]["20020201"]["SPLITS_FROM"] = { "ALL": { "SPLITS_TO": "1?" } } self.relationships_members = { @@ -82,7 +82,7 @@ class TestJobList(unittest.TestCase): "ALL": { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" } } @@ -94,7 +94,7 @@ class TestJobList(unittest.TestCase): "1": { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" } } @@ -104,7 +104,7 @@ class TestJobList(unittest.TestCase): "1": { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" }, "2": { @@ -122,7 +122,7 @@ class TestJobList(unittest.TestCase): "1": { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" } } @@ -131,7 +131,7 @@ class TestJobList(unittest.TestCase): self.relationships_general = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" } # Create a mock Job object @@ -147,6 +147,36 @@ class TestJobList(unittest.TestCase): self.mock_job.chunk = None self.mock_job.split = None + def test_unify_to_filter(self): + """Test the _unify_to_fitler function""" + # :param unified_filter: Single dictionary with all filters_to + # :param filter_to: Current dictionary that contains the filters_to + # :param filter_type: "DATES_TO", "MEMBERS_TO", "CHUNKS_TO", "SPLITS_TO" + # :return: unified_filter + unified_filter = \ + { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "all", + "SPLITS_TO": "1" + } + filter_to = \ + { + "DATES_TO": "20020205,[20020207:20020208]", + "MEMBERS_TO": "fc2,fc3", + "CHUNKS_TO": "all" + } + filter_type = "DATES_TO" + result = self.JobList._unify_to_filter(unified_filter, filter_to, filter_type) + expected_output = \ + { + "DATES_TO": "20020201,20020205,20020207,20020208,", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "all", + "SPLITS_TO": "1" + } + self.assertEqual(result, expected_output) + def test_simple_dependency(self): result_d = self.JobList._check_dates({}, self.mock_job) result_m = self.JobList._check_members({}, self.mock_job) @@ -202,6 +232,19 @@ class TestJobList(unittest.TestCase): result = self.JobList._parse_filter_to_check("[:20020203:]",self.date_list) expected_output = self.date_list[:3] self.assertEqual(result, expected_output) + # test with a member N:N + result = self.JobList._parse_filter_to_check("[fc2:fc3]",self.member_list) + expected_output = ["fc2","fc3"] + self.assertEqual(result, expected_output) + # test with a chunk + result = self.JobList._parse_filter_to_check("[1:2]",self.chunk_list,level_to_check="CHUNKS_FROM") + expected_output = [1,2] + self.assertEqual(result, expected_output) + # test with a split + result = self.JobList._parse_filter_to_check("[1:2]",self.split_list,level_to_check="SPLITS_FROM") + expected_output = [1,2] + self.assertEqual(result, expected_output) + def test_check_dates(self): # Call the function to get the result @@ -211,15 +254,13 @@ class TestJobList(unittest.TestCase): self.mock_job.split = 1 result = self.JobList._check_dates(self.relationships_dates, self.mock_job) expected_output = { - "DATES_TO": "20020201", + "DATES_TO": "20020201*,20020202*,20020203", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" } self.assertEqual(result, expected_output) - self.mock_job.date = datetime.strptime("20020202", "%Y%m%d") - result = self.JobList._check_dates(self.relationships_dates, self.mock_job) - self.assertEqual(result, {}) + def test_check_members(self): # Call the function to get the result self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") @@ -229,7 +270,7 @@ class TestJobList(unittest.TestCase): expected_output = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" } self.assertEqual(result, expected_output) @@ -237,6 +278,7 @@ class TestJobList(unittest.TestCase): result = self.JobList._check_members(self.relationships_members, self.mock_job) self.assertEqual(result, {}) + def test_check_splits(self): # Call the function to get the result @@ -245,13 +287,14 @@ class TestJobList(unittest.TestCase): expected_output = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" } self.assertEqual(result, expected_output) self.mock_job.split = 2 result = self.JobList._check_splits(self.relationships_splits, self.mock_job) self.assertEqual(result, {}) + def test_check_chunks(self): # Call the function to get the result @@ -260,23 +303,23 @@ class TestJobList(unittest.TestCase): expected_output = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" } self.assertEqual(result, expected_output) self.mock_job.chunk = 2 result = self.JobList._check_chunks(self.relationships_chunks, self.mock_job) self.assertEqual(result, {}) - # test splits_from - self.mock_job.split = 5 - result = self.JobList._check_chunks(self.relationships_chunks2, self.mock_job) - expected_output2 = { - "SPLITS_TO": "2" - } - self.assertEqual(result, expected_output2) - self.mock_job.split = 1 - result = self.JobList._check_chunks(self.relationships_chunks2, self.mock_job) - self.assertEqual(result, {}) + # # test splits_from + # self.mock_job.split = 5 + # result = self.JobList._check_chunks(self.relationships_chunks2, self.mock_job) + # expected_output2 = { + # "SPLITS_TO": "2" + # } + # self.assertEqual(result, expected_output2) + # self.mock_job.split = 1 + # result = self.JobList._check_chunks(self.relationships_chunks2, self.mock_job) + # self.assertEqual(result, {}) def test_check_general(self): # Call the function to get the result @@ -289,7 +332,7 @@ class TestJobList(unittest.TestCase): expected_output = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" } self.assertEqual(result, expected_output) @@ -297,7 +340,7 @@ class TestJobList(unittest.TestCase): def test_valid_parent(self): # Call the function to get the result - date_list = ["20020201"] + date_list = ["20020201", "20020202", "20020203", "20020204", "20020205", "20020206", "20020207", "20020208", "20020209", "20020210"] member_list = ["fc1", "fc2", "fc3"] chunk_list = [1, 2, 3] self.mock_job.splits = 10 @@ -306,7 +349,7 @@ class TestJobList(unittest.TestCase): filter_ = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" } # PArent job values @@ -320,7 +363,7 @@ class TestJobList(unittest.TestCase): filter_ = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1?" } result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) @@ -328,7 +371,7 @@ class TestJobList(unittest.TestCase): filter_ = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1?" } self.mock_job.split = 2 @@ -338,7 +381,7 @@ class TestJobList(unittest.TestCase): filter_ = { "DATES_TO": "[20020201:20020205]", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" } self.mock_job.split = 1 @@ -347,7 +390,7 @@ class TestJobList(unittest.TestCase): filter_ = { "DATES_TO": "[20020201:20020205]", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" } self.mock_job.date = datetime.strptime("20020206", "%Y%m%d") @@ -364,6 +407,17 @@ class TestJobList(unittest.TestCase): self.mock_job.split = 1 result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) self.assertEqual(result, True) + filter_ = { + "DATES_TO": "[20020201:20020202],20020203,20020204,20020205", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "[2:4]*,5", + "SPLITS_TO": "[1:5],6" + } + self.mock_job.date = datetime.strptime("20020204", "%Y%m%d") + self.mock_job.chunk = 5 + self.mock_job.split = 6 + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) + self.assertEqual(result, True) diff --git a/test/unit/test_job_list.py b/test/unit/test_job_list.py index 0a3f6b3b4..e546b764d 100644 --- a/test/unit/test_job_list.py +++ b/test/unit/test_job_list.py @@ -275,7 +275,6 @@ class TestJobList(TestCase): job.type = randrange(0, 2) return job -import inspect class FakeBasicConfig: def __init__(self): pass -- GitLab From 9da19f52995ed2468181c609df80ecd51bbb06b5 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 19 Jul 2023 15:56:56 +0200 Subject: [PATCH 17/68] Added failure tests --- test/unit/test_dependencies.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 085700ac6..175a6e3e9 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -260,6 +260,11 @@ class TestJobList(unittest.TestCase): "SPLITS_TO": "1" } self.assertEqual(result, expected_output) + # failure + self.mock_job.date = datetime.strptime("20020301", "%Y%m%d") + result = self.JobList._check_dates(self.relationships_dates, self.mock_job) + self.assertEqual(result, {}) + def test_check_members(self): # Call the function to get the result @@ -277,6 +282,10 @@ class TestJobList(unittest.TestCase): self.mock_job.member = "fc3" result = self.JobList._check_members(self.relationships_members, self.mock_job) self.assertEqual(result, {}) + # FAILURE + self.mock_job.member = "fc99" + result = self.JobList._check_members(self.relationships_members, self.mock_job) + self.assertEqual(result, {}) def test_check_splits(self): @@ -294,6 +303,10 @@ class TestJobList(unittest.TestCase): self.mock_job.split = 2 result = self.JobList._check_splits(self.relationships_splits, self.mock_job) self.assertEqual(result, {}) + # failure + self.mock_job.split = 99 + result = self.JobList._check_splits(self.relationships_splits, self.mock_job) + self.assertEqual(result, {}) def test_check_chunks(self): # Call the function to get the result @@ -310,16 +323,13 @@ class TestJobList(unittest.TestCase): self.mock_job.chunk = 2 result = self.JobList._check_chunks(self.relationships_chunks, self.mock_job) self.assertEqual(result, {}) - # # test splits_from - # self.mock_job.split = 5 - # result = self.JobList._check_chunks(self.relationships_chunks2, self.mock_job) - # expected_output2 = { - # "SPLITS_TO": "2" - # } - # self.assertEqual(result, expected_output2) - # self.mock_job.split = 1 - # result = self.JobList._check_chunks(self.relationships_chunks2, self.mock_job) - # self.assertEqual(result, {}) + # failure + self.mock_job.chunk = 99 + result = self.JobList._check_chunks(self.relationships_chunks, self.mock_job) + self.assertEqual(result, {}) + + + def test_check_general(self): # Call the function to get the result -- GitLab From 6c80abff2f364e970ab8a0c1b4f226407ba08695 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 20 Jul 2023 08:46:24 +0200 Subject: [PATCH 18/68] Ready --- autosubmit/job/job_list.py | 29 ++++++++++++++++------ test/unit/test_dependencies.py | 45 +++++++++++++++++++++++++--------- 2 files changed, 56 insertions(+), 18 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 90f2448e0..b7918dfcd 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -412,7 +412,7 @@ class JobList(object): return True elif "NONE".casefold() == str(to_filter[0]).casefold(): return False - elif str(parent_value).casefold() in ( str(filter_).casefold() for filter_ in to_filter): + elif len( [ filter_ for filter_ in to_filter if str(parent_value).casefold() in str(filter_).casefold() ] )>0: return True else: return False @@ -791,7 +791,7 @@ class JobList(object): return filters_to_apply @staticmethod - def _valid_parent(parent, member_list, date_list, chunk_list, is_a_natural_relation, filter_): + def _valid_parent(parent, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child): ''' Check if the parent is valid for the current job :param parent: job to check @@ -815,6 +815,7 @@ class JobList(object): members_to = str(filter_.get("MEMBERS_TO", "natural")).lower() chunks_to = str(filter_.get("CHUNKS_TO", "natural")).lower() splits_to = str(filter_.get("SPLITS_TO", "natural")).lower() + if not is_a_natural_relation: if dates_to == "natural": dates_to = "none" @@ -824,15 +825,29 @@ class JobList(object): chunks_to = "none" if splits_to == "natural": splits_to = "none" - if dates_to == "natural": + if "natural" in dates_to: associative_list["dates"] = [date2str(parent.date)] if parent.date is not None else date_list - if members_to == "natural": + if "natural" in members_to: associative_list["members"] = [parent.member] if parent.member is not None else member_list - if chunks_to == "natural": + if "natural" in chunks_to: associative_list["chunks"] = [parent.chunk] if parent.chunk is not None else chunk_list - if splits_to == "natural": + if "natural" in splits_to: associative_list["splits"] = [parent.split] if parent.split is not None else parent.splits parsed_parent_date = date2str(parent.date) if parent.date is not None else None + # Check for each * char in the filters + # Get all the dates that match * in the filter in a list separated by , + if "*" in dates_to: + dates_to = [ dat for dat in date_list.split(",") if dat is not None and "*" not in dat or ("*" in dat and date2str(child.date,"%Y%m%d") in dat) ] + dates_to = ",".join(dates_to) + if "*" in members_to: + members_to = [ mem for mem in member_list.split(",") if mem is not None and "*" not in mem or ("*" in mem and str(child.member) in mem) ] + members_to = ",".join(members_to) + if "*" in chunks_to: + chunks_to = [ chu for chu in chunk_list.split(",") if chu is not None and "*" not in chu or ("*" in chu and str(child.chunk) in chu) ] + chunks_to = ",".join(chunks_to) + if "*" in splits_to: + splits_to = [ spl for spl in splits_to.split(",") if child.split is None or spl is None or "*" not in spl or ("*" in spl and str(child.split) in spl) ] + splits_to = ",".join(splits_to) # Apply all filters to look if this parent is an appropriated candidate for the current_job valid_dates = JobList._apply_filter(parsed_parent_date, dates_to, associative_list["dates"], "dates") valid_members = JobList._apply_filter(parent.member, members_to, associative_list["members"], "members") @@ -913,7 +928,7 @@ class JobList(object): # If the parent is valid, add it to the graph if JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship, - filters_to_apply): + filters_to_apply,child): job.add_parent(parent) self._add_edge(graph, job, parent) # Could be more variables in the future diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 175a6e3e9..dbf93565e 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -1,3 +1,4 @@ +import copy import inspect import mock import tempfile @@ -134,6 +135,12 @@ class TestJobList(unittest.TestCase): "CHUNKS_TO": "all", "SPLITS_TO": "1" } + self.relationships_general_1_to_1 = { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "all", + "SPLITS_TO": "1*,2*,3*,4*,5*" + } # Create a mock Job object self.mock_job = mock.MagicMock(spec=Job) @@ -349,6 +356,7 @@ class TestJobList(unittest.TestCase): def test_valid_parent(self): + child = copy.deepcopy(self.mock_job) # Call the function to get the result date_list = ["20020201", "20020202", "20020203", "20020204", "20020205", "20020206", "20020207", "20020208", "20020209", "20020210"] member_list = ["fc1", "fc2", "fc3"] @@ -367,7 +375,7 @@ class TestJobList(unittest.TestCase): self.mock_job.member = "fc2" self.mock_job.chunk = 1 self.mock_job.split = 1 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) # it returns a tuple, the first element is the result, the second is the optional flag self.assertEqual(result, True) filter_ = { @@ -376,7 +384,7 @@ class TestJobList(unittest.TestCase): "CHUNKS_TO": "all", "SPLITS_TO": "1?" } - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) self.assertEqual(result, True) filter_ = { "DATES_TO": "20020201", @@ -386,7 +394,7 @@ class TestJobList(unittest.TestCase): } self.mock_job.split = 2 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) self.assertEqual(result, False) filter_ = { "DATES_TO": "[20020201:20020205]", @@ -395,7 +403,7 @@ class TestJobList(unittest.TestCase): "SPLITS_TO": "1" } self.mock_job.split = 1 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) self.assertEqual(result, True) filter_ = { "DATES_TO": "[20020201:20020205]", @@ -404,7 +412,7 @@ class TestJobList(unittest.TestCase): "SPLITS_TO": "1" } self.mock_job.date = datetime.strptime("20020206", "%Y%m%d") - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) self.assertEqual(result, False) filter_ = { "DATES_TO": "[20020201:20020205]", @@ -415,20 +423,35 @@ class TestJobList(unittest.TestCase): self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") self.mock_job.chunk = 2 self.mock_job.split = 1 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) self.assertEqual(result, True) + + + def test_valid_parent_1_to_1(self): + child = copy.deepcopy(self.mock_job) + + date_list = ["20020201", "20020202", "20020203", "20020204", "20020205", "20020206", "20020207", "20020208", "20020209", "20020210"] + member_list = ["fc1", "fc2", "fc3"] + chunk_list = [1, 2, 3] + is_a_natural_relation = False + + # Test 1_to_1 filter_ = { "DATES_TO": "[20020201:20020202],20020203,20020204,20020205", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "[2:4]*,5", - "SPLITS_TO": "[1:5],6" + "CHUNKS_TO": "1,2,3,4,5,6", + "SPLITS_TO": "1*,2*,3*,4*,5*,6" } + self.mock_job.split = 1 self.mock_job.date = datetime.strptime("20020204", "%Y%m%d") self.mock_job.chunk = 5 - self.mock_job.split = 6 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) + child.split = 1 + self.mock_job.split = 1 + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) self.assertEqual(result, True) - + child.split = 2 + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) + self.assertEqual(result, False) if __name__ == '__main__': -- GitLab From 37b15ceae9ade3cb1226687345fd8ce5ae06c221 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 20 Jul 2023 08:52:53 +0200 Subject: [PATCH 19/68] redo the split section TODO image --- .../userguide/defining_workflows/index.rst | 67 ++++++++++++++++--- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/docs/source/userguide/defining_workflows/index.rst b/docs/source/userguide/defining_workflows/index.rst index 35b4e3859..5ca0d27df 100644 --- a/docs/source/userguide/defining_workflows/index.rst +++ b/docs/source/userguide/defining_workflows/index.rst @@ -204,6 +204,47 @@ For the new format, consider that the priority is hierarchy and goes like this D * You can define a MEMBERS_FROM inside the DEPENDENCY and DEPENDENCY.DATES_FROM. * You can define a CHUNKS_FROM inside the DEPENDENCY, DEPENDENCY.DATES_FROM, DEPENDENCY.MEMBERS_FROM, DEPENDENCY.DATES_FROM.MEMBERS_FROM +Start conditions +~~~~~~~~~~~~~~~~ + +Sometimes you want to run a job only when a certain condition is met. For example, you may want to run a job only when a certain task is running. +This can be achieved using the START_CONDITIONS feature based on the dependencies rework. + +Start conditions are achieved by adding the keyword `STATUS` and optionally `FROM_STEP` keywords into any dependency that you want. + +The `STATUS` keyword can be used to select the status of the dependency that you want to check. The possible values are: + +* `running`: The dependency must be running. +* `completed`: The dependency must have completed. +* `failed`: The dependency must have failed. +* `queuing`: The dependency must be queuing. +* `submitted`: The dependency must have been submitted. +* `ready`: The dependency must be ready to be submitted. + +The `FROM_STEP` keyword can be used to select the step of the dependency that you want to check. The possible value is an integer. +Additionally, the target dependency, must call to %AS_CHECKPOINT% inside their scripts. This will create a checkpoint that will be used to check the amount of steps processed. + +.. code-block:: yaml + + ini: + FILE: ini.sh + RUNNING: member + + sim: + FILE: sim.sh + DEPENDENCIES: ini sim-1 + RUNNING: chunk + + postprocess: + FILE: postprocess.sh + DEPENDENCIES: + SIM: + STATUS: "RUNNING" + FROM_STEP: 0 + RUNNING: chunk + + + Job frequency ~~~~~~~~~~~~~ @@ -318,14 +359,12 @@ The resulting workflow of setting SYNCHRONIZE parameter to 'date' can be seen in Job split ~~~~~~~~~ -For jobs running at chunk level, it may be useful to split each chunk into different parts. +For jobs running at any level, it may be useful to split each task into different parts. This behaviour can be achieved using the SPLITS attribute to specify the number of parts. -It is possible to define dependencies to specific splits within [], as well as to a list/range of splits, -in the format [1:3,7,10] or [1,2,3] +It is also possible to specify the splits for each task using the SPLITS_FROM and SPLITS_TO attributes. -.. hint:: - This job parameter works with jobs with RUNNING parameter equals to 'chunk'. +There is also an special character '*' that can be used to specify that the split is 1-to-1 dependency. .. code-block:: yaml @@ -347,15 +386,27 @@ in the format [1:3,7,10] or [1,2,3] post: FILE: post.sh RUNNING: chunk - DEPENDENCIES: asim1: asim1:+1 + DEPENDENCIES: + asim: + SPLITS_FROM: + 2,3: + splits_to: 1,2*,3* + SPLITS: 2 + +In this example: + +Post job will be split into 2 parts. +Each part will depend on the 1st part of the asim job. +The 2nd part of the post job will depend on the 2nd part of the asim job. +The 3rd part of the post job will depend on the 3rd part of the asim job. The resulting workflow can be seen in Figure :numref:`split` -.. figure:: fig/split.png +.. figure:: fig/split_todo.png :name: split :width: 100% :align: center - :alt: simple workflow plot + :alt: TODO Example showing the job ASIM divided into 3 parts for each chunk. -- GitLab From 38420aab2ccb5570eedec826f76caf3b76b58499 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 24 Jul 2023 16:04:57 +0200 Subject: [PATCH 20/68] fix offset --- autosubmit/job/job.py | 2 +- autosubmit/job/job_list.py | 25 +++++++++++++------------ autosubmit/platforms/slurmplatform.py | 10 ++++------ 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index f2b2c44f7..876c70a37 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1525,7 +1525,7 @@ class Job(object): template_file.close() else: if self.type == Type.BASH: - template = '%CURRENT_TESTNAME%;%AS_CHECKPOINT%;sleep 320;%AS_CHECKPOINT%;sleep 320' + template = 'sleep 5' elif self.type == Type.PYTHON2: template = 'time.sleep(5)' + "\n" elif self.type == Type.PYTHON3 or self.type == Type.PYTHON: diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index b7918dfcd..374c5619d 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -398,6 +398,7 @@ class JobList(object): :return: boolean """ filter_value = filter_value.strip("?") + filter_value = filter_value.strip("*") if "NONE".casefold() in str(parent_value).casefold(): return True to_filter = JobList._parse_filters_to_check(filter_value,associative_list,level_to_check) @@ -412,7 +413,7 @@ class JobList(object): return True elif "NONE".casefold() == str(to_filter[0]).casefold(): return False - elif len( [ filter_ for filter_ in to_filter if str(parent_value).casefold() in str(filter_).casefold() ] )>0: + elif len( [ filter_ for filter_ in to_filter if str(parent_value).casefold() == str(filter_).casefold() ] )>0: return True else: return False @@ -670,7 +671,7 @@ class JobList(object): value_list = self._chunk_list level_to_check = "CHUNKS_FROM" elif filter_type == "SPLITS_TO": - value_list = self._split_list + value_list = [] level_to_check = "SPLITS_FROM" if "all".casefold() not in unified_filter[filter_type].casefold(): aux = filter_to.pop(filter_type, None) @@ -688,20 +689,20 @@ class JobList(object): if isinstance(parsed_element, list): # check if any element is natural or none for ele in parsed_element: - if ele.lower() in ["natural", "none"]: + if type(ele) is str and ele.lower() in ["natural", "none"]: skip = True else: - if parsed_element.lower() in ["natural", "none"]: + if type(parsed_element) is str and parsed_element.lower() in ["natural", "none"]: skip = True if skip and len(unified_filter[filter_type]) > 0: continue else: for ele in parsed_element: - if ele not in unified_filter[filter_type]: + if str(ele) not in unified_filter[filter_type]: if len(unified_filter[filter_type]) > 0 and unified_filter[filter_type][-1] == ",": - unified_filter[filter_type] += ele + extra_data + unified_filter[filter_type] += str(ele) + extra_data else: - unified_filter[filter_type] += "," + ele + extra_data + "," + unified_filter[filter_type] += "," + str(ele) + extra_data + "," return unified_filter @staticmethod @@ -837,16 +838,16 @@ class JobList(object): # Check for each * char in the filters # Get all the dates that match * in the filter in a list separated by , if "*" in dates_to: - dates_to = [ dat for dat in date_list.split(",") if dat is not None and "*" not in dat or ("*" in dat and date2str(child.date,"%Y%m%d") in dat) ] + dates_to = [ dat for dat in date_list.split(",") if dat is not None and "*" not in dat or ("*" in dat and date2str(child.date,"%Y%m%d") == dat.split("*")[0]) ] dates_to = ",".join(dates_to) if "*" in members_to: - members_to = [ mem for mem in member_list.split(",") if mem is not None and "*" not in mem or ("*" in mem and str(child.member) in mem) ] + members_to = [ mem for mem in member_list.split(",") if mem is not None and "*" not in mem or ("*" in mem and str(child.member) == mem.split("*")[0]) ] members_to = ",".join(members_to) if "*" in chunks_to: - chunks_to = [ chu for chu in chunk_list.split(",") if chu is not None and "*" not in chu or ("*" in chu and str(child.chunk) in chu) ] + chunks_to = [ chu for chu in chunk_list.split(",") if chu is not None and "*" not in chu or ("*" in chu and str(child.chunk) == chu.split("*")[0]) ] chunks_to = ",".join(chunks_to) if "*" in splits_to: - splits_to = [ spl for spl in splits_to.split(",") if child.split is None or spl is None or "*" not in spl or ("*" in spl and str(child.split) in spl) ] + splits_to = [ spl for spl in splits_to.split(",") if child.split is None or spl is None or "*" not in spl or ("*" in spl and str(child.split) == spl.split("*")[0]) ] splits_to = ",".join(splits_to) # Apply all filters to look if this parent is an appropriated candidate for the current_job valid_dates = JobList._apply_filter(parsed_parent_date, dates_to, associative_list["dates"], "dates") @@ -928,7 +929,7 @@ class JobList(object): # If the parent is valid, add it to the graph if JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship, - filters_to_apply,child): + filters_to_apply,job): job.add_parent(parent) self._add_edge(graph, job, parent) # Could be more variables in the future diff --git a/autosubmit/platforms/slurmplatform.py b/autosubmit/platforms/slurmplatform.py index e867ff062..a64d386e8 100644 --- a/autosubmit/platforms/slurmplatform.py +++ b/autosubmit/platforms/slurmplatform.py @@ -18,18 +18,16 @@ # along with Autosubmit. If not, see . import locale import os -from contextlib import suppress -from time import sleep +from datetime import datetime from time import mktime +from time import sleep from time import time -from datetime import datetime from typing import List, Union - from xml.dom.minidom import parseString from autosubmit.job.job_common import Status, parse_output_number -from autosubmit.platforms.paramiko_platform import ParamikoPlatform from autosubmit.platforms.headers.slurm_header import SlurmHeader +from autosubmit.platforms.paramiko_platform import ParamikoPlatform from autosubmit.platforms.wrappers.wrapper_factory import SlurmWrapperFactory from log.log import AutosubmitCritical, AutosubmitError, Log @@ -88,8 +86,8 @@ class SlurmPlatform(ParamikoPlatform): try: jobs_id = self.submit_Script(hold=hold) except AutosubmitError as e: - Log.error(f'TRACE:{e.trace}\n{e.message}') jobnames = [job.name for job in valid_packages_to_submit[0].jobs] + Log.error(f'TRACE:{e.trace}\n{e.message} JOBS:{jobnames}') for jobname in jobnames: jobid = self.get_jobid_by_jobname(jobname) #cancel bad submitted job if jobid is encountered -- GitLab From 74e0a2a6799555e26daeb238dd8eecd9fc1c4968 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 25 Jul 2023 16:01:50 +0200 Subject: [PATCH 21/68] Fixed few things , fixed docs, fixed and modify some tests --- autosubmit/job/job_common.py | 6 +- autosubmit/job/job_list.py | 33 ++++++----- .../defining_workflows/fig/splits_1_to_1.png | Bin 0 -> 15019 bytes .../userguide/defining_workflows/index.rst | 45 +++++++++------ test/unit/test_checkpoints.py | 52 +++++++++++++----- 5 files changed, 85 insertions(+), 51 deletions(-) create mode 100644 docs/source/userguide/defining_workflows/fig/splits_1_to_1.png diff --git a/autosubmit/job/job_common.py b/autosubmit/job/job_common.py index 042c6e330..f6d34ccff 100644 --- a/autosubmit/job/job_common.py +++ b/autosubmit/job/job_common.py @@ -13,11 +13,10 @@ # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. - # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . -import textwrap import datetime +import textwrap class Status: @@ -41,6 +40,7 @@ class Status: # Note: any change on constants must be applied on the dict below!!! VALUE_TO_KEY = {-3: 'SUSPENDED', -2: 'UNKNOWN', -1: 'FAILED', 0: 'WAITING', 1: 'READY', 2: 'SUBMITTED', 3: 'QUEUING', 4: 'RUNNING', 5: 'COMPLETED', 6: 'HELD', 7: 'PREPARED', 8: 'SKIPPED', 9: 'DELAYED'} + LOGICAL_ORDER = ["WAITING", "DELAYED", "PREPARED", "READY", "SUBMITTED", "HELD", "QUEUING", "RUNNING", "SKIPPED", "FAILED", "UNKNOWN", "COMPLETED", "SUSPENDED"] def retval(self, value): return getattr(self, value) @@ -131,7 +131,7 @@ class StatisticsSnippetBash: ################### # AS CHECKPOINT FUNCTION ################### - # Creates a new checkpoint file upton call based on the current numbers of calls to the function + # Creates a new checkpoint file upon call based on the current numbers of calls to the function AS_CHECKPOINT_CALLS=0 function as_checkpoint { diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 374c5619d..b642fcc2b 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -13,14 +13,13 @@ # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. - +# You should have received a copy of the GNU General Public License +# along with Autosubmit. If not, see . import copy import datetime import math import os import pickle -# You should have received a copy of the GNU General Public License -# along with Autosubmit. If not, see . import re import traceback from bscearth.utils.date import date2str, parse_date @@ -413,7 +412,7 @@ class JobList(object): return True elif "NONE".casefold() == str(to_filter[0]).casefold(): return False - elif len( [ filter_ for filter_ in to_filter if str(parent_value).casefold() == str(filter_).casefold() ] )>0: + elif len( [ filter_ for filter_ in to_filter if str(parent_value).casefold() == str(filter_).strip("*").strip("?").casefold() ] )>0: return True else: return False @@ -2102,13 +2101,21 @@ class JobList(object): if job.status != Status.WAITING: continue if status in ["RUNNING", "FAILED"]: + # check checkpoint if any if job.platform.connected: # This will be true only when used under setstatus/run job.get_checkpoint_files() + non_completed_parents_current = 0 + completed_parents = len([parent for parent in job.parents if parent.status == Status.COMPLETED]) for parent in job.edge_info[status].values(): if status in ["RUNNING", "FAILED"] and parent[1] and int(parent[1]) >= job.current_checkpoint_step: continue else: - jobs_to_check.append(parent[0]) + status_str = Status.VALUE_TO_KEY[parent[0].status] + if Status.LOGICAL_ORDER.index(status_str) >= Status.LOGICAL_ORDER.index(status): + non_completed_parents_current += 1 + if ( non_completed_parents_current + completed_parents ) == len(job.parents): + jobs_to_check.append(job) + return jobs_to_check @@ -2185,16 +2192,12 @@ class JobList(object): save = True # Check checkpoint jobs, the status can be Any for job in self.check_special_status(): - # Check if all jobs fullfill the conditions to a job be ready - tmp = [parent for parent in job.parents if - parent.status == Status.COMPLETED or parent in self.jobs_edges["ALL"]] - if len(tmp) == len(job.parents): - job.status = Status.READY - job.id = None - job.packed = False - job.wrapper_type = None - save = True - Log.debug(f"Special condition fullfilled for job {job.name}") + job.status = Status.READY + job.id = None + job.packed = False + job.wrapper_type = None + save = True + Log.debug(f"Special condition fullfilled for job {job.name}") # if waiting jobs has all parents completed change its State to READY for job in self.get_completed(): if job.synchronize is not None and len(str(job.synchronize)) > 0: diff --git a/docs/source/userguide/defining_workflows/fig/splits_1_to_1.png b/docs/source/userguide/defining_workflows/fig/splits_1_to_1.png new file mode 100644 index 0000000000000000000000000000000000000000..0e85db4e25fbbf3ba36316155235d02377299630 GIT binary patch literal 15019 zcmbWe2{e{#zdx>0nL-;8GBi*~AtF;GLYaj!hmPz0ZE%^Lx)eYn}Cf*4le}?8p6F_jP@T&+xqil@w(5ky(FbVK(HeL z|L73y#BV-iY%bzIyKE(8)rg3QJ`X8=#al|di<)+-Rwj0i1~$e7s;2gKcE&b0ygm{W z5U>!)N=c|W^-X`drA;+9aCVNEl8CVSngH#2!d@f$w4FP@k*Hmi%8<*ymLZy-S9nt0 zRYWSpgIw)(ZN8mEttCl*Hvu;b2@z4idyWTpubx-lv4iS~&xkWmYivZ!F~4K1Z)ex# zvdE$vm*Q8ApEPSz-#dSvfFMYhWPB&V`Q{x2ETdSREdfD*5CK6ABLRUSIROFB-v4}i zH)_|vK8^qT-{1bnAOHQ<`IQehUG2AT-z>kB@K1`5zPR8|O+Dwk±NaOYyRX}-nh zf;5e+zTV#8Zi{=eJ6B85f-@kWV{-La)!Jd4}!lK-Ms{1jg z#-rioz1V@cmMujYe8pcXZL(6!YY$osz3f%TX*X&9Q1IVXn#5OAy!^aPLdU&p~6Z@ z(@kH`lsotzi{GBTH5+;&&Bb!nJ=XjOd`icHXik} z$Gd$SA9tGjTGigJ@@N;O?dio&<_=m~y?M7f#_dYd(#({V_e)UXP7?0CN>9H`N$CG> zBJE3Or`_uO_g&WsE?%s4UtgY>oZMW9-!eBhm%50xT)3>GBg(_0EF&}a{xK(RS%fKM zWzg0@M@MJmj5c@Vvz#mkFBYGh9eeIp=`#vi-)%9O?alRfQ$(Ol>2JhvkXC~6JDmYG@5V9MmfH?-hB< zksT+TwJ&cberCO`q!gN*d=o3y^W2FT_lOa)eH9ls@ajs;^6a3Mg~gY)wmaWyi0$o{ zXgIHyEp=Z{R*c_V>e1^pU+&Jh-ZGAuV1f}Q3PZ_-Qoaw2kE>KFz{rSQs zw~BAM{hIkQ)syvLvoTW7IVgkiDz5!#^h{CtciS_4MM)Y)NsrN9g?D$;!%3EWO-KdVs7^Q4w|O z^fNlLI=_7kA1PS;tzXjoQ|0=*HvLy2+kP9mt0>w2k-VZJtyr(y;>46h96>p`lgnwcV@TG9hD&;SdUm@TD4xaqM}_%!K}5;> z`%C|!n*Lo7|DW>o??3*JYMSJ8n3Cut>1lm)f$h9Qf{%DUxPPDRa@6zGR9Q+NFR$tX zi_hKZTHd-|Loa=}?^ZqUn84V^>9W@!MRD33Qk9#L$UUfj>ondKp^{%+-0|yQ` zI5>@Gy~KGcM=@^TvL&DN66wzfAeOSkRF5wDLv zWgB?sNK8yjO+DeV2PYu6vbq|mo1>ws`Z@0wRmpDr$_0UcQl~^sc4AAf$jZu! zf{5sRAv%flsjaU+rJ4Qwv(>oK5e^PiB~nf_kUA1R zK0a(&Z|kq&5b*<>=~8uP&YV$l9H;rvQWff9!7=oJ>K_j6KN{p;Cw{Ma&Z$f$gy{vd zzSP@80dsn{ZrwU~@F4nzpP!#uPbN)A+|#Eeo?DwMD^BNPxkvXEYtKw{B%9s7t){O2 zxv!7wn#ae-LqkIk2|J*^)YaAo-g30H6~6P$XC>$e(P0jb z`|pN+d?|Ir2T`+xgoI*aV@F3vIW;o5I5>Fu_+o?|xqT(y6c*|gSsN%Tx238kyNF$$ zsP-k@y?b{_Ny)&#K!YF-wcAa z{4R-&J+x|dF1F%EJ^B3L8Tw%J{R*isUJQNxs)W1_4?ny{j*zact+hM%J2;F4HEau6 zFDEZA>^P&1NVs{EU4qiy-X6J#C5{l{UyS7Jr%q*MIS9nWhKA17-K0)SNs*F~>FMkY z-HH{w^KGOtV*RO8U*0YBy~^LqF{U>pCA~sIDCG782#xQ(cb<}wvA@6n$JCT78VOp+ zenMq+^^24~bNAkj9N|byOLMiTMECuxRW*z))V7)}jJ9A6F#;C%0|O_<$61J~Yilny zF9NP)zJ6WezAmt;Cl((c7dJCBbno>JL-MtIEhdqVA76QR>CQ8CPtMkT;Z+04z?>T(r$Pp2d{_gJX zuCB)-l_sX9$NQ6y5GA~P>B~ZNP%JLvu96z2s7Z2;il2g zZ?#XKKCRqbn<{txm7bp7@PG!WV%#6eo1LAF5G1BzBVsto#YIL&*1&hiNJKeRjg~fY z`+0$nBiN@V zCc^esj{M{etRD^wCX`rNU+3iH1X>~8%RV(Zc{qz`7unsz5sgx(g0`EMp6BnS$*we% zBiGpT8X9Mh9s7>JM~5dTCr?dHZE0y?Vqy{y5I`z?|Ni~hv133y{T1%Q&NG93eI?h| zn<7u$xN!qfG+|eQHXqeihK#bcv3cLrG&MC95fL%_rL^#7Yus3Ce0^OVLR5gCpNEc2 zRkZ`qxTvUTW5W%rprWFp5AH^Wqd4oIMn-V!mPN(HtWS1v;_mJY*Dcv!Jj~DERZ~M4 zY5aoyz4g9Ua+P*$Lvr%3v3Sq>2a_K+eJ>~|K&(&_wsm$MJ!gF3!Y*G)R=vsPg7Sg~ zJ?-t)`lp}9#F&|x<>%!A_#CIc|43}5!hIdNF_*tjbSSR-O@Q1>l3i)n>GA!``4ob8 z4~A7r;bM4674AYcKgtvWq|`GdzqYnkq(Fkk@l%jdp|DiR_W+kB%j%yEnkD&P4Z3yC z`;UK-hX1{wzHPW0Q!bs1FGb1ZSqk%2wAp*UIoJkBLhhxXkI$l;EDPn_s#_}gqhpjr zNx!6u&w3Dwi;Jfdd|Y?gm8k3BP=IX?3~YQUADMJ6Q8t3@jV9Mbxu=fK7aT0AYjt(? zIbmUCtx&=bEx|T8(WMwP)YI({pCMG5!A%S3!%Gkkkb`;h8q;8qu@#EtQt>UeX zjbUyLRbgJzi9FHGUS3{6VG7Z_-`8g=>r>e7Mlo;}X^m?$Vh!o&Ya2ee@Ljz4a+Z2-d1z{WiaMm&)YY{#Q11Gq|XFLm3@>;`W% z@(Brw%8zYRDgUi+#KXrYf9aAS_|@4#H@+KnLH+&Oi^K@Cs7H^e!T!XzHu#WlY;4jR z8no{15%KYZ-g_xVTjOU(M-MPE2F2*^w>`}(;=Xp~`0<9;*6l+90kEq4EwCm*DzSHA0@fHe;K=*Kx^Z>_iBY=zGIAmoH!b$o~6!8x3`Vd9TW~Yp*jhEX>TBjCh5G zh1*UB2!)aYxS`YLnqWOo2yw>FO$6 zvKzb)r2t0^tTH`3tg5D#XFVjj7}#uTW+rp-;xTsi`uh6My}df64*T4_qP$rTFfa%k zzkjs;Yj&{Q)sbt^RlE$Pra^e?_wU~TgNWcZc2tm^D76Uat+{s;_@w8;=oJPAhQ-Fy z2`Z^0v0GVsl?6FD_k)5)7FNd(fU{9L&myeL!@`uo9G{%LZarK6Eb7cH73}Th%Vf@D zTd~59A93CBLdXe%!om^WYPB^r;}a7v6i0%wQ3eJ>&xCBXb#(5GiKA~1_2%Bh?N(G& zq+QM6;Nn`GpO=hpICk9RIgs`uH0P8Lib-?T^04%pxeHG-I1}e)@Z()2nBx%=7%96XZiUr#s?)$OO1T+ z#4WN#G=8_)4w#iKLBK@g_5KCgXY`+CI;k7)xZ6`}|1YxM#em>s7#-&B!YKpo%n3lqX$X-JzkOxVSg~ zJrTE+({Te_vxF>%`1$!cucm+O?A%!XY~?I6VEQ`>c<|b_Ywj63$qKRbwa2(QITz;V zdC!~?u^lD5>g4R_-7QfJD$y#<6heFO;4Ne0uC_Lw$=UK`l~golHIDqA$c^8>W$rAU zJb4nO_}#mAjz2yPO-yKAyVhs>w7ObSWSrR5_Qb7`6XGfLdlVVz>gozI)Mh2j@z`a3*}`0`h>(ZZZEYeMW$)0TLkXpPz$pv@ z=5)Eg22hO@qfSdK*7R#`IzRXItp!T>V3K=rFU9arC)`OY1fv5-T?2|Yg;0P&1W!G^ zLnvWgj1=wX@pv~3ij9`m=$ZEVwTp|(!`@7!Iq05hA-fpFX!feeghH0POGcXmsW}lqH|uj;QhRw?i@Y|UFcJR z1%tq`nrF{kmVZQtgoKQ1ISlPjLuK~zlDvNXGs30bdm2af(?!Q5Ccs2t*0@Og+D=RC5#)iBHwdpX;D<}}Ku~k)3!FA^6 z=H5OzM$6OF-!Jmymy*wiXF?HSVG?`#YXYgzP}r_KJ#(x5=(*T|tY2C=MvX;xhW8YZ ziEqq#Kl{}V=1{RQ-vqVr{Zl1^1h;afr8JVgITFkmpAw9?vngla5^Qp~ZX<#oj)#lb#FhTA7+mE5(_{h({qY!V7 z7q9$$WmcQ}7a$$V9i&Ob0HF8n+qVJNv;LdVt660I0#wY+Z37(@V2SqRwa)3-n5R!A zFI-4fn*@{k^5x6-1!vpxW#_EvKcEW|3B-i+SqJ<7z+_V#QbWXAvB4K^_kM9 zCnsb0%??B~p5fywb(ns7R6hLCqa>7x+)XM4xxiRfxP;sHv%uxQvGmt>+(gr;)sT zvV_J^swY9uH`FK*;s_BLSzc}~A3y&-a`G7pjZT$n$&v5hm%6xRSvOrP663x57S#jVG#*sZ zN`?nZf|V#LDhkSAH-^@qQ(j=S-Fc5iP_U@DxVNY07gPi!F&O*n*Y{b7_IR_Bd3dp) zCAkP()zA=kSu}-si&F4I;Oq6XUnHlYpoNo#MaIWN9k#%USXp`g{C2S|Qow3JTSKGp zR>vz~W1GuDw6wH5R{1y#WDBYvDMddx$M>;2o8hzxD3CqNrH-?5DgpQQs|F}v-CpQ? zuw>D7$K@Z}h)T12q*Nh3Xb;nCpkC{6D2?;&T|_1>6{hJNM8S! zp8xyl1Q5>kYu5nhx9U!8A)>+F{vdqgwmJ{B0fSR=W{MOg1U0y8QdKFw6!eimIk~%u&S<9OTBqgo5YD{0( zhEih#t7~dT-rn1V({h6bfi~tPd1k^4QOr$+YpCOSXdjntTIt*jkdAzvB)Ol++H`a z%A_Qp`r$XdKOkn^y?YmVZSxeE9f8D47Zevf62|2mw-kKyI>Z&gm;slm3`p7G;o)e% zXa>vtcc2sER81d1e$LHZfD+Wz-7QTi1;FF%Ts+tHm!1?6nc1*-l;UiIC(%OZ;GiDb z5s2#b>oMl=q7G-h+IOmv6ueR5g_4U!WHx0r1$W~kD=wd?Yj&J90V&%D9Ukn}I!0<0 z@pWMp)a{%9Lk2Tn%9jmBytQQ;$YvZ%CB!>-otn0;t}C*##o$#JFZz0WUqHXIwarDu z2M1pPOMy2hE8B<;21<~bl41(GC-+T7g_wXqK}pF|h2-m+nu|+(zAGy$XdVd(3HSjo zA0Hp&Z;04hO3&_6gc?dlJD;&^7@{vvkW* zLO{cwtwdzKdGls-bMp_~rP*63nCJym9?z<%y;uU`@|{-aUmo>kF*Y%&tFOoPCMPC> znZ+xz0*{^WdH7Jl#KffanuzD7b{^%cSFfNSTu^xcNGT*J*bGLX?U4E}E8P?r;ti_6gK+nK{R&enqj~B_$>B2$0ZECXa{02$X zz`4Y}aNGaVbDYm+me;KXf*EpfuO=X%7v7!)V4Rzqn15C2d;X-f3o9L&-(3Ci<34sb zEQx+PmEkOP_1b%SW*zn?>rw3@;h7}8cP%I=3f}D(%P^{I+4>TNq|?8j(kTTf>k)Mt z8r3j*S8dxkXV~^o8+Ov}%pWI>Vr0Lv9VEV8X8}39l5}Ni&m49^o)#LjCKkizW=EV_%|9ciMl+hHQZ<7sKRvo z_;K^zoR@idYQXw+b!;Lc+IEZwoEfJ!U=>wsIjCuEk>TnM zLHuglW&yeC@94-ySnT0DL>fuICchlX<|LvE@%Zs$b;t-{0oK-Yb4;hdJ*U~5p(6D{ z2{H=owB*at-@bm$2J2m3b~w>uc3J24_$e*jr;xR*ZEVijj9h|*^`Mqs++!o#)9uf% zowT#FL)Qq8h|uEd23>$7DUdfSZgPr$F*5Jel`@wl4naYufihRXZHo5#RiFijI8zWHfzQ2g}iIk5K&^9A_eja8Kkl|8dek!*gqMFWF_V|k6=L)@Gh zC}%i$@Px*%p#>)qWZJ}+(+{E;I`(FKQA+K^1}Uk3*Vm$~aQh4>%B&AsmgZ~_!# z1QOW4f9OwlRq*cFs+WFM5Qq-pYS<*U2qF7PwF?*CVNeM2Q^OdODs<`wCny-zUkBZ(^h>ml{65|zo9%?FS8}ms7yQYO9X%Qq9Uc6r)wf&-TSJPjZo>e6 zHj5z=2<8?$BkBEiIo~pDOSlMpJ@S%x=xht3LFU`peTJE?Vl1P6?hKFz*8tf?MMUa? z8MawAssp?Ueg4PC`1$i;1@fD9n|~>Rl%S@pd)qS^tNDFU-j_hyy$xABc?jqB; zPq|P{zv-tKhVWAQtC`wRa3@iGzkO@Z65f(H4hOQTYA8A01AP!pg!wHL@bB$}hM5fa zx%S>XIG?|a4#Y^FjA;?j=94EaAP7K;9knT@COO8-tHNX{e)-a+m6hQTH1x!T1iz7? z!$7sn9xDb&Je+GuEH85==?JkQOtDZJQ z>)cN$@lply3tks{bkOGv^tSO8AP2NDUq3(if8;~L0E*j^78VHhja+0}R)nJkvY9gP z8TPJeu*BZXDKfZq40^q^qGD@D$3m6&-V5uaP^$2&SD-Gi0sP8Iwr3MgH#~ausNs*} zaHNkP=h7|HJY5j{H43)$S}a=LJjfNkqGQ^ubm{um!3Yo)+z~5w6N;dn@)^)0DYY9)>TqE?Ye9Z zN;Xj&x^hjg%;_iiW=?7~rn_9f$iHkeh4lkVkL&1BAL&GUaq?s(quh{pCb{@)VlP;8 z>DfHu;+~i`;$8Qvw3`SA%1%tY0mn6)2YW2}f{uZq(rr~`Pm>a0%Ysw|lxzS9<6Qb5 z1LZL+0e9~bw6**A`C--wo7(i8%ExskK~D%*hoFOyz~r95@2^u+pK~cOgIQjFi07;q z4K3}j^t9aET=P*T+^-*(ca_u6Z@Y_crKY58L!{zjEtA-f78TmuJ(v{$?#9d-Mj4vE zzt=yA9^O0M8Iq7-C?Ro=%;Yyj@9^Z$scN^Oo}FjWf1?Qo0Ur{MYmPQUBk6c`)-y%1 zRouGracT@o!EQas^%h4wFnv8!z}Ehdh_mg*`h?;9Fm zimyZoyRBr8?~#6?1ag=p+JTZ~N&DmRqem5MlW7LMc~Y%Zzt;!1W(xG-NFY(em(!6m z7d=BD>tGDY<5p|lix&o{$O#MT%F3{^;2t&3Wk!Inp%f+fK5sL{`010X==YZO8IVmP zS|*fhp4oNv^gPHM2Q6=p8%~r7LD?w0aR$8|Mzeu9aSjHQ;>KR2s=XeM7CsRz?pcwo zc$``eBv$Qa{qRpHb1oj5$Bw;ZtxPy~H)|k{^i@Ja4yH!6m4VwU9!SCv!yF&Z z1sr$cvNg(h7M(dfcdBMlr?RMGB3|QfIksF7T2J+iVNUOL7}$L z)wc!Q4jKq8d3lR?i^-jPm@ryx`eR7d50b9%M9W(HOL;;GD3h{Q2LQ>Bcm}!WA7ym&|KRGi4_vSSkA%qir{)Lj8n_H5j=9QVN z=yKrWKkCDdmM7cS-iPW$W5dvGE>QaVH4aH`<4yBjdk2SbnM8!Y-MQF5%@5fXYtptG zPf(D7;`cf6>`~e+77rL9$VeX3GHNZj^WNL$<{*V+`3cV4q@)@E4;V^vvwASpE$gZ| zh&;QGIMaBn4Ea|D8Rw$JU}>A1>Ttcac4JcayUji_vYKZ`bV*==VU-hMgRT>Mb$mU- z7J2qSjX;k;;@EtKgAO@FQO;^}q>;+e@fUEQ+38>6%)$=lp^DETgys|$#sm54TN-*o z>;oW8>Qu%zVGzYoL63Z%+zn_RJ*LVn`jC)~QKp!IZR)PArhpql(8c z@1H}w!SHe6^{KEUGD$8YVsE345LFKhc+B({V;S4ztzF^0(&~^u^PVlX<>h63LrV2s zckr;p3KBrNZ7XYo>s3n1ajg%Qiz1KYJ``s1>{JiktI`+4R80MG_y1{_RlD-1lgRuK z$z6`;Bq`NOVQZop0C|{|!7=^U=;PqOM<00#r7PT4F@Mi1F0O~zz|1>5clRq=YHDbR zYFb(-?w7=FLL9i$7=Ele`O?Db1|}xwGmrv$l^*4WwIqA@?i~__K?#}&xdiSoEHuEA z*Qu!o>F73qM(xh&$jSz`m>`6KE|@~!}3@O|Oo3~Sa-2}*N;bqsdl#4x&T$aaAy zNn?9Z-l}4i?bInn4Gl*ql-ql!LgR9>3aT7HyNnb~5`!&3Q{cmgnHGJ;NJ!xI+}zM*FUZYBZ^zmymuJ4fr|Ac` z0B}LkMume~VpH5N>bh)t`!>i>jN3;`n31U2^pup@xZ(LjU}JzI_)~eyF39hoybb_0 zrLz_CnJS+PEL7FiMXkRQNKoQI1&k}&6xr`{rfJAH3>+vbs76c_b&Q3`FiB0KWN*e_PZk^%MzU@{RxmeU&6ncHVk zIA)RUImS)6$66y^m>`)trEDfHZTU4g?7_jxq9Q?2(b#ltsFOG2-P!t;kTGIw3yq!2aQN{_g3>2CS}RL$Tf7IhvnjfVMH6``DJjj(&)eA9Ju=#%wELFY zi^Rkur%!*xcy?#!8F(MS1JGFI)W`D)kED=XIPm{?Hxoa893@IX!OzK2S-AxR2i^@x zs^0Vn?L}z3QG4lQqN961ek7SULzVSAcex}Se0}<;x)P>kJ36R7w0-!H zsn^(^#j$v8rx(jeG06qyfAnm;+(OTISFt36$FLA>pKaDMipumr`TEx?Z$s(ei37J! z>6XncEaF^(i7vsYqUF}HDz zQ#VnkJE~@2nug+k=Pzq!X{xRJ_v`8gj(d5A5C36GRYHtVOH*<_`Zr*EmIG=($d0^N z2he>Xs=mFUf|`PY0xW)5kPZ&Z0C!g|UxuJUqU<$sv^+fOHIl8b`W-9ZQWpyDimc|pJh?!$UHZdZ+>W)pnkUi)jJ_rJ%s`47j6XipS~b=}9W<8*&K$cvStc>*h*6A}va*t`4y z%<;hkc{H9vTV_P4^V(wSTrwHpXmuov6tR`BySO7c3%efQq1i*BPFKwVw0Y&3pz_}v zua1uO@5fLY2KQogbPpk+(aoC?9Y)5+80CM&NLA5;M`s!y9`Nkr=izzt`t@W+;Q(9! zPCN+%PJv>tM-@SQNq2g3^uehLK(aN6+)|Eh5Ilh7Ay{52NJ>ryz#(NfH8+1I;+!)# ze_)k;L^)6-)R^WHyw~u{ejJw)N!-NroMN{9=f*~2GO|Z1Gy|GDozo=G-oMPI8a{!U zM*$G;iakYL|95vpL6;*RqCl^%o*pv;!*w+J7)ePjH{lcYYtY`FJit~98z7&Qz>)r9F37su_i*=+keuo=d6B~A|z}?aK@6YCJd`hUSvotluy!42C!rJY~Hz>L#CFcO` zumZ~|Re5<*?cMk&N``z~du$HUp4s*?!@jlDdJZ{CP^L^XIpz)b;>lZ@SI}hg{mnax@20 zuP37;E-#g{QwZmwj^I^ZA*DM(%XhC#f~HQ5<}bTLC8Iik6BX-E%cH_FpaX`7P^YD5 z@?Nza>U63jJfj#q29U`*g`|8w_I-I%fg)w+YmCa8knOqg&Tn6P43pAsHG|7l9o{>( zA7}hup3nLFah)R<#4VS>!oxq;j|Lw~@e+Yomr^|*BxG}3fKGR$VeE9-Q8&7i4aBm{ zX{$iT5yO1d`auk56|P*dc!&WzNGs7Gu?Yx{P;*QC>ygTT nSi}Fn2QdHPc_WqiEdr)?_hSp|m(B6GDS_-o1*uF)eXsuyd+D~P literal 0 HcmV?d00001 diff --git a/docs/source/userguide/defining_workflows/index.rst b/docs/source/userguide/defining_workflows/index.rst index 5ca0d27df..0c17a3ae2 100644 --- a/docs/source/userguide/defining_workflows/index.rst +++ b/docs/source/userguide/defining_workflows/index.rst @@ -212,14 +212,23 @@ This can be achieved using the START_CONDITIONS feature based on the dependencie Start conditions are achieved by adding the keyword `STATUS` and optionally `FROM_STEP` keywords into any dependency that you want. -The `STATUS` keyword can be used to select the status of the dependency that you want to check. The possible values are: - -* `running`: The dependency must be running. -* `completed`: The dependency must have completed. -* `failed`: The dependency must have failed. -* `queuing`: The dependency must be queuing. -* `submitted`: The dependency must have been submitted. -* `ready`: The dependency must be ready to be submitted. +The `STATUS` keyword can be used to select the status of the dependency that you want to check. The possible values ( case-insensitive ) are: + +* "WAITING": The task is waiting for its dependencies to be completed. +* "DELAYED": The task is delayed by a delay condition. +* "PREPARED": The task is prepared to be submitted. +* "READY": The task is ready to be submitted. +* "SUBMITTED": The task is submitted. +* "HELD": The task is held. +* "QUEUING": The task is queuing. +* "RUNNING": The task is running. +* "SKIPPED": The task is skipped. +* "FAILED": The task is failed. +* "UNKNOWN": The task is unknown. +* "COMPLETED": The task is completed. # Default +* "SUSPENDED": The task is suspended. + +The status are ordered, so if you select "RUNNING" status, the task will be run if the dependency is running or any status after it. The `FROM_STEP` keyword can be used to select the step of the dependency that you want to check. The possible value is an integer. Additionally, the target dependency, must call to %AS_CHECKPOINT% inside their scripts. This will create a checkpoint that will be used to check the amount of steps processed. @@ -370,28 +379,28 @@ There is also an special character '*' that can be used to specify that the spli ini: FILE: ini.sh - RUNNING: member + RUNNING: once sim: FILE: sim.sh DEPENDENCIES: ini sim-1 - RUNNING: chunk + RUNNING: once asim: FILE: asim.sh DEPENDENCIES: sim - RUNNING: chunk + RUNNING: once SPLITS: 3 post: FILE: post.sh - RUNNING: chunk + RUNNING: once DEPENDENCIES: asim: SPLITS_FROM: - 2,3: - splits_to: 1,2*,3* - SPLITS: 2 + 2,3: # [2:3] is also valid + splits_to: 1,2*,3* # 1,[2:3]* is also valid, you can also specify the step with [2:3:step] + SPLITS: 3 In this example: @@ -400,10 +409,10 @@ Each part will depend on the 1st part of the asim job. The 2nd part of the post job will depend on the 2nd part of the asim job. The 3rd part of the post job will depend on the 3rd part of the asim job. -The resulting workflow can be seen in Figure :numref:`split` +The resulting workflow can be seen in Figure :numref:`split_1_to_1` -.. figure:: fig/split_todo.png - :name: split +.. figure:: fig/splits_1_to_1.png + :name: split_1_to_1 :width: 100% :align: center :alt: TODO diff --git a/test/unit/test_checkpoints.py b/test/unit/test_checkpoints.py index 8772e3ae2..35dca3350 100644 --- a/test/unit/test_checkpoints.py +++ b/test/unit/test_checkpoints.py @@ -3,7 +3,7 @@ from unittest import TestCase import inspect import shutil import tempfile -from mock import Mock +from mock import Mock, MagicMock from random import randrange from autosubmit.job.job import Job @@ -24,35 +24,57 @@ class TestJobList(TestCase): self.temp_directory = tempfile.mkdtemp() self.job_list = JobList(self.experiment_id, FakeBasicConfig, YAMLParserFactory(), JobListPersistenceDb(self.temp_directory, 'db'), self.as_conf) - + dummy_serial_platform = MagicMock() + dummy_serial_platform.name = 'serial' + dummy_platform = MagicMock() + dummy_platform.serial_platform = dummy_serial_platform + dummy_platform.name = 'dummy_platform' # creating jobs for self list self.completed_job = self._createDummyJobWithStatus(Status.COMPLETED) + self.completed_job.platform = dummy_platform self.completed_job2 = self._createDummyJobWithStatus(Status.COMPLETED) + self.completed_job2.platform = dummy_platform self.completed_job3 = self._createDummyJobWithStatus(Status.COMPLETED) + self.completed_job3.platform = dummy_platform self.completed_job4 = self._createDummyJobWithStatus(Status.COMPLETED) - + self.completed_job4.platform = dummy_platform self.submitted_job = self._createDummyJobWithStatus(Status.SUBMITTED) + self.submitted_job.platform = dummy_platform self.submitted_job2 = self._createDummyJobWithStatus(Status.SUBMITTED) + self.submitted_job2.platform = dummy_platform self.submitted_job3 = self._createDummyJobWithStatus(Status.SUBMITTED) + self.submitted_job3.platform = dummy_platform self.running_job = self._createDummyJobWithStatus(Status.RUNNING) + self.running_job.platform = dummy_platform self.running_job2 = self._createDummyJobWithStatus(Status.RUNNING) + self.running_job2.platform = dummy_platform self.queuing_job = self._createDummyJobWithStatus(Status.QUEUING) + self.queuing_job.platform = dummy_platform self.failed_job = self._createDummyJobWithStatus(Status.FAILED) + self.failed_job.platform = dummy_platform self.failed_job2 = self._createDummyJobWithStatus(Status.FAILED) + self.failed_job2.platform = dummy_platform self.failed_job3 = self._createDummyJobWithStatus(Status.FAILED) + self.failed_job3.platform = dummy_platform self.failed_job4 = self._createDummyJobWithStatus(Status.FAILED) - + self.failed_job4.platform = dummy_platform self.ready_job = self._createDummyJobWithStatus(Status.READY) + self.ready_job.platform = dummy_platform self.ready_job2 = self._createDummyJobWithStatus(Status.READY) + self.ready_job2.platform = dummy_platform self.ready_job3 = self._createDummyJobWithStatus(Status.READY) + self.ready_job3.platform = dummy_platform self.waiting_job = self._createDummyJobWithStatus(Status.WAITING) + self.waiting_job.platform = dummy_platform self.waiting_job2 = self._createDummyJobWithStatus(Status.WAITING) + self.waiting_job2.platform = dummy_platform self.unknown_job = self._createDummyJobWithStatus(Status.UNKNOWN) + self.unknown_job.platform = dummy_platform self.job_list._job_list = [self.completed_job, self.completed_job2, self.completed_job3, self.completed_job4, @@ -72,7 +94,7 @@ class TestJobList(TestCase): def test_add_edge_job(self): special_variables = dict() - special_variables["STATUS"] = Status.COMPLETED + special_variables["STATUS"] = Status.VALUE_TO_KEY[Status.COMPLETED] special_variables["FROM_STEP"] = 0 for p in self.waiting_job.parents: self.waiting_job.add_edge_info(p, special_variables) @@ -83,28 +105,28 @@ class TestJobList(TestCase): def test_add_edge_info_joblist(self): special_conditions = dict() - special_conditions["STATUS"] = Status.COMPLETED + special_conditions["STATUS"] = Status.VALUE_TO_KEY[Status.COMPLETED] special_conditions["FROM_STEP"] = 0 self.job_list._add_edge_info(self.waiting_job, special_conditions["STATUS"]) - self.assertEqual(len(self.job_list.jobs_edges.get(Status.COMPLETED,[])),1) + self.assertEqual(len(self.job_list.jobs_edges.get(Status.VALUE_TO_KEY[Status.COMPLETED],[])),1) self.job_list._add_edge_info(self.waiting_job2, special_conditions["STATUS"]) - self.assertEqual(len(self.job_list.jobs_edges.get(Status.COMPLETED,[])),2) + self.assertEqual(len(self.job_list.jobs_edges.get(Status.VALUE_TO_KEY[Status.COMPLETED],[])),2) def test_check_special_status(self): self.waiting_job.edge_info = dict() self.job_list.jobs_edges = dict() # Adds edge info for waiting_job in the list - self.job_list._add_edge_info(self.waiting_job, Status.COMPLETED) - self.job_list._add_edge_info(self.waiting_job, Status.READY) - self.job_list._add_edge_info(self.waiting_job, Status.RUNNING) - self.job_list._add_edge_info(self.waiting_job, Status.SUBMITTED) - self.job_list._add_edge_info(self.waiting_job, Status.QUEUING) - self.job_list._add_edge_info(self.waiting_job, Status.FAILED) + self.job_list._add_edge_info(self.waiting_job, Status.VALUE_TO_KEY[Status.COMPLETED]) + self.job_list._add_edge_info(self.waiting_job, Status.VALUE_TO_KEY[Status.READY]) + self.job_list._add_edge_info(self.waiting_job, Status.VALUE_TO_KEY[Status.RUNNING]) + self.job_list._add_edge_info(self.waiting_job, Status.VALUE_TO_KEY[Status.SUBMITTED]) + self.job_list._add_edge_info(self.waiting_job, Status.VALUE_TO_KEY[Status.QUEUING]) + self.job_list._add_edge_info(self.waiting_job, Status.VALUE_TO_KEY[Status.FAILED]) # Adds edge info for waiting_job special_variables = dict() for p in self.waiting_job.parents: - special_variables["STATUS"] = p.status + special_variables["STATUS"] = Status.VALUE_TO_KEY[p.status] special_variables["FROM_STEP"] = 0 self.waiting_job.add_edge_info(p,special_variables) # call to special status -- GitLab From 1255b823a2b5eb1eb50b0695a9869ef85cb77630 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 27 Jul 2023 15:36:13 +0200 Subject: [PATCH 22/68] todo --- autosubmit/job/job_list.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index b642fcc2b..a6a71a29a 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -397,7 +397,13 @@ class JobList(object): :return: boolean """ filter_value = filter_value.strip("?") - filter_value = filter_value.strip("*") + if "*" in filter_value: + filter_value,split_info = filter_value.split("*") + split_info = int(split_info.split("\\")[-1]) + else: + split_info = None + # strip substring from char "*" + if "NONE".casefold() in str(parent_value).casefold(): return True to_filter = JobList._parse_filters_to_check(filter_value,associative_list,level_to_check) -- GitLab From 37a9d1b7c7f0561044601769c3c02e8bcd31ca08 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 31 Jul 2023 09:38:00 +0200 Subject: [PATCH 23/68] 1-to-N and n-to-1 implemented --- autosubmit/job/job.py | 4 +- autosubmit/job/job_list.py | 127 +++++++++++++++++++++++---------- test/unit/test_dependencies.py | 5 +- 3 files changed, 97 insertions(+), 39 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 876c70a37..7bf33b5d7 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1638,8 +1638,10 @@ class Job(object): except: pass for key, value in parameters.items(): + # parameters[key] can have '\\' characters that are interpreted as escape characters + # by re.sub. To avoid this, we use re.escape template_content = re.sub( - '%(? child_splits: + lesser = str(child_splits) + greater = str(parent_splits) + lesser_value = "child" + else: + lesser = str(parent_splits) + greater = str(child_splits) + lesser_value = "parent" + to_look_at_lesser = [associative_list[i:i + 1] for i in range(0, int(lesser), 1)] + for lesser_group in range(len(to_look_at_lesser)): + if lesser_value == "parent": + if str(parent_value) in to_look_at_lesser[lesser_group]: + break + else: + if str(child.split) in to_look_at_lesser[lesser_group]: + break + else: + to_look_at_lesser = associative_list + lesser_group = -1 + if "?" in filter_value: + # replace all ? for "" + filter_value = filter_value.replace("?", "") + if "*" in filter_value: + aux_filter = filter_value + filter_value = "" + for filter_ in aux_filter.split(","): + if "*" in filter_: + filter_,split_info = filter_.split("*") + if "\\" in split_info: + split_info = int(split_info.split("\\")[-1]) + else: + split_info = 1 + # split_info: if a value is 1, it means that the filter is 1-to-1, if it is 2, it means that the filter is 1-to-2, etc. + if (split_info == 1 or level_to_check.casefold() != "splits".casefold()) and str(parent_value).casefold() == str(filter_).casefold() : + if child.split == parent_value: + return True + elif split_info > 1 and level_to_check.casefold() == "splits".casefold(): + # 1-to-X filter + to_look_at_greater = [associative_list[i:i + split_info] for i in + range(0, int(greater), split_info)] + if lesser_value == "parent": + if str(child.split) in to_look_at_greater[lesser_group]: + return True + else: + if str(parent_value) in to_look_at_greater[lesser_group]: + return True + else: + filter_value += filter_ + "," + filter_value = filter_value[:-1] to_filter = JobList._parse_filters_to_check(filter_value,associative_list,level_to_check) if to_filter is None: return False @@ -418,7 +474,7 @@ class JobList(object): return True elif "NONE".casefold() == str(to_filter[0]).casefold(): return False - elif len( [ filter_ for filter_ in to_filter if str(parent_value).casefold() == str(filter_).strip("*").strip("?").casefold() ] )>0: + elif len( [ filter_ for filter_ in to_filter if str(parent_value).casefold() == str(filter_).casefold() ] )>0: return True else: return False @@ -666,6 +722,8 @@ class JobList(object): :param filter_type: "DATES_TO", "MEMBERS_TO", "CHUNKS_TO", "SPLITS_TO" :return: unified_filter """ + if len(unified_filter[filter_type]) > 0 and unified_filter[filter_type][-1] != ",": + unified_filter[filter_type] += "," if filter_type == "DATES_TO": value_list = self._date_list level_to_check = "DATES_FROM" @@ -703,11 +761,12 @@ class JobList(object): continue else: for ele in parsed_element: - if str(ele) not in unified_filter[filter_type]: - if len(unified_filter[filter_type]) > 0 and unified_filter[filter_type][-1] == ",": - unified_filter[filter_type] += str(ele) + extra_data - else: - unified_filter[filter_type] += "," + str(ele) + extra_data + "," + if extra_data: + check_whole_string = str(ele)+extra_data+"," + else: + check_whole_string = str(ele)+"," + if str(check_whole_string) not in unified_filter[filter_type]: + unified_filter[filter_type] += check_whole_string return unified_filter @staticmethod @@ -813,8 +872,17 @@ class JobList(object): associative_list["members"] = member_list associative_list["chunks"] = chunk_list - if parent.splits is not None: - associative_list["splits"] = [str(split) for split in range(1, int(parent.splits) + 1)] + if not child.splits: + child_splits = 0 + else: + child_splits = int(child.splits) + if not parent.splits: + parent_splits = 0 + else: + parent_splits = int(parent.splits) + splits = max(child_splits, parent_splits) + if splits > 0: + associative_list["splits"] = [str(split) for split in range(1, int(splits) + 1)] else: associative_list["splits"] = None dates_to = str(filter_.get("DATES_TO", "natural")).lower() @@ -840,25 +908,10 @@ class JobList(object): if "natural" in splits_to: associative_list["splits"] = [parent.split] if parent.split is not None else parent.splits parsed_parent_date = date2str(parent.date) if parent.date is not None else None - # Check for each * char in the filters - # Get all the dates that match * in the filter in a list separated by , - if "*" in dates_to: - dates_to = [ dat for dat in date_list.split(",") if dat is not None and "*" not in dat or ("*" in dat and date2str(child.date,"%Y%m%d") == dat.split("*")[0]) ] - dates_to = ",".join(dates_to) - if "*" in members_to: - members_to = [ mem for mem in member_list.split(",") if mem is not None and "*" not in mem or ("*" in mem and str(child.member) == mem.split("*")[0]) ] - members_to = ",".join(members_to) - if "*" in chunks_to: - chunks_to = [ chu for chu in chunk_list.split(",") if chu is not None and "*" not in chu or ("*" in chu and str(child.chunk) == chu.split("*")[0]) ] - chunks_to = ",".join(chunks_to) - if "*" in splits_to: - splits_to = [ spl for spl in splits_to.split(",") if child.split is None or spl is None or "*" not in spl or ("*" in spl and str(child.split) == spl.split("*")[0]) ] - splits_to = ",".join(splits_to) - # Apply all filters to look if this parent is an appropriated candidate for the current_job valid_dates = JobList._apply_filter(parsed_parent_date, dates_to, associative_list["dates"], "dates") valid_members = JobList._apply_filter(parent.member, members_to, associative_list["members"], "members") valid_chunks = JobList._apply_filter(parent.chunk, chunks_to, associative_list["chunks"], "chunks") - valid_splits = JobList._apply_filter(parent.split, splits_to, associative_list["splits"], "splits") + valid_splits = JobList._apply_filter(parent.split, splits_to, associative_list["splits"], "splits", child, parent) if valid_dates and valid_members and valid_chunks and valid_splits: return True return False diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index dbf93565e..1432abdbe 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -356,7 +356,7 @@ class TestJobList(unittest.TestCase): def test_valid_parent(self): - child = copy.deepcopy(self.mock_job) + # Call the function to get the result date_list = ["20020201", "20020202", "20020203", "20020204", "20020205", "20020206", "20020207", "20020208", "20020209", "20020210"] member_list = ["fc1", "fc2", "fc3"] @@ -375,6 +375,7 @@ class TestJobList(unittest.TestCase): self.mock_job.member = "fc2" self.mock_job.chunk = 1 self.mock_job.split = 1 + child = copy.deepcopy(self.mock_job) result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) # it returns a tuple, the first element is the result, the second is the optional flag self.assertEqual(result, True) @@ -429,6 +430,7 @@ class TestJobList(unittest.TestCase): def test_valid_parent_1_to_1(self): child = copy.deepcopy(self.mock_job) + child.splits = 6 date_list = ["20020201", "20020202", "20020203", "20020204", "20020205", "20020206", "20020207", "20020208", "20020209", "20020210"] member_list = ["fc1", "fc2", "fc3"] @@ -442,6 +444,7 @@ class TestJobList(unittest.TestCase): "CHUNKS_TO": "1,2,3,4,5,6", "SPLITS_TO": "1*,2*,3*,4*,5*,6" } + self.mock_job.splits = 6 self.mock_job.split = 1 self.mock_job.date = datetime.strptime("20020204", "%Y%m%d") self.mock_job.chunk = 5 -- GitLab From 70f05463c0b13f1c6bae5a00c439c7209f5d5000 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 31 Jul 2023 10:20:41 +0200 Subject: [PATCH 24/68] Added tests and missing doc --- .../defining_workflows/fig/splits_1_to_n.png | Bin 0 -> 8323 bytes .../defining_workflows/fig/splits_n_to_1.png | Bin 0 -> 9094 bytes .../userguide/defining_workflows/index.rst | 52 ++++++++- test/unit/test_dependencies.py | 105 ++++++++++++++++++ 4 files changed, 153 insertions(+), 4 deletions(-) create mode 100644 docs/source/userguide/defining_workflows/fig/splits_1_to_n.png create mode 100644 docs/source/userguide/defining_workflows/fig/splits_n_to_1.png diff --git a/docs/source/userguide/defining_workflows/fig/splits_1_to_n.png b/docs/source/userguide/defining_workflows/fig/splits_1_to_n.png new file mode 100644 index 0000000000000000000000000000000000000000..5e700a40087500a6c39f93e00e9354cf45bb5e60 GIT binary patch literal 8323 zcmaKSby!s2+BPB*ic-=oDJ@7S2r__liHLNEFd)()-JqhNGz^^z(m9NDcMd7iLpKaC zyo=xUUFUq~`p*0D$C^EB*4}&VC+_=xo=t>?n*80{G`F#^u9~xVpGlI++I!5@2C5VJXT= zzw(&cnsqn0KQV>3TMo@SxyO|sZu9CRJ|{^Ve$r$g9A%g3nf2mP)eLtWt=WC$K-Nc* zB($WZWaKwU1pFUp6W`LjAy?-=nsdAU*XDW0j61)MgXQM9bCoCJ zGGzETI9OQEn>o|(fK45NyZn1j*X5=_+D6a<^G(+_uAQmpUOuw zW-%_kmXm8MPe}=}6UpTE+G(0y%S%jL!>cV;46yS#`IxYv^o+D#SwU5u;Q4}&9GhhkK^2tE z(t->XYd~C4&FDIxX$KYbjf|aW*XEvStfo522zOJnlzZ{gKuhLQT!`yPZ3%|B()$+c z^Cau};WvT>(PDZjiT8DPlr{IHRa_K(w;M{vE2{78r&%>PKu@=8{9e3F*)cjH+N6_A zd74TL!3nHGp~96hlp{tOioPb7N*%RPxY1N+e;f`M>}fysh55K&{8&SJ8QzsYl;$d1 zUzlBM(H9j+_%Vf2GGrk;iCx9j|0pYxA=%Zq8)Y4tW^I4EE(+;vE)l)h-_qOpRS#ub zE}r6T|15>pF>_y$a0=Y==vbF~g4{_d``$6@&aBWe7*5<=@Po%s_ppt^9hFmVzPC&6JFjLfW6G&|8}O&7PdN_furx z6?C%u>f0TTDTg_<+pJSBNqDW5qWnHIx*rhY%ZVt`8~Ay>lWvsn5Le5?V1BzfskRl> z3^I{y&E^#rOWXJDJa#BE*U{=fJr{)Fr()usMMqy9OS#~t67N4?`7}Ml17#HVjBHET zn@!~PU$WNHHEoKY_8S})VhVoq@qBlL|8=8B%3S{{OR@&cLP=YM?WJ#ruTZQ22kJSrj# zCx$-o_&C*=hsNxQ?RxIdmX6Q!2rJK?jokfA2qZiyuF6r+Qn+&eZ*%dK@yg1~-C7Dt zuhfp^NI&u<)P^xRc>@<$=fDS2kG~4u=ZFLABO)nVn~k(9SHv`gl4~nO$Lod#Px5|w zS`7~d8ySpd)tZ``dNn#07ELXWPmO63b5M4+*S2)lVquomHq2TiQKyZb(5?qty$&l| za-}=_Fy0wSD(IvJ;+$8V6z@fJJ%M!}!sR$IU zMeO6|({R2=x<8DI<;R_E11~>+UxG6Q?@mvRR=VOdOE^S1mc8cFQPnCca)aQzlA1hs zX~--xh zg~pZrkCLLI;*DxCufOWG18uat&$i#n$hZ=|r}^uGM_<3^@Dnhr z?{MU_%if+gCSNHh|K9HRcq_I1)|W_V9X-|VV>V4?^{cl^e;2*?NGJzn4Qs=pF;8nS zqd9w@M?Qrn)bl4x?YBf6ysjHw|0v~qVH!4&7;QDc@4L6Tv`rtx!!xpI?sj@1oi&8d z{La3Fi#vmqWwT*^-au4~W4wN!bHU-Fk4X}-HAS#faDd6Rv*7U}Fcuxnlb!eaM%=Sn zZTNIk*mcgDT_%hS;sle3tXWdWkARa*YII}_*uY3|+OIwXzmW%0=gfwd-XIii(E^hkRx=jAKq9 zigII}y$$g}Vu9-COL?tr?X65%OCz<;tBog<=hJ=low;+?BRMTby!_7Zf`d)!oA+Wl zU7iLhaX`Yo!i>#{A&*^Lgk4<35jjO}5fI*tvZSZ3>Oxo+WMu?Lip2yt*bWC9)nNf$ zW&(-u6b3A0Fo>@}hE%-#g8h?2j}7HTx?DB=-4i1^2B?s`bvyxD@mFa-G3}=A zN&V5r1SJJUZgzHfxfluL2qae}QP6F3@)ND-17fBv9PB0?W1q3fNh)gUhy8zL z@gE5Z-615rI9-hH>+NmCpzT_*j}$2`_Ivo_`Hj^S6o^|wX8M2p{CN`_n}W~4#=$|- zx}PriX7*IEVf_OZme|->;m40#uh#F~yNAJGxL^OElKfB;8%x&AoNGwB#z8_t!b8E$ z%4%Ukx%fNarmd|lH8piOzCuZnPD$CDAJf(Eh&Ih?>gzYw)|Qr*m>!tV3giHlFq z&0)@W5bn;-w7VhiU0va>8{=^A6GB2lVwx{ci5MkQ1TDLpg77Aa4aNV6AHw`Fo_cyT zxE+x|bv4^74-4 zs>W8B?V`rG_4V~BDJhMOX9TktO?mw0rz_rSIXF0&n4~Y=2WePU<))yZU}3S;=yOiY z^5r&mfXP^)c4cLyALcAimcD?Bh9)#P7zYQ(!`quh3 zcvz5=v$L(ua`KmHPaLnzujkEM)7AVE5*KrDZ?GV5@y{dxga>afE5opTmP-&AGr`0g zH*Ul@ zSw%&wcN)*tnkikueJJ^jRMgZMBs>JL@krN(vgCn(@$S*RLHNRh`3sAR>||wSoty-5 zJC2`><~3>>8%w<%&IZfyaBvhbYeC1y$8u2=aF4wQ%*>|GeNp&K5KqtApFg)gQVXfU zBze!2r>CaST^z8{v9VdWw~nqZ^@5B~49^Eizqtm_<>lql($e9w01*)p-@_sK_V#vB zhZ*97pJrIHfA&23`}?s2n7Fu*1TWLHamG0X>wu2czQhjZe(~kFeoU; zW=Bz5TbPH3hlR!b{CF#zN#XN#f>2W8;o>SOzkd*HV`mo@7B*k&Y`rrF4>w_KK84Ti z9~_ALpj}i{K1M`DxbW7NmcDm%{FRlpxVz9|XlO{HMCAzb7#JA*Tvt~&T)89@b`S2h zl@t}#1C~4zs~Ru~S9sUj-d={>RF3CU+~G2)bqZ-SlarHUP7?Ul+|rV$vI4R(+nxO4 z;^ICzIXI;xUGm7l$OwSX97wBdA33+3xt~9OUTz|VTJPg3Sy|?S|W({wS0!KtVjb zFd^%Hh4VU%AaiFYC$IqAH(mqgRYTAZlR{)?v-0t&fj#1m=jY^Dq7T;zab+TLRRf?l z_V%eyoY|v0qCS6)%hwM)q5@7Q|2V%9@l(G#H#ZlcS^-g}gRTB}6`e0Shy=jZ3Ar!&CjgXz*`6-K~@fFENy)ER_rzp+Y` z-(G408Odi*^X1Ez^pX;O<&ClTj+=F^8^mi2&1^L%q>PLMR^HNv6V_Y$3+p1!nXp{1o2%z&l~ zYXP`YUR^Eaef*;rmW~D}35rl)>7&C#V)RKMdX%^MQI?{jBES+sL?ei1%*7di8W7Eb zLby1rvikbd0Eukv>=yt2{r2sfg|CmNL|RmoGH{TEMRxDU8ze-)vJ~#_C#R>Uw4x53 zGQhHf>xWU4{QDja{D_M(IGmmq>oB7)S=8Fn@)ofD^@?9_e%MVEfB*j7)R70JHDB+v ze2tAIhV>#YYZ_;|cCM~Rz??NTHHP(WrnLTzPQbrXm{S%`P6gDq4wCli)2F}*($dl( zv?p=rZt^v`wk|uFo0&bW%FfN@;^u})c)pYrzYY;i&Am-zIX$hil9H3Nb7`f&&C$`( zyP1gX?b&*$$oc+q&pwiQ^v~qRM2S1JcVK{@kFR|^aL;?kZEH#xH)vsTk>%on$fHN^ zotCHGEFtV`dxZ_i|4&UXO<_*{YqGq<}j1XAM-Pn_c_J3qg=j?P3&i%i;i>-01z zf)&NYM8IB^l{dl|eSeP!q7NO)t@>VSXrzy<+l~JyC@F!7z5m;rCriwWMyO1xq49AB?_&!MN!_mb!)WxZctNrc=#zGG}Dcj+X}>4L}5e8cL_cD8m|je*6ACDl#(vuj~`g zeO9krAv&qcj(p9mva+(LRRUCm?6K6u!~*Wy2DMvpNl8h7Pb58d{d<-%6UCnbo59UK zef;=q`8O?459!iDfKlc;+1c1~{`@iF;NYm;N(9$Ympo$?7eD_51@r^N5xB~vhvUT| zMKq%n!0YXXypfri1|XlZzZh{qp^9T;M}WVV%k?WN zDu9%0Up)|)YR}EdS)MF4pL9wu2caI+Wa84#PV-5`Zk(s~neXt(h_j1}tO$Jmt5`)< z6-Y?LkWge~BF1}X@3s>eU?n9di_eoq^;sn+B>{>eyL3L; zHdI#b=|{?D4G9Vg4ocB4oJJ5sFzAgEG#dTcx_He6Jz?yZ8XsS6)^__a`qL*>TU!*6 zbB~3Efq_D{2yt5`CMYj&(i;L&t*q2OHD-_PvJk9(H=h<8d$67d%k-n8rxy|uQcr!F zQ(Qc^mvT3%EoNE>Cgw|OxzHRkx{gotsog|?j*c!((kIek{bS1lbn+L-Vrh@#7mA7n z1qB4pZbH=5)JE@O3Pu+B`1m|MJ=fy;0evIRR+2v>fzIt(+#3{v{oWC;XOmHqUKyb-{fS7!r$07atwsDXg9d8{`ucaA>V_{_o4 z<~B)x|9%DVK42+Y+T?4=!2yg=Z{jrFMqNk?xI5_eEN_JnkL9&LxUA$alJ>(^CpM{{J-i zYHw{#5qIxzY6|GH;$UMVrvy^Ap#~1WJUdtgo|U>h90|fBy2mI98p1$ydqeP#m}dX{ z`4b+FZ@Ym6Q41t$Z(m<*Ow94l{5z1c>+27_syNhAYTS3`Fqh}lgt%cXj|2pE4-X%U zipKXEpz;LZI@ixcp39-0MknE+p{NMNhTYm=#@yW8gS6fVfxCa=8Q3^DtU(}yiic7Y zX2Gv=`4Scw7`VT`pU?4u2Smxq_AG!NfI)_p`ntNV2VcJeqXD_mGce${`b~mir}Hbz zXdlYX))pY-_zztfxXb3`BTyl)kK{2*ewgg;w(W{xYj%HYYwJ{NAd>8})cN(HRxXcG zLjtkj`@gO47uz+JIlZa?Yk)EbfG4OmySlppi36b)Gua_=ET1q}jYpgta9ax?Zb?Z= zU?9ZFX&-DkD>D<6vta^?;4zqkdzR7X=hc;!UDNSK2m`-5m-U#`RQe+@;|jd#;!+WQ zybbiy#>#;v6bh|Na_Ucc5={Iwb?z(&S#GVhg+wCh8P9}Fn{jMQeTe65#2j)HpuidkJ=)_(n(hMJl{U4w>(2Iy5iU0ryk&+#U5 zeSJM6BV+!&jFps?X@za?2Q@JTCnP2s*10Ik$yu%s=e(jB zNO^MLV{K<+lmGLlD%{5;f>%W*l+d$;;dufsS90)1&w?We9eR5Duc|R;Iq@+unK?OQ zsT`C+1dH++lS1uGBPq^iII^?Z{NPv*#a`* z>({TEnwk;}>KDm+=H})=_JBV%wf>Mhl);vM2P=J`ng!8j0QYjWwJls@iV(Au0Q{{e zCM2n0iYfvdLO)c|*4AcVU~pdP&1lRZW>>tU<>TWcE89Apqr7~eT%vOnF|b-xSokso zfJ|9VPEKxaD9wY++}!c^>OVn+T6D}T6WW{b$UMEO3ZNDcq~H%|$dr_n&*!%QY`D3( z_4o9ECvbLmUv(7Q5ncu40p^dlLfH&WQ7zY-U{NVCg4cN+*_xUzs7qbmCnXIOP&7&S zOfOyq1YzSd+i#fyN49wpT^8Wn@v*U6I5?mr(rg+T9c2v$Ds906*uc;b4!Vud)93c~ z^!(&6Z?c5|4xOEyrQ~_z2%34`{K)r|Wk>HQ`2S4QDfp&gj2DJhZ&#!=plK~LnUL4f3S}8>gG7TfXdaoh0wEf53)Zrrk|t4G&_!0 zGTz-CNn?uC7IzIOPL?4(`+zyWXSB5E0W|cxO@^0J?0*i<=xG6hcK7rUFoQe;vD&mKSov-BEum#}5yVX^1-m2+_We6s3Og5< zEl8J65WZwlZ7)&&w(BcM5hT7=3E#ye5}BBkEjs@m1%$^6OAyp#|6 z5D>z=?>Xpd@vyNicQE>8yYTKgV>F&uZD;EJ?1RhjW>>3@dfZ@LX79Y>ATb2!EP%=V%dlR%M zz~ca3!q>VADFC%Yox(x8&sg0+PtV4H)YZk&gfSbd3MlD7K7ddS4Gk3}%bl5-(Jp%Z zzSe*uYOMUp=4`zv4^QvN2pMwJ&$PS0-_6x^tN*Bf&Q~H;E$^=^#ydfZSr(K;U^55$ z*BcupEzHl?K%~dH;gdBV;(}_FVL~sM0LjWg7=Z%K+`&Cu+N;WR!WXOhJcyk`e*tC{ zu#lCPwMcSod>nmx+Hk~1OG}HgqJ<|i5VHppm_iaj=fcB(eS3S(Z{%ik@bww76~7Oh z!Q0q>y-WW0JL7*%{g>Cv=_T+*j5mPXtJ4mtLDkE=Qi+wUd(* zs8-fw-fy~`{4MYT5d?U9tp)SqnZfC7tm`WC_3L`G(2f7zHvj($xBt{~|EC!X^>XXlHmvJ5^BISvX63cj4Iq#6ne8YkFqhM<8hrr5hv@P_Up zA*TU>K>jYPEP+EJS1BD=bq5Pq4`XL@6m=^%S66dq)1WbI6ckz%IZ1JiPqX{;9tI?v z%~(fp`9mVfvLdn;2*d=P^pOI4m?T32#sfhe=gChi)kKlxlguxRexd1XfBou(PZ#bg zB^mjFB)+6Eph^Mmb(b`UuMIa3v}5l$WXklJOJ1K(zDTyrO!ly_WTuM=2eznEu{Pho zeFd9*$p72?=jeZZisakEil7d=mV^YFm%O;u>y)rC8E?J5krBV+l?PsG;@9WyKr?lK@$%kDUYCg;n%gIf+Dt6R(KGojn5;(EjMIJgJDzRLE!;=h zKD^}?q(7K^C+G6S{?b9X<=juL^Ba?=26=jMzLfH&Qxlqq!%io|+q7OHjF_w5Gm0mj zNqTxhoDCi6Z-v9SU5*jsotxwO2eX6H=?y4ysIjjNdr*!&TenODi>FsGe2_;kD>7_& zx1W;N^NX!o`e~l-6`yWdnFWwQMd)I-OpY8HOgO_ObKYetWj0u#M@nnUU6UmzH_UvJ z!BZ8SC@jv^=Of0bp1ZRgM*6ASvXuB;xbMItn<-f^scu`xMH?$-8 zX{N*wEA&mYef4oIg|@wP+OUGcLma(@Ze|uAdwA}&?V+ZgfB@g`(cr!L>a(>FXa6*=&vKwF5xKwOC8)=%&o-kt!v#zVEK9Tjm`&43x=DVPg9a` zFmAu~e&$Svq%7Cfb|*~n$2^x3mtQi&Q{7sinmS}<95uY$Gajj{DSR*Fcex7JVOjJT zZ+SQR^9Z-nTrKCVJt6sH+V1?(?p1LTR=56ybdOc8?pq{VdVgBWwP1W+G<&2+BTAj9 zZ`UjP_c!#2y_xMA9^{QuHM;8)H1whJGO&1hNawHNw!q&_l-Dh%x_BJr*wiPkr}%od zK9iOcmK?e)H=XRa2cpsz8?((`u8eWCkb;Ez`iczmiwqHck7<_~3~{U79p9rOC7gHx z-+P;TwjRy58n|Nj4qdLU3A3b+p-rM_LfQ z@t`Pu`$VPKRP%XH=wCeY*{AJ`Yr0>pOIUZ|bt{QzB3$b`41bO3yVS4o>l7le6Ggt! zLf)IwS607EP%*n*jIsE}cx%HMxp4a^T}ias)zRfnke6ft%cW_q$I0mS8*TXPPDzAl zg_E^)SB*iz(DycTH(Jws!xB)OC@6qep80JH?6L*XT9@eT<9&y2S-lS+Nfgl_R}#vT z%T{GCYx!ctD(Wks8URfYF)8$$-Kq1$ikIHSTXaCMNL|Dc-#!^_apAYPkgv2`^qEra zeFn)n8tpx!tdB{Snvml1y~?SmmmV*hexRa)tex`SI|Ai5;K3bBtt{uV%fBa>XnIF5 zv}r_gr+IV;Ke&i6?iJV;DS&+WoNHSh1<&a8nG zZ!I|Vz8%$?oN5^NyYqNqL2UO2Yo2)rRp;y4CMSGln(@e!Y3=4*m>ihmWI4tExv?OEmL4t}(J}D3dx5rdprmq=)JZr&`s|l9)MceysFXe)D zZg0S-qnVi8hvD_*%5j!3TN?(N2>-uQ&0WS%L*DvowKm&9=^`iJ?pU*6TQrB}McIEa zi$HmI;*(Y{1w)u9qY zFk;aPr>;?WCY=s?j8Z!6Yb0`WJyF$MuV3@>jy;V)x)Z4UnE03Y3E?S}7oCz9E%sBi z#vyTfV?TZ8MoC0_%=hNsx%LD9(1WvSK3+y_cjIB&q z&Hs4+5uCF3^a>62#U!M^oA>CeYB_dN+KeI-bt)QcEH(c1u;ZWeL85Rdqs4*CMybC* z9jn+H!^>gmFj@#}om<#i;l;k*?dOh|rT)BH2w<*8Cm5e#4<&5UsaUm-<+Zi6u)QBc<|VoSxv2usOM)ymZ>NIr~fTyiP+9)&+8V zr6YsCNhL-^0e^kdjJvk>z@W{F&pznW+dW~&s(nW+1V%xMja|SRY5LzM^mT5o=}ap0 zM(vhu+%I@zZgj0)3GkwaS?SzYtSjc{?EES3r4RR^9Buw5u4LpV>d~wj{oV$Rz{Wjb z%~3-#Dy{uftPGLlz?1@AOi@LBs1N^N@2R*wJjCqDN(atDeBuxSd6Os>YY|fjOUZ~z z;4K>1e2DQ3x<70(7wQ#6+Zc&Yqxb|n#788Pa)-3E*erx*!GYUR7GB!f7$<||->O)J zws{@@QWO#RPl%lA|us@}G>1T3%*X4jM!nUxIntoPMb}Dc6BAGg2 z-dp~T>M*YC^pcwiJ1AVx&2}fK``0OQiJbVR1?h!>Li zLQAnxmf0uCee>t_`kF5)`OX`aAKZN~HExB{Ufpd#DzF6J;9}g??j|z%U7eje_~_Z{ zuFn$D%1xN#shZecI^5}U5r65kiIiQmot@o4sUxF`#2foi2>U9V$tP8LQGpnT@saT# zb7!ksP1Ri7p54Wm^OGiu>`>CK7pkPXZ5{FrENXkmCL|K9c{_ZHuQcfA|i5Se)tD@_t~onqX=C}MjUmvp~tdODLN}F%RN$P%<4|q z&pv6evHpSV!)q|>x8`2UI61@K`2V=U} z^P=a_T^U}#q31*#=L#`Ukv_x_&sH05)`_`8etG~Jb{RuiJXCk5XJ#mf7806}aCsd^ zkY|UwBL;&lp|H^Ty5i$`pYut&eizw!L@QJY=(mQVZ!~2o>jLFMYsFQF#S2~VZa=BntJ?A244%&i5nups`=Z)d}n5FnO9-9BV_;LB%7OzeC{2q99@h-Y#8lA z8;iQ%wQDGz^*x=Y-;J}@=aj0iuU}qXPMOuEBH#cE_WGmuHYPX5qLMwbXRVr6Ha(v0|_son=kB8b?oh8g^f2MkM>DOm|e*QME zvrgk5?8+HJO+HtM%gal-IL2(gtRFwdfBpIehhNR{3z;^4=XW~EVTQk_M7Fr zRsD(-QFQn4u&jZzu(04oQcJ~rTU#^p@VNMlMOdLzjn;;mheb%;*x0xSujw5fC2WJ` z_5aXG28na*KbLN_)x#>1p% zVZk;GjIu_jB>lI)GgZoMInwI1+SB#r5sV_X$5=X&+}zaE1?VAx^eBQrLSVc>yU$a> z8xCrbgM~Kl;S`=Ne7rFphp7_HL{3Amqou&6CVsEgRoFg@dI2;(KEI%V{OQx|w@apx zk&&joQTY?wZywNnv>X9T@9&pmcp{%g;zov#j}IdA=g%MQ3Io_B?r%Mt?yscp-o4x2 z-rjHhfc;wk-Mg5!wzk5;&Fr^fV-!l*A)Oy53gV&Ay!?D{P4-)nfXqhZTYdeTxh9te zheZ*e%NJg`#lAP^A;H02-QC#)fv&(S9gfQqs;a71R?Of-4*mMMHg7&UIyAQ;&o_qW zfw?l#Ajs=$Ygufv$H&L60{=j;w{X70GK9TdU0o06Tf!=64VzuR$mdJT$mFs)!X?-~ zd3i0gcpL&xU}9pXx3W=-`U!cRZt>Xs*_)|Gr@-uM|19WnFjrPqwzp@?&0V9sKrxyj z?BeQrdUA4danW1<3|sykcS&jK{@xy=QVIzr0f*!bb& z$Dslk<@uGBG>;y|rKPpHY>te~5h^6G8`r=$M>A!pBv`D!#n8*x!+)3qhlhlOq&uET z(ZOOF6yA6E`EHLh7#bSd5{_a1ySX65#RUdOM@Ju~30L{wAwk}4ZCTxl{o+4TJ4RtG&gsvALIf)+TGZ2-Zd$t<<6rl$7F z=kmyWFu|=vJhZP*Rz+oWYilc++tP$0z#Z@^^+IJ1E-n=Xg<^Yil>%TeTh z!(tVF7D>OjkI!wyQ*MD#{JEvJMsQL#t*)LP3rkCn-9Px*GAyyFJw4J0=b=*VO6#LKGh1Ho(=C{^v^1Z? zLRST4-+Z-cuaB?qBVyv7D5}z5rJfr@Pr0}%MUj`Lx0lC|<~_#;hJ&@WwIdnAsZi+Y zZe@q{jsQ+b=heyP^q)WX2}Te|#}4WimLuh|F~q{bF*WHnG(2orZwnt4y^9bC&ZU#W zCLlOJJ5y3nsHv%$Xsy%l@cZ}W5tEv_Iu<^8u+Hb8*RNkkPzfgVn=vvnS}&Bhl$MlG zQc{NEQ#_`kip*TeCSe(yHu6u8k9WI6`r@}dBcdiEkUzOTv+?%63Bi3r{l-!3cMoZS zO6GeVourr;V=Jq0v4+Q(TYAr!nZrjnEA<5WNl8QcRz~IqYCyVzh=PJV1-19K5(124okK_NlvYxU_#`DKf3&ro%bs3`*I3fa zL>bgsYch5GeJ_|zQfIFE39;7C!@~nAbH9-fqZ7STyw2LjCOs_;OntnMSGuFTckKzn zRzXpI{OGybgFkmvE9_WNbY>`>9~v6^`0-==zxCy-6(O0KM)vmh1_qQnL`A`Yf#SBd zwrXnQz#PE?y)uKQ9|Z*)oguhIZ=K($h1LuIg-Yknw|dslbNYa8QeW?GZOys@DItCI z=xr`+Wp!0>rIQA@R9|0^jSbhyRur03e{yzKS6j=@#dYW?MM4O{I;-~k=LM=2l&42o zcf0omXI#IK(+UP%_~2uWvg&H5$)a}vgSN%~@2cjHu6#g8C@CqKoScku-aSAdoI&iK za$8XHJ7YYA;zD!WJRjkh<+x3%rP-BGSSqUclm}5 z_J)3bei|AiJAfFFAA=%2I!fZURJOA#|74=9JaW9+tJ$al@Jz|*>rsHCs)Mq!a>p&= z8*r_>y!^yAxYN|sREb_mYild0hXVBx&}~5}O%$rEq7pfWPMGg&8yW)ib^yQu2m%QX zA+?Zf;e?%?oo1a7z$^_`w}}EJw7`J0A8xx^;L@QXMGV}@R{>#=z&^>Tsimc*6huS;vdcIEJn_lN{U2ULQSu|MHZ#BZ8>{N* z6vc+X>+L8$|H_)!=C&C6Py>JX@ZoFn{gv)-kVi#L*VkU&-rjtC4Y9GYFHL)1km2Ek ztZZx)#D+YCU}X9ru*Hdhb-#ihfMR$a^Sm)j5YcV_6!UR05#}A_ESk18)vt*3CGKqmzOiBgGQ7P zA0MMBo_sLZ#O-(c2|*FtZL4@@)_87GE^@r{EBmVxhWd2ayRuV~~m%Y1nq};OP`Yw-c)02YLPJ}d z&pVJMSpsxc={_;x_{7A&aJXDE7i+vcL#4TDN^md+ONVC?zl+7&x9B;HGGtjcPsk$^ zzJIT>`^(E3KfCvji0gf8(a<_JA>k98kYVZ@wc^R}a0r5uIH0+@s_N+Y7zTrVJQrUD ziNMM#pFHS-yuGpr6G&1l0Fi2F;PcZ6Mip>_rliaQRuEm&_w#2!?<$Mk%e}olk%f5E zSIMB=1M;`&g_Z2pr}=&oZp z0Ff3J7S>dCCSs}qm0M^336eZjDFa}>wuT0+*w{P3Rdn_A{2XV*vsK@>`xK`=2#=1w z`%v`#*N4+RIQPq!4`IJh(wn(oynql8L`6sQ@$it+P`^wb8cgJ3vqV)+d-H7MJx+*3 zE)0{9IxQo^4|E+s`7j^?U%A>Z=|n{rdwatqvcnb7;`$>ZA~a^V(mu7OD;G{!)t>E6 zZ*Fe3G&j=;-~=s__8Un_b%W5YuCDe^@V0qhfU$WUssN*! zvnLR!fI^`T9slZ!io#o~y!*aCBcu`0I1K~yagdIGrlw-ttS*n1v$M0&u?QsJkl@Hj z;qaq+Um<{&`2aWud8C$(&Lvo&l>Zj6Lzn$=q;xLK3g8N;?{AMeB&dQ2?1qwFMpFyV zRvDu=8E>wy0|st@T?}$Xrf|Z|%?%wLo%_`*)i6hmN$IAhrk5{YvKmF~?Cc~ZC9SNi z^v5v?3JS8F)^z^<4Vahr*$$80Y;CkZVfO2{VFYJKM`G_k67!}~5fNq0ntuXJru*BI z!NEZ$#iSP&Lu4U-T7agDyH=j-e1T3T9}e%HlOQBl^3 zc{w>bB_#uY{%ASTJR`vYHz+}9&_Tetl7fOzO6Nnot(0aH(8ceOGll)0bO4%}-MIYf z@i)NlDiJgS-#yrSJQ}30kvE56dHDo_?ga$}!^jnx_YVSoec0OFE%i})^ym@rli(<( zv`iF>IAEfrv+X!CYbenE@o_a+OVyTH{7-XS6qEop3=|Y*De#xU{pLSM|HUaPWpQ!T zt1KZDFUh3_l9E!s%IM%AXn6BYE`{tWYW)Cn@bL}X zyy}27IQ6*a>_S2Y<8$kSiPKY4`Gti~NJs#d9NRTD8E|HfUK|~fy)?lPCjx2+NDY%$ z$wJf)M@zpz>;X~WU}cS6aK1R02UHK#B3S4Hw7}flRX}AYm)*OwUc#S$Oo)I_&hD!_ z!Og(GJSMB4jf(REwxDcozMjHkbNlZf79QF6l$2)B_ZI!{()-QiQcgmLTU29YiAeq1IY+z#+GudeWi;C6tviuznufhH(M`^F!63#ik^Q1XPyAZU1^ z&TF#3wkJkD$3Th9%*=qh{`@IM9vRAkF`KERX3?LdR6Z|4nyB>SH8lmrH``bObXOVy zPXtg!>j9|fkRV4Vr{c=W=!gjDt8`ijV;sNRjs{>`k11$G{cc-5Pt1&xfMiTbN&>w5 zJE0W}02x5@*}7f86n1xZAVEZ<*GxGYC1GJ$O)r3r6%`eL;bMr7S=BzkeGC?*myp0u z8I_&Ae7^q|q~;(HRX`Gn_}yNDh0h3yal)UQC$PgO>+jwEyzRK8gw^bk{5#x*4*$_* z$1bibA7r_HL*FT_@AY;esF1|hhPSbU>&{IEx*{=i4A5>Ui~82-2BH2iV7>+h&;LD3 z9CY*WV1ncT=JlUur7tu%HKm!5&}3jg5qkx<%#8632}XrKl9&G@t5Wp1rWpTk;mVKV z?jE49zAG_H5rf3KyIH(z_dMAk6uqsopKk_TJct%~y<6Gb+*}Yn`ihT_is~-_pVYaH33pmx8*;>FMdkX1Bq=e<|Q-z#on1Nb``8JQ|G2_lx4o0~g`HYG!tZq(9!_fI)| zdyQXlJ8V-vtFm&j!`~mV2taciBu&bQo4b2`Lql{_6ahZIu-|QCb@hFnm^<9gRGR`O zrPQm2VkXBd2M3Ddxw#g=iBb{mgTG8auw|s=NkIUKi4j0ca_Urx2~`80rNLUa{j?io zsDi>E^4|@gVB0q2b4nJ>YE4%wh<1EpqTe3~7myDRFfm_|MF0pMSkO~c914F(|Dpol zPWJwnz-Yl-t5ie-cKia&sqKn`2^NeKGWzLY{OPAa@3VM3hgd zsAszbq6ySCpf!NsX3SI?b%=k)>_-ZogG8{hv5BVOQ@%A1T>SZY51WC(W-V4#{x1j@ zv(#6KjHh1bdr!rQ)==N14s$Xy(?V8&k{lTxW(dOt{Nd_+UtdKk$LcP+83gE!Hx;hR{PWx;6qwDKy&@sV`YCW=i3p_RO`2Jad z{&57zJurn(Qd9&e0VF0M282`sd5CP;EuDL_o)5>qOr5RqW#vqL&J$&8{f+yF@XeyhleMD90M~oeFF`U?og=f{wxoo^3tchriPoJO;q$=;7DX2 zf#4yL$hGJOP^X}1`;KjJalZk!CiLm(1NdpQz$JiIKr-AQ2D$WYufQP1`{ID~izHBs zq+I=9N!h-?7%K?|GX)!7o97Hr&^0#x17G=0W^wGyN`(A8(mFabKtq+vZ#{eU3sw@GzbdAVpKe^}o3PBCchU0uP0>p)n7tEh#&`kWQQd2u0- z7cX7_)E7d?$3YVl^>uZ@xJtIeT~}U0u0MqUswzKH(ACxT^XJcj0eMQ&k3bz;8ykxw zZ&sD$Ra7eB`-=p zcFfPm6Dp$p>nbSUmHHT=bl<^Xlbx1U49su?Y$bRL_x_S0XzTM+#mgy zr^Gh1s~g^u;)G~wYC@`BIW-x$1{IVP7mG`w;QSg!xEm{%_R6Z0^iI^I? vCnZ!6PW}JklMLYLhyTX&4*#>OhL1bQm Date: Mon, 31 Jul 2023 10:23:36 +0200 Subject: [PATCH 25/68] n-to-1 change test to precise syntax --- test/unit/test_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 8e8766d4b..ab8b4e357 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -526,7 +526,7 @@ class TestJobList(unittest.TestCase): "DATES_TO": "[20020201:20020202],20020203,20020204,20020205", "MEMBERS_TO": "fc2", "CHUNKS_TO": "1,2,3,4,5,6", - "SPLITS_TO": "1*\\2,2*\\2" + "SPLITS_TO": "1*\\2,2*\\2,3*\\2,4*\\2" } child.split = 1 self.mock_job.split = 1 -- GitLab From ccf4be61a5b6bd8f5eda39642729639c166741c1 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 3 Aug 2023 08:39:00 +0200 Subject: [PATCH 26/68] Fixed part of reviews, Fixed as_checkpoint() in python --- autosubmit/job/job.py | 18 ++++++------------ autosubmit/job/job_common.py | 3 ++- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 7bf33b5d7..112e69380 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -186,7 +186,6 @@ class Job(object): self.x11 = False self._local_logs = ('', '') self._remote_logs = ('', '') - self._checkpoint = None self.script_name = self.name + ".cmd" self.status = status self.prev_status = status @@ -254,18 +253,13 @@ class Job(object): @property @autosubmit_parameter(name='checkpoint') def checkpoint(self): - """Generates a checkpoint step for this job based on job.type""" - return self._checkpoint - - @checkpoint.setter - def checkpoint(self): - """Generates a checkpoint step for this job based on job.type""" + '''Generates a checkpoint step for this job based on job.type.''' if self.type == Type.PYTHON: - self._checkpoint = "checkpoint()" + return "checkpoint()" elif self.type == Type.R: - self._checkpoint = "checkpoint()" - else: # bash - self._checkpoint = "as_checkpoint" + return "checkpoint()" + else: # bash + return "as_checkpoint" def get_checkpoint_files(self): """ @@ -1452,7 +1446,7 @@ class Job(object): parameters['EXPORT'] = self.export parameters['PROJECT_TYPE'] = as_conf.get_project_type() self.wchunkinc = as_conf.get_wchunkinc(self.section) - for key,value in as_conf.jobs_data.get(self.section,{}).items(): + for key,value in as_conf.jobs_data[self.section].items(): parameters["CURRENT_"+key.upper()] = value return parameters diff --git a/autosubmit/job/job_common.py b/autosubmit/job/job_common.py index f6d34ccff..d705b1d1d 100644 --- a/autosubmit/job/job_common.py +++ b/autosubmit/job/job_common.py @@ -200,13 +200,14 @@ class StatisticsSnippetPython: stat_file = open(job_name_ptrn + '_STAT', 'w') stat_file.write('{0:.0f}\\n'.format(time.time())) stat_file.close() - ################### # Autosubmit Checkpoint ################### # Creates a new checkpoint file upton call based on the current numbers of calls to the function AS_CHECKPOINT_CALLS = 0 def as_checkpoint(): + global AS_CHECKPOINT_CALLS + global job_name_ptrn AS_CHECKPOINT_CALLS = AS_CHECKPOINT_CALLS + 1 open(job_name_ptrn + '_CHECKPOINT_' + str(AS_CHECKPOINT_CALLS), 'w').close() ################### -- GitLab From 1cb0171eb762cb57502f0bfb2be1eb34f7767a74 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 3 Aug 2023 08:43:32 +0200 Subject: [PATCH 27/68] changed is false for not --- autosubmit/job/job_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index ee7a5c6e3..58402d439 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -135,7 +135,7 @@ class JobList(object): processed_job_list = [] for job in self._job_list: # We are assuming that the jobs are sorted in topological order (which is the default) if ( - job.member is None and found_member is False) or job.member in self._run_members or job.status not in [ + job.member is None and not found_member) or job.member in self._run_members or job.status not in [ Status.WAITING, Status.READY]: processed_job_list.append(job) if job.member is not None and len(str(job.member)) > 0: -- GitLab From b7e9cb6e3d889cdcc84a7f93e14492de5a57dc8e Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 3 Aug 2023 09:36:33 +0200 Subject: [PATCH 28/68] fixed two issues --- autosubmit/job/job_list.py | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 58402d439..2304cc096 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -359,7 +359,6 @@ class JobList(object): except Exception as e: pass if parameters.get(section, None) is None: - Log.printlog("WARNING: SECTION {0} is not defined in jobs.conf".format(section)) continue # raise AutosubmitCritical("Section:{0} doesn't exists.".format(section),7014) dependency_running_type = str(parameters[section].get('RUNNING', 'once')).lower() @@ -387,7 +386,7 @@ class JobList(object): @staticmethod - def _apply_filter(parent_value, filter_value, associative_list, level_to_check="DATES_FROM",child=None,parent=None): + def _apply_filter(parent_value, filter_value, associative_list, level_to_check="DATES_FROM", child=None, parent=None): """ Check if the current_job_value is included in the filter_value :param parent_value: @@ -399,7 +398,6 @@ class JobList(object): """ if "NONE".casefold() in str(parent_value).casefold(): return True - if parent and child and level_to_check.casefold() == "splits".casefold(): if not parent.splits: parent_splits = -1 @@ -412,6 +410,9 @@ class JobList(object): if parent_splits == child_splits: to_look_at_lesser = associative_list lesser_group = -1 + lesser = str(parent_splits) + greater = str(child_splits) + lesser_value = "parent" else: if parent_splits > child_splits: lesser = str(child_splits) @@ -446,19 +447,22 @@ class JobList(object): else: split_info = 1 # split_info: if a value is 1, it means that the filter is 1-to-1, if it is 2, it means that the filter is 1-to-2, etc. - if (split_info == 1 or level_to_check.casefold() != "splits".casefold()) and str(parent_value).casefold() == str(filter_).casefold() : - if child.split == parent_value: - return True - elif split_info > 1 and level_to_check.casefold() == "splits".casefold(): - # 1-to-X filter - to_look_at_greater = [associative_list[i:i + split_info] for i in - range(0, int(greater), split_info)] - if lesser_value == "parent": - if str(child.split) in to_look_at_greater[lesser_group]: - return True - else: - if str(parent_value) in to_look_at_greater[lesser_group]: + if child and parent: + if (split_info == 1 or level_to_check.casefold() != "splits".casefold()) and str(parent_value).casefold() == str(filter_).casefold(): + if child.split == parent_value: return True + elif split_info > 1 and level_to_check.casefold() == "splits".casefold(): + # 1-to-X filter + to_look_at_greater = [associative_list[i:i + split_info] for i in + range(0, int(greater), split_info)] + if lesser_value == "parent": + if str(child.split) in to_look_at_greater[lesser_group]: + return True + else: + if str(parent_value) in to_look_at_greater[lesser_group]: + return True + else: + filter_value += filter_ + "," else: filter_value += filter_ + "," filter_value = filter_value[:-1] @@ -582,6 +586,9 @@ class JobList(object): if end is None: end = value_list[-1] try: + if level_to_check == "CHUNKS_TO": + start = int(start) + end = int(end) return value_list[slice(value_list.index(start), value_list.index(end)+1, int(step))] except ValueError: return value_list[slice(0,len(value_list)-1,int(step))] -- GitLab From a1e23119280ddf1ef57b898dd216a190edadd150 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 3 Aug 2023 09:39:46 +0200 Subject: [PATCH 29/68] added `%AS_CHECKPOINT%` --- docs/source/userguide/defining_workflows/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/userguide/defining_workflows/index.rst b/docs/source/userguide/defining_workflows/index.rst index 49c35401d..ac4462b37 100644 --- a/docs/source/userguide/defining_workflows/index.rst +++ b/docs/source/userguide/defining_workflows/index.rst @@ -231,7 +231,7 @@ The `STATUS` keyword can be used to select the status of the dependency that you The status are ordered, so if you select "RUNNING" status, the task will be run if the dependency is running or any status after it. The `FROM_STEP` keyword can be used to select the step of the dependency that you want to check. The possible value is an integer. -Additionally, the target dependency, must call to %AS_CHECKPOINT% inside their scripts. This will create a checkpoint that will be used to check the amount of steps processed. +Additionally, the target dependency, must call to `%AS_CHECKPOINT%` inside their scripts. This will create a checkpoint that will be used to check the amount of steps processed. .. code-block:: yaml -- GitLab From 77c71673ede5e8470a5827377fcb0569e704da9f Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 3 Aug 2023 10:12:37 +0200 Subject: [PATCH 30/68] fixed color --- autosubmit/monitor/monitor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/autosubmit/monitor/monitor.py b/autosubmit/monitor/monitor.py index 53293ef11..bb6b841ca 100644 --- a/autosubmit/monitor/monitor.py +++ b/autosubmit/monitor/monitor.py @@ -235,10 +235,10 @@ class Monitor: def _check_final_status(self, job, child): # order of self._table - # child.edge_info is a tuple, I want to get first element of each tuple with a lambda + # the dictionary is composed by: label = None if len(child.edge_info) > 0: - if job in child.edge_info.get("FAILED",{}): + if job.name in child.edge_info.get("FAILED",{}): color = self._table.get(Status.FAILED,None) label = child.edge_info["FAILED"].get(job.name,0)[1] elif job.name in child.edge_info.get("RUNNING",{}): @@ -263,7 +263,7 @@ class Monitor: elif job.name in child.edge_info.get("SUBMITTED",{}): color = self._table.get(Status.SUBMITTED,None) else: - color = self._table.get(Status.COMPLETED,None) + return None, None if label and label == 0: label = None return color,label -- GitLab From f9cc9afe3dff058edaabfa9eb3b49ece932624bb Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 3 Aug 2023 11:41:09 +0200 Subject: [PATCH 31/68] update doc --- .../userguide/defining_workflows/index.rst | 33 +++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/source/userguide/defining_workflows/index.rst b/docs/source/userguide/defining_workflows/index.rst index ac4462b37..39a41d7b2 100644 --- a/docs/source/userguide/defining_workflows/index.rst +++ b/docs/source/userguide/defining_workflows/index.rst @@ -228,9 +228,38 @@ The `STATUS` keyword can be used to select the status of the dependency that you * "COMPLETED": The task is completed. # Default * "SUSPENDED": The task is suspended. -The status are ordered, so if you select "RUNNING" status, the task will be run if the dependency is running or any status after it. +The status are ordered, so if you select "RUNNING" status, the task will be run if the parent is in any of the following statuses: "RUNNING", "QUEUING", "HELD", "SUBMITTED", "READY", "PREPARED", "DELAYED", "WAITING". + +The `FROM_STEP` keyword can be used to select the **internal** step of the dependency that you want to check. The possible value is an integer. + +To select an specific task, you have to combine the `STATUS` and `CHUNKS_TO` , `MEMBERS_TO` and `DATES_TO`, `SPLITS_TO` keywords. + +```yaml +JOBS: + A: + FILE: a + RUNNING: once + SPLITS: 1 + B: + FILE: b + RUNNING: once + SPLITS: 2 + DEPENDENCIES: A + C: + FILE: c + RUNNING: once + SPLITS: 1 + DEPENDENCIES: B + RECOVER_B_2: + FILE: fix_b + RUNNING: once + DEPENDENCIES: + B: + SPLIT_TO: "2" + STATUS: "RUNNING" + FROM_STEP: 1 +``` -The `FROM_STEP` keyword can be used to select the step of the dependency that you want to check. The possible value is an integer. Additionally, the target dependency, must call to `%AS_CHECKPOINT%` inside their scripts. This will create a checkpoint that will be used to check the amount of steps processed. .. code-block:: yaml -- GitLab From 33d3d0281c2b1d828208738a9a7a1b74894338ab Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 3 Aug 2023 11:43:12 +0200 Subject: [PATCH 32/68] update doc --- .../userguide/defining_workflows/index.rst | 50 ++++++++----------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/docs/source/userguide/defining_workflows/index.rst b/docs/source/userguide/defining_workflows/index.rst index 39a41d7b2..652bdc01e 100644 --- a/docs/source/userguide/defining_workflows/index.rst +++ b/docs/source/userguide/defining_workflows/index.rst @@ -230,12 +230,31 @@ The `STATUS` keyword can be used to select the status of the dependency that you The status are ordered, so if you select "RUNNING" status, the task will be run if the parent is in any of the following statuses: "RUNNING", "QUEUING", "HELD", "SUBMITTED", "READY", "PREPARED", "DELAYED", "WAITING". -The `FROM_STEP` keyword can be used to select the **internal** step of the dependency that you want to check. The possible value is an integer. +.. code-block:: yaml + + ini: + FILE: ini.sh + RUNNING: member + + sim: + FILE: sim.sh + DEPENDENCIES: ini sim-1 + RUNNING: chunk + + postprocess: + FILE: postprocess.sh + DEPENDENCIES: + SIM: + STATUS: "RUNNING" + RUNNING: chunk + + +The `FROM_STEP` keyword can be used to select the **internal** step of the dependency that you want to check. The possible value is an integer. Additionally, the target dependency, must call to `%AS_CHECKPOINT%` inside their scripts. This will create a checkpoint that will be used to check the amount of steps processed. To select an specific task, you have to combine the `STATUS` and `CHUNKS_TO` , `MEMBERS_TO` and `DATES_TO`, `SPLITS_TO` keywords. -```yaml -JOBS: +.. code-block:: yaml + A: FILE: a RUNNING: once @@ -257,31 +276,6 @@ JOBS: B: SPLIT_TO: "2" STATUS: "RUNNING" - FROM_STEP: 1 -``` - -Additionally, the target dependency, must call to `%AS_CHECKPOINT%` inside their scripts. This will create a checkpoint that will be used to check the amount of steps processed. - -.. code-block:: yaml - - ini: - FILE: ini.sh - RUNNING: member - - sim: - FILE: sim.sh - DEPENDENCIES: ini sim-1 - RUNNING: chunk - - postprocess: - FILE: postprocess.sh - DEPENDENCIES: - SIM: - STATUS: "RUNNING" - FROM_STEP: 0 - RUNNING: chunk - - Job frequency ~~~~~~~~~~~~~ -- GitLab From b550b5393d822af20c14ea624bb76c31d26fbe99 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 4 Aug 2023 12:39:04 +0200 Subject: [PATCH 33/68] docs --- .../userguide/defining_workflows/index.rst | 52 +++++++++++++++++-- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/docs/source/userguide/defining_workflows/index.rst b/docs/source/userguide/defining_workflows/index.rst index 652bdc01e..4f498daea 100644 --- a/docs/source/userguide/defining_workflows/index.rst +++ b/docs/source/userguide/defining_workflows/index.rst @@ -210,9 +210,9 @@ Start conditions Sometimes you want to run a job only when a certain condition is met. For example, you may want to run a job only when a certain task is running. This can be achieved using the START_CONDITIONS feature based on the dependencies rework. -Start conditions are achieved by adding the keyword `STATUS` and optionally `FROM_STEP` keywords into any dependency that you want. +Start conditions are achieved by adding the keyword ``STATUS`` and optionally ``FROM_STEP`` keywords into any dependency that you want. -The `STATUS` keyword can be used to select the status of the dependency that you want to check. The possible values ( case-insensitive ) are: +The ``STATUS`` keyword can be used to select the status of the dependency that you want to check. The possible values ( case-insensitive ) are: * "WAITING": The task is waiting for its dependencies to be completed. * "DELAYED": The task is delayed by a delay condition. @@ -249,9 +249,53 @@ The status are ordered, so if you select "RUNNING" status, the task will be run RUNNING: chunk -The `FROM_STEP` keyword can be used to select the **internal** step of the dependency that you want to check. The possible value is an integer. Additionally, the target dependency, must call to `%AS_CHECKPOINT%` inside their scripts. This will create a checkpoint that will be used to check the amount of steps processed. +The ``FROM_STEP`` keyword can be used to select the **internal** step of the dependency that you want to check. The possible value is an integer. Additionally, the target dependency, must call to `%AS_CHECKPOINT%` inside their scripts. This will create a checkpoint that will be used to check the amount of steps processed. -To select an specific task, you have to combine the `STATUS` and `CHUNKS_TO` , `MEMBERS_TO` and `DATES_TO`, `SPLITS_TO` keywords. +.. code-block:: yaml + + A: + FILE: a.sh + RUNNING: once + SPLITS: 2 + A_2: + FILE: a_2.sh + RUNNING: once + DEPENDENCIES: + A: + SPLIT_TO: "2" + STATUS: "RUNNING" + FROM_STEP: 2 + +There is now a new function that is automatically added in your scripts which is called ``as_checkpoint``. This is the function that is generating the checkpoint file. You can see the function below: + +.. code-block:: bash + + ################### + # AS CHECKPOINT FUNCTION + ################### + # Creates a new checkpoint file upon call based on the current numbers of calls to the function + + AS_CHECKPOINT_CALLS=0 + function as_checkpoint { + AS_CHECKPOINT_CALLS=$((AS_CHECKPOINT_CALLS+1)) + touch ${job_name_ptrn}_CHECKPOINT_${AS_CHECKPOINT_CALLS} + } + +And what you would have to include in your target dependency or dependencies is the call to this function which in this example is a.sh. + +The amount of calls is strongly related to the ``FROM_STEP`` value. + +``$expid/proj/$projname/as.sh`` + +.. code-block:: bash + + ##compute somestuff + as_checkpoint + ## compute some more stuff + as_checkpoint + + +To select an specific task, you have to combine the ``STATUS`` and ``CHUNKS_TO`` , ``MEMBERS_TO`` and ``DATES_TO``, ``SPLITS_TO`` keywords. .. code-block:: yaml -- GitLab From 2d331d06216388588ac3d8a66f86628f363e8878 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 22 Jun 2023 16:00:20 +0200 Subject: [PATCH 34/68] first step --- autosubmit/job/job_list.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index edd67d1c5..215350ced 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -70,6 +70,7 @@ class JobList(object): self._persistence_file = "job_list_" + expid self._job_list = list() self._base_job_list = list() + self.jobs_edges = dict() self._expid = expid self._config = config self.experiment_data = as_conf.experiment_data @@ -732,6 +733,18 @@ class JobList(object): return True, True return True, False return False,False + + def _add_edge_info(self,job,parent): + """ + Special relations to be check in the update_list method + :param job: Current job + :param parent: parent jobs to check + :return: + """ + if job.name not in self.jobs_edges: + self.jobs_edges[job.name] = [] + else: + self.jobs_edges[job.name].append(parent) @staticmethod def _manage_job_dependencies(dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys, dependencies, graph): @@ -784,8 +797,17 @@ class JobList(object): job.add_parent(parent) JobList._add_edge(graph, job, parent) # Could be more variables in the future - if optional: - job.add_edge_info(parent.name,special_variables={"optional":True}) + # todo + checkpoint = False + if optional and checkpoint: + JobList._add_edge_info(job,parent) + job.add_edge_info(parent.name,special_variables={"optional":True,"checkpoint":True}) + if optional and not checkpoint: + #JobList._add_edge_info(job) + job.add_edge_info(parent.name, special_variables={"optional": True}) + if not optional and checkpoint: + JobList._add_edge_info(job,parent) + job.add_edge_info(parent.name, special_variables={"checkpoint": True}) JobList.handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member, member_list, dependency.section, graph, other_parents) -- GitLab From 7ec04520d52f5d8992c27b3add671b4d225d0265 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 26 Jun 2023 11:51:20 +0200 Subject: [PATCH 35/68] adding checkpoints --- autosubmit/job/job.py | 22 ++++++++++++++++++++- autosubmit/job/job_common.py | 33 +++++++++++++++++++++++++++++--- autosubmit/job/job_list.py | 14 +++++++++----- autosubmit/platforms/platform.py | 22 +++++++++++++++++++++ 4 files changed, 82 insertions(+), 9 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 4de01df59..0471fb3e4 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -221,7 +221,9 @@ class Job(object): self.total_jobs = None self.max_waiting_jobs = None self.exclusive = "" - + self._check_point = "" + # internal + self.current_checkpoint_step = 0 @property @autosubmit_parameter(name='tasktype') def section(self): @@ -253,6 +255,23 @@ class Job(object): self._fail_count = value @property + @autosubmit_parameter(name='checkpoint') + def checkpoint(self): + """Generates a checkpoint step for this job based on job.type""" + return self._checkpoint + @checkpoint.setter + def checkpoint(self): + """Generates a checkpoint step for this job based on job.type""" + if self.type == Type.PYTHON: + self._checkpoint = "checkpoint()" + elif self.type == Type.R: + self._checkpoint = "checkpoint()" + else: # bash + self._checkpoint = "as_checkpoint" + + def get_checkpoint_files(self): + """Downloads checkpoint files from remote host. If they aren't already in local.""" + self.platform.get_checkpoint_files(self) @autosubmit_parameter(name='sdate') def sdate(self): """Current start date.""" @@ -1346,6 +1365,7 @@ class Job(object): return parameters def update_job_parameters(self,as_conf, parameters): + parameters["AS_CHECKPOINT"] = self.checkpoint parameters['JOBNAME'] = self.name parameters['FAIL_COUNT'] = str(self.fail_count) parameters['SDATE'] = self.sdate diff --git a/autosubmit/job/job_common.py b/autosubmit/job/job_common.py index 4d05d985c..042c6e330 100644 --- a/autosubmit/job/job_common.py +++ b/autosubmit/job/job_common.py @@ -128,6 +128,16 @@ class StatisticsSnippetBash: job_name_ptrn='%CURRENT_LOGDIR%/%JOBNAME%' echo $(date +%s) > ${job_name_ptrn}_STAT + ################### + # AS CHECKPOINT FUNCTION + ################### + # Creates a new checkpoint file upton call based on the current numbers of calls to the function + + AS_CHECKPOINT_CALLS=0 + function as_checkpoint { + AS_CHECKPOINT_CALLS=$((AS_CHECKPOINT_CALLS+1)) + touch ${job_name_ptrn}_CHECKPOINT_${AS_CHECKPOINT_CALLS} + } ################### # Autosubmit job ################### @@ -190,11 +200,19 @@ class StatisticsSnippetPython: stat_file = open(job_name_ptrn + '_STAT', 'w') stat_file.write('{0:.0f}\\n'.format(time.time())) stat_file.close() - - + + ################### + # Autosubmit Checkpoint + ################### + # Creates a new checkpoint file upton call based on the current numbers of calls to the function + AS_CHECKPOINT_CALLS = 0 + def as_checkpoint(): + AS_CHECKPOINT_CALLS = AS_CHECKPOINT_CALLS + 1 + open(job_name_ptrn + '_CHECKPOINT_' + str(AS_CHECKPOINT_CALLS), 'w').close() ################### # Autosubmit job ################### + """) @@ -254,7 +272,16 @@ class StatisticsSnippetR: fileConn<-file(paste(job_name_ptrn,"_STAT", sep = ''),"w") writeLines(toString(trunc(as.numeric(Sys.time()))), fileConn) close(fileConn) - + ################### + # Autosubmit Checkpoint + ################### + # Creates a new checkpoint file upton call based on the current numbers of calls to the function + AS_CHECKPOINT_CALLS = 0 + as_checkpoint <- function() { + AS_CHECKPOINT_CALLS <<- AS_CHECKPOINT_CALLS + 1 + fileConn<-file(paste(job_name_ptrn,"_CHECKPOINT_",AS_CHECKPOINT_CALLS, sep = ''),"w") + close(fileConn) + } ################### # Autosubmit job ################### diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 215350ced..43449ab6c 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -797,11 +797,13 @@ class JobList(object): job.add_parent(parent) JobList._add_edge(graph, job, parent) # Could be more variables in the future - # todo - checkpoint = False + # todo, default to TRUE for testing propouses + checkpoint = "!r" + #checkpoint = "!r1" + #checkpoint = "!r1,2,3" if optional and checkpoint: JobList._add_edge_info(job,parent) - job.add_edge_info(parent.name,special_variables={"optional":True,"checkpoint":True}) + job.add_edge_info(parent.name,special_variables={"optional":True,"checkpoint":checkpoint}) if optional and not checkpoint: #JobList._add_edge_info(job) job.add_edge_info(parent.name, special_variables={"optional": True}) @@ -1908,6 +1910,10 @@ class JobList(object): def parameters(self, value): self._parameters = value + def check_checkpoint(self, job, parent): + """ Check if a checkpoint step exists for this edge""" + return job.get_checkpoint_files(parent.name) + def update_list(self, as_conf, store_change=True, fromSetStatus=False, submitter=None, first_time=False): # type: (AutosubmitConfig, bool, bool, object, bool) -> bool """ @@ -2001,8 +2007,6 @@ class JobList(object): Log.debug( "Resetting sync job: {0} status to: WAITING for parents completion...".format( job.name)) - - Log.debug('Updating WAITING jobs') if not fromSetStatus: all_parents_completed = [] diff --git a/autosubmit/platforms/platform.py b/autosubmit/platforms/platform.py index 1f23cc6fc..e5c5a80d8 100644 --- a/autosubmit/platforms/platform.py +++ b/autosubmit/platforms/platform.py @@ -512,6 +512,28 @@ class Platform(object): (job_out_filename, job_err_filename) = remote_logs self.get_files([job_out_filename, job_err_filename], False, 'LOG_{0}'.format(exp_id)) + def get_checkpoint_files(self,job): + """ + Get all the checkpoint files + + :param job_name: name of the job + :type job_name: str + """ + from pathlib import Path + local_checkpoint_path = Path(f"{self.get_files_path()}/CHECKPOINT_{job.current_checkpoint_step}") + if Path(local_checkpoint_path).exists(): + return True + while self.check_file_exists(f'{job.name}_CHECKPOINT_{job.current_checkpoint_step}'): + # check if it exists locally + if not self.check_file_exists(f'{job.name}_CHECKPOINT_{job.current_checkpoint_step}', False): + if self.get_file('{0}_CHECKPOINT'.format(job.name), must_exist=False): + return True + else: + return False + else: + return False + + self.get_files(['{0}_checkpoint'.format(job_name), '{0}_checkpoint.json'.format(job_name)], False) def get_completed_files(self, job_name, retries=0, recovery=False, wrapper_failed=False): """ Get the COMPLETED file of the given job -- GitLab From a178dfa54a25a5c80c535013ec72feee27ee8af8 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 28 Jun 2023 12:00:55 +0200 Subject: [PATCH 36/68] almost done --- autosubmit/job/job.py | 4 +- autosubmit/job/job_list.py | 126 ++++++++++++++++++++++--------- autosubmit/platforms/platform.py | 39 +++++----- test/unit/test_dependencies.py | 42 ++++++++++- 4 files changed, 156 insertions(+), 55 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 0471fb3e4..4dc86ed1c 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -224,6 +224,7 @@ class Job(object): self._check_point = "" # internal self.current_checkpoint_step = 0 + @property @autosubmit_parameter(name='tasktype') def section(self): @@ -589,6 +590,7 @@ class Job(object): :type value: HPCPlatform """ self._partition = value + @property def children(self): """ @@ -1520,7 +1522,7 @@ class Job(object): template_file.close() else: if self.type == Type.BASH: - template = 'sleep 5' + template = '%AS_CHECKPOINT%;sleep 320;%AS_CHECKPOINT%;sleep 320' elif self.type == Type.PYTHON2: template = 'time.sleep(5)' + "\n" elif self.type == Type.PYTHON3 or self.type == Type.PYTHON: diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 43449ab6c..acd47c0a3 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -275,8 +275,7 @@ class JobList(object): raise AutosubmitCritical("Some section jobs of the wrapper:{0} are not in the current job_list defined in jobs.conf".format(wrapper_section),7014,str(e)) - @staticmethod - def _add_dependencies(date_list, member_list, chunk_list, dic_jobs, graph, option="DEPENDENCIES"): + def _add_dependencies(self,date_list, member_list, chunk_list, dic_jobs, graph, option="DEPENDENCIES"): jobs_data = dic_jobs._jobs_data.get("JOBS",{}) for job_section in jobs_data.keys(): Log.debug("Adding dependencies for {0} jobs".format(job_section)) @@ -295,7 +294,7 @@ class JobList(object): dependencies_keys[dependency] = {} if dependencies_keys is None: dependencies_keys = {} - dependencies = JobList._manage_dependencies(dependencies_keys, dic_jobs, job_section) + dependencies = self._manage_dependencies(dependencies_keys, dic_jobs, job_section) for job in dic_jobs.get_jobs(job_section): num_jobs = 1 @@ -303,7 +302,7 @@ class JobList(object): num_jobs = len(job) for i in range(num_jobs): _job = job[i] if num_jobs > 1 else job - JobList._manage_job_dependencies(dic_jobs, _job, date_list, member_list, chunk_list, dependencies_keys, + self._manage_job_dependencies(dic_jobs, _job, date_list, member_list, chunk_list, dependencies_keys, dependencies, graph) pass @@ -423,24 +422,48 @@ class JobList(object): else: return False + @staticmethod + def _parse_checkpoint(data): + checkpoint = {"STATUS": None, "FROM_STEP": None} + data = data.lower() + if data[0] == "r": + checkpoint["STATUS"] = Status.RUNNING + if len(data) > 1: + checkpoint["FROM_STEP"] = data[1:] + else: + checkpoint["FROM_STEP"] = "1" + elif data[0] == "f": + checkpoint["STATUS"] = Status.FAILED + if len(data) > 1: + checkpoint["FROM_STEP"] = data[1:] + else: + checkpoint["FROM_STEP"] = "1" + elif data[0] == "q": + checkpoint["STATUS"] = Status.QUEUING + elif data[0] == "s": + checkpoint["STATUS"] = Status.SUBMITTED + return checkpoint @staticmethod - def _check_relationship(relationships,level_to_check,value_to_check): + def _check_relationship(relationships, level_to_check, value_to_check): """ Check if the current_job_value is included in the filter_value - :param relationship: current filter level to check. + :param relationships: current filter level to check. :param level_to_check: can be a date, member, chunk or split. :param value_to_check: Can be None, a date, a member, a chunk or a split. :return: """ filters = [] - for filter_range,filter_data in relationships.get(level_to_check,{}).items(): + for filter_range, filter_data in relationships.get(level_to_check, {}).items(): if not value_to_check or str(filter_range).upper() in "ALL" or str(value_to_check).upper() in str(filter_range).upper(): if filter_data: if "?" in filter_range: filter_data["OPTIONAL"] = True else: filter_data["OPTIONAL"] = relationships["OPTIONAL"] + if "!" in filter_range: + filter_data["CHECKPOINT"] = "!"+filter_range.split("!")[1] + #JobList._parse_checkpoint(filter_range.split("!")[1]) filters.append(filter_data) # Normalize the filter return if len(filters) == 0: @@ -486,22 +509,22 @@ class JobList(object): # Will enter, go recursivily to the similar methods and in the end it will do: # Will enter members_from, and obtain [{DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", CHUNKS_FROM{...}] if "MEMBERS_FROM" in filter: - filters_to_apply_m = JobList._check_members({"MEMBERS_FROM": (filter.pop("MEMBERS_FROM")),"OPTIONAL":optional}, current_job) + filters_to_apply_m = JobList._check_members({"MEMBERS_FROM": (filter.pop("MEMBERS_FROM")), "OPTIONAL":optional}, current_job) if len(filters_to_apply_m) > 0: filters_to_apply[i].update(filters_to_apply_m) # Will enter chunks_from, and obtain [{DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"] if "CHUNKS_FROM" in filter: - filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM")),"OPTIONAL":optional}, current_job) + filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM")), "OPTIONAL":optional}, current_job) if len(filters_to_apply_c) > 0 and len(filters_to_apply_c[0]) > 0: filters_to_apply[i].update(filters_to_apply_c) - #IGNORED + # IGNORED if "SPLITS_FROM" in filter: filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_s) > 0: filters_to_apply[i].update(filters_to_apply_s) # Unify filters from all filters_from where the current job is included to have a single SET of filters_to if optional: - for i,filter in enumerate(filters_to_apply): + for i in range(0, len(filters_to_apply)): filters_to_apply[i]["OPTIONAL"] = True filters_to_apply = JobList._unify_to_filters(filters_to_apply) # {DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"} @@ -517,19 +540,19 @@ class JobList(object): """ filters_to_apply = JobList._check_relationship(relationships, "MEMBERS_FROM", current_job.member) optional = False - for i,filter in enumerate(filters_to_apply): - optional = filter.pop("OPTIONAL", False) - if "CHUNKS_FROM" in filter: - filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM")),"OPTIONAL":optional}, current_job) + for i, filter_ in enumerate(filters_to_apply): + optional = filter_.pop("OPTIONAL", False) + if "CHUNKS_FROM" in filter_: + filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter_.pop("CHUNKS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_c) > 0: filters_to_apply[i].update(filters_to_apply_c) - if "SPLITS_FROM" in filter: - filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) + if "SPLITS_FROM" in filter_: + filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter_.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) if len(filters_to_apply_s) > 0: filters_to_apply[i].update(filters_to_apply_s) if optional: - for i,filter in enumerate(filters_to_apply): + for i in range(0, len(filters_to_apply) > 0): filters_to_apply[i]["OPTIONAL"] = True filters_to_apply = JobList._unify_to_filters(filters_to_apply) return filters_to_apply @@ -584,11 +607,22 @@ class JobList(object): if aux: aux = aux.split(",") for element in aux: - if element.lower().strip("?") in ["natural","none"] and len(unified_filter[filter_type]) > 0: + # element is SECTION(alphanumeric) then ? or ! can figure and then ! or ? can figure + # Get only the first alphanumeric part + parsed_element = re.findall(r"[\w']+", element)[0].lower() + # Get the rest + data = element[len(parsed_element):] + if parsed_element in ["natural", "none"] and len(unified_filter[filter_type]) > 0: continue else: - if filter_to.get("OPTIONAL",False) and element[-1] != "?": - element += "?" + if filter_to.get("OPTIONAL", False) or "?" in data: + if "?" not in element: + element += "?" + if "!" in data: + element = parsed_element+data + elif filter_to.get("CHECKPOINT", None): + element = parsed_element+filter_to.get("CHECKPOINT", None) + unified_filter[filter_type].add(element) @staticmethod def _normalize_to_filters(filter_to,filter_type): @@ -631,8 +665,7 @@ class JobList(object): @staticmethod def _filter_current_job(current_job,relationships): - ''' - This function will filter the current job based on the relationships given + ''' This function will filter the current job based on the relationships given :param current_job: Current job to filter :param relationships: Relationships to apply :return: dict() with the filters to apply, or empty dict() if no filters to apply @@ -658,6 +691,8 @@ class JobList(object): if relationships is not None and len(relationships) > 0: if "OPTIONAL" not in relationships: relationships["OPTIONAL"] = False + if "CHECKPOINT" not in relationships: + relationships["CHECKPOINT"] = None # Look for a starting point, this can be if else becasue they're exclusive as a DATE_FROM can't be in a MEMBER_FROM and so on if "DATES_FROM" in relationships: filters_to_apply = JobList._check_dates(relationships, current_job) @@ -669,6 +704,7 @@ class JobList(object): filters_to_apply = JobList._check_splits(relationships, current_job) else: relationships.pop("OPTIONAL", None) + relationships.pop("CHECKPOINT", None) relationships.pop("CHUNKS_FROM", None) relationships.pop("MEMBERS_FROM", None) relationships.pop("DATES_FROM", None) @@ -745,8 +781,8 @@ class JobList(object): self.jobs_edges[job.name] = [] else: self.jobs_edges[job.name].append(parent) - @staticmethod - def _manage_job_dependencies(dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys, dependencies, + + def _manage_job_dependencies(self,dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys, dependencies, graph): ''' Manage the dependencies of a job @@ -795,20 +831,19 @@ class JobList(object): # If the parent is valid, add it to the graph if valid: job.add_parent(parent) - JobList._add_edge(graph, job, parent) + self._add_edge(graph, job, parent) # Could be more variables in the future # todo, default to TRUE for testing propouses - checkpoint = "!r" - #checkpoint = "!r1" - #checkpoint = "!r1,2,3" + # Do parse checkpoint + checkpoint= {"status":Status.RUNNING,"from_step":2} if optional and checkpoint: - JobList._add_edge_info(job,parent) + self._add_edge_info(job,parent) job.add_edge_info(parent.name,special_variables={"optional":True,"checkpoint":checkpoint}) if optional and not checkpoint: #JobList._add_edge_info(job) job.add_edge_info(parent.name, special_variables={"optional": True}) if not optional and checkpoint: - JobList._add_edge_info(job,parent) + self._add_edge_info(job,parent) job.add_edge_info(parent.name, special_variables={"checkpoint": True}) JobList.handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member, member_list, dependency.section, graph, other_parents) @@ -1914,6 +1949,18 @@ class JobList(object): """ Check if a checkpoint step exists for this edge""" return job.get_checkpoint_files(parent.name) + def check_checkpoint_parent_status(self): + """ + Check if all parents of a job have the correct status for checkpointing + :return: jobs that fullfill the special conditions """ + jobs_to_check = [] + for job, parent_to_check in self.jobs_edges.keys(): + checkpoint_info = job.edge_info.get(parent_to_check.name, {}).get("checkpoint", None) + if checkpoint_info: + if job.get_checkpoint_files(checkpoint_info["from_step"]): + if parent_to_check.status != checkpoint_info["status"]: + jobs_to_check.append(job) + return jobs_to_check def update_list(self, as_conf, store_change=True, fromSetStatus=False, submitter=None, first_time=False): # type: (AutosubmitConfig, bool, bool, object, bool) -> bool """ @@ -1936,7 +1983,7 @@ class JobList(object): write_log_status = False if not first_time: for job in self.get_failed(): - if self.jobs_data[job.section].get("RETRIALS",None) is None: + if self.jobs_data[job.section].get("RETRIALS", None) is None: retrials = int(as_conf.get_retrials()) else: retrials = int(job.retrials) @@ -1950,7 +1997,7 @@ class JobList(object): else: aux_job_delay = int(job.delay_retrials) - if self.jobs_data[job.section].get("DELAY_RETRY_TIME",None) or aux_job_delay <= 0: + if self.jobs_data[job.section].get("DELAY_RETRY_TIME", None) or aux_job_delay <= 0: delay_retry_time = str(as_conf.get_delay_retry_time()) else: delay_retry_time = job.retry_delay @@ -1958,7 +2005,7 @@ class JobList(object): retry_delay = job.fail_count * int(delay_retry_time[:-1]) + int(delay_retry_time[:-1]) elif "*" in delay_retry_time: retry_delay = int(delay_retry_time[1:]) - for retrial_amount in range(0,job.fail_count): + for retrial_amount in range(0, job.fail_count): retry_delay += retry_delay * 10 else: retry_delay = int(delay_retry_time) @@ -1985,6 +2032,17 @@ class JobList(object): job.status = Status.FAILED job.packed = False save = True + # Check checkpoint jobs, the status can be Ready, Running, Queuing + for job in self.check_checkpoint_parent_status(): + # Check if all jobs fullfill the conditions to a job be ready + tmp = [parent for parent in job.parents if parent.status == Status.COMPLETED or parent in self.jobs_edges[job] ] + if len(tmp) == len(job.parents): + job.status = Status.READY + job.id = None + job.packed = False + job.wrapper_type = None + save = True + Log.debug(f"Special condition fullfilled for job {job.name}") # if waiting jobs has all parents completed change its State to READY for job in self.get_completed(): if job.synchronize is not None and len(str(job.synchronize)) > 0: diff --git a/autosubmit/platforms/platform.py b/autosubmit/platforms/platform.py index e5c5a80d8..1689aec93 100644 --- a/autosubmit/platforms/platform.py +++ b/autosubmit/platforms/platform.py @@ -1,5 +1,6 @@ import locale import os +from pathlib import Path import traceback from autosubmit.job.job_common import Status @@ -512,28 +513,19 @@ class Platform(object): (job_out_filename, job_err_filename) = remote_logs self.get_files([job_out_filename, job_err_filename], False, 'LOG_{0}'.format(exp_id)) - def get_checkpoint_files(self,job): + def get_checkpoint_files(self, job, step_to_check): """ - Get all the checkpoint files - - :param job_name: name of the job - :type job_name: str + Get all the checkpoint files of the given job """ - from pathlib import Path - local_checkpoint_path = Path(f"{self.get_files_path()}/CHECKPOINT_{job.current_checkpoint_step}") - if Path(local_checkpoint_path).exists(): - return True - while self.check_file_exists(f'{job.name}_CHECKPOINT_{job.current_checkpoint_step}'): - # check if it exists locally - if not self.check_file_exists(f'{job.name}_CHECKPOINT_{job.current_checkpoint_step}', False): - if self.get_file('{0}_CHECKPOINT'.format(job.name), must_exist=False): - return True - else: - return False - else: - return False - self.get_files(['{0}_checkpoint'.format(job_name), '{0}_checkpoint.json'.format(job_name)], False) + if step_to_check >= job.current_checkpoint_step: + return True + local_checkpoint_path = Path(f"{self.get_files_path()}/CHECKPOINT_{step_to_check}") + while self.check_file_exists(local_checkpoint_path): + job.current_checkpoint_step += 1 + self.remove_checkpoint_file(local_checkpoint_path) + else: + return False def get_completed_files(self, job_name, retries=0, recovery=False, wrapper_failed=False): """ Get the COMPLETED file of the given job @@ -604,6 +596,15 @@ class Platform(object): Log.debug('{0} been removed', filename) return True return False + def remove_checkpoint_file(self, filename): + """ + Removes *CHECKPOINT* files from remote + + :param job_name: name of job to check + :return: True if successful, False otherwise + """ + if self.check_file_exists(filename): + self.delete_file(filename) def check_file_exists(self, src, wrapper_failed=False): return True diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index a08a4a73d..0af18081b 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -11,6 +11,7 @@ class TestJobList(unittest.TestCase): # Define common test case inputs here self.relationships_dates = { "OPTIONAL": False, + "CHECKPOINT": None, "DATES_FROM": { "20020201": { "MEMBERS_FROM": { @@ -34,6 +35,7 @@ class TestJobList(unittest.TestCase): self.relationships_members = { "OPTIONAL": False, + "CHECKPOINT": None, "MEMBERS_FROM": { "fc2": { "SPLITS_FROM": { @@ -49,6 +51,7 @@ class TestJobList(unittest.TestCase): } self.relationships_chunks = { "OPTIONAL": False, + "CHECKPOINT": None, "CHUNKS_FROM": { "1": { "DATES_TO": "20020201", @@ -60,6 +63,7 @@ class TestJobList(unittest.TestCase): } self.relationships_chunks2 = { "OPTIONAL": False, + "CHECKPOINT": None, "CHUNKS_FROM": { "1": { "DATES_TO": "20020201", @@ -77,9 +81,9 @@ class TestJobList(unittest.TestCase): } } - self.relationships_splits = { "OPTIONAL": False, + "CHECKPOINT": None, "SPLITS_FROM": { "1": { "DATES_TO": "20020201", @@ -108,6 +112,42 @@ class TestJobList(unittest.TestCase): self.mock_job.member = None self.mock_job.chunk = None self.mock_job.split = None + + def test_parse_checkpoint(self): + data = "r2" + correct = {"FROM_STEP": '2', "STATUS":Status.RUNNING} + result = JobList._parse_checkpoint(data) + self.assertEqual(result, correct) + data = "r" + correct = {"FROM_STEP": '1', "STATUS":Status.RUNNING} + result = JobList._parse_checkpoint(data) + self.assertEqual(result, correct) + data = "f2" + correct = {"FROM_STEP": '2', "STATUS":Status.FAILED} + result = JobList._parse_checkpoint(data) + self.assertEqual(result, correct) + data = "f" + correct = {"FROM_STEP": '1', "STATUS":Status.FAILED} + result = JobList._parse_checkpoint(data) + self.assertEqual(result, correct) + data = "s" + correct = {"FROM_STEP": None, "STATUS":Status.SUBMITTED} + result = JobList._parse_checkpoint(data) + self.assertEqual(result, correct) + data = "s2" + correct = {"FROM_STEP": None, "STATUS":Status.SUBMITTED} + result = JobList._parse_checkpoint(data) + self.assertEqual(result, correct) + data = "q" + correct = {"FROM_STEP": None, "STATUS":Status.QUEUING} + result = JobList._parse_checkpoint(data) + self.assertEqual(result, correct) + data = "q2" + correct = {"FROM_STEP": None, "STATUS":Status.QUEUING} + result = JobList._parse_checkpoint(data) + self.assertEqual(result, correct) + + def test_simple_dependency(self): result_d = JobList._check_dates({}, self.mock_job) result_m = JobList._check_members({}, self.mock_job) -- GitLab From a0f9dd9cbaaf3218259aef510d65497d1b4e5fd2 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 3 Jul 2023 12:06:35 +0200 Subject: [PATCH 37/68] start_conditions fix checkpoint_files --- autosubmit/job/job.py | 14 +++++++++----- autosubmit/job/job_list.py | 25 +++++++++++++++---------- autosubmit/platforms/platform.py | 23 ++++++++++++++++------- 3 files changed, 40 insertions(+), 22 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 4dc86ed1c..03ac937c0 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -191,6 +191,7 @@ class Job(object): self.x11 = False self._local_logs = ('', '') self._remote_logs = ('', '') + self._checkpoint = None self.script_name = self.name + ".cmd" self.status = status self.prev_status = status @@ -221,7 +222,6 @@ class Job(object): self.total_jobs = None self.max_waiting_jobs = None self.exclusive = "" - self._check_point = "" # internal self.current_checkpoint_step = 0 @@ -270,9 +270,13 @@ class Job(object): else: # bash self._checkpoint = "as_checkpoint" - def get_checkpoint_files(self): - """Downloads checkpoint files from remote host. If they aren't already in local.""" - self.platform.get_checkpoint_files(self) + def get_checkpoint_files(self,steps): + """ + Downloads checkpoint files from remote host. If they aren't already in local. + :param steps: list of steps to download + :return: the max step downloaded + """ + return self.platform.get_checkpoint_files(self,steps) @autosubmit_parameter(name='sdate') def sdate(self): """Current start date.""" @@ -711,7 +715,7 @@ class Job(object): """ self.children.add(new_child) - def add_edge_info(self,parent_name, special_variables): + def add_edge_info(self,parent_name, special_variables={}): """ Adds edge information to the job diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index acd47c0a3..aefe961f0 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -770,7 +770,7 @@ class JobList(object): return True, False return False,False - def _add_edge_info(self,job,parent): + def _add_edge_info(self,job,special_variables): """ Special relations to be check in the update_list method :param job: Current job @@ -778,9 +778,12 @@ class JobList(object): :return: """ if job.name not in self.jobs_edges: - self.jobs_edges[job.name] = [] + self.jobs_edges[job] = special_variables.get("FROMSTEP", 0) else: - self.jobs_edges[job.name].append(parent) + if special_variables.get("FROMSTEP", 0) > self.jobs_edges[job]: + self.jobs_edges[job] = special_variables.get("FROMSTEP", 0) + + def _manage_job_dependencies(self,dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys, dependencies, graph): @@ -837,7 +840,7 @@ class JobList(object): # Do parse checkpoint checkpoint= {"status":Status.RUNNING,"from_step":2} if optional and checkpoint: - self._add_edge_info(job,parent) + self._add_edge_info(job,special_variables={"optional":True,"checkpoint":checkpoint}) job.add_edge_info(parent.name,special_variables={"optional":True,"checkpoint":checkpoint}) if optional and not checkpoint: #JobList._add_edge_info(job) @@ -1954,12 +1957,14 @@ class JobList(object): Check if all parents of a job have the correct status for checkpointing :return: jobs that fullfill the special conditions """ jobs_to_check = [] - for job, parent_to_check in self.jobs_edges.keys(): - checkpoint_info = job.edge_info.get(parent_to_check.name, {}).get("checkpoint", None) - if checkpoint_info: - if job.get_checkpoint_files(checkpoint_info["from_step"]): - if parent_to_check.status != checkpoint_info["status"]: - jobs_to_check.append(job) + for job, checkpoint_step in self.jobs_edges.items(): + if checkpoint_step > 0: + max_step = job.get_checkpoint_files(checkpoint_step) + else: + max_step = None + for parent in parent_to_check: #if checkpoint_info: + if parent.status != checkpoint_info["status"]: + jobs_to_check.append(job) return jobs_to_check def update_list(self, as_conf, store_change=True, fromSetStatus=False, submitter=None, first_time=False): # type: (AutosubmitConfig, bool, bool, object, bool) -> bool diff --git a/autosubmit/platforms/platform.py b/autosubmit/platforms/platform.py index 1689aec93..ba355ebb3 100644 --- a/autosubmit/platforms/platform.py +++ b/autosubmit/platforms/platform.py @@ -513,19 +513,28 @@ class Platform(object): (job_out_filename, job_err_filename) = remote_logs self.get_files([job_out_filename, job_err_filename], False, 'LOG_{0}'.format(exp_id)) - def get_checkpoint_files(self, job, step_to_check): + def get_checkpoint_files(self, job, max_step): """ - Get all the checkpoint files of the given job + Get all the checkpoint files of a job + :param job: Get the checkpoint files + :type job: Job + :param max_step: max step possible + :type max_step: int """ - if step_to_check >= job.current_checkpoint_step: - return True - local_checkpoint_path = Path(f"{self.get_files_path()}/CHECKPOINT_{step_to_check}") - while self.check_file_exists(local_checkpoint_path): + if job.current_checkpoint_step >= max_step: + return job.current_checkpoint_step + local_checkpoint_path = Path(f"{self.get_files_path()}/CHECKPOINT_{job.current_checkpoint_step}") + self.get_file(local_checkpoint_path, False, ignore_log=True) + while self.check_file_exists(local_checkpoint_path) and job.current_checkpoint_step <= max_step: + self.remove_checkpoint_file(local_checkpoint_path) job.current_checkpoint_step += 1 + local_checkpoint_path = Path(f"{self.get_files_path()}/CHECKPOINT_{job.current_checkpoint_step}") + self.get_file(local_checkpoint_path, False, ignore_log=True) self.remove_checkpoint_file(local_checkpoint_path) else: - return False + self.remove_checkpoint_file(local_checkpoint_path) + return job.current_checkpoint_step def get_completed_files(self, job_name, retries=0, recovery=False, wrapper_failed=False): """ Get the COMPLETED file of the given job -- GitLab From ba9cbc555a6940fd70ee98af0bae1e5e8780cb19 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 4 Jul 2023 17:24:27 +0200 Subject: [PATCH 38/68] added step in [] add STATUS and FROM_STEP syntax Changed ? meaning --- autosubmit/job/job.py | 5 +- autosubmit/job/job_list.py | 168 +++++++++++++++++-------------- autosubmit/platforms/platform.py | 14 +-- 3 files changed, 103 insertions(+), 84 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 03ac937c0..d5da089d1 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -715,7 +715,7 @@ class Job(object): """ self.children.add(new_child) - def add_edge_info(self,parent_name, special_variables={}): + def add_edge_info(self, parent_name, special_variables): """ Adds edge information to the job @@ -727,8 +727,9 @@ class Job(object): if parent_name not in self.edge_info: self.edge_info[parent_name] = special_variables else: + #TODO self.edge_info[parent_name].update(special_variables) - pass + def delete_parent(self, parent): """ Remove a parent from the job diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index aefe961f0..198002cb8 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -24,6 +24,7 @@ import pickle import traceback import math import copy +from collections import defaultdict from time import localtime, strftime, mktime from shutil import move from autosubmit.job.job import Job @@ -55,7 +56,6 @@ def threaded(fn): return thread return wrapper - class JobList(object): """ Class to manage the list of jobs to be run by autosubmit @@ -70,7 +70,10 @@ class JobList(object): self._persistence_file = "job_list_" + expid self._job_list = list() self._base_job_list = list() - self.jobs_edges = dict() + # convert job_edges_structure function to lambda + self.jobs_edges = {} + #"now str" + self._expid = expid self._config = config self.experiment_data = as_conf.experiment_data @@ -444,26 +447,78 @@ class JobList(object): checkpoint["STATUS"] = Status.SUBMITTED return checkpoint + @staticmethod + def _parse_filter_to_check(value_to_check): + """ + Parse the filter to check and return the value to check. + Selection process: + value_to_check can be: + a range: [0:], [:N], [0:N], [:-1], [0:N:M] ... + a value: N + a list of values : 0,2,4,5,7,10 ... + a range with step: [0::M], [::2], [0::3], [::3] ... + :param value_to_check: value to check. + :return: parsed value to check. + """ + # regex + value_to_check = str(value_to_check).upper() + if value_to_check is None: + return None + elif value_to_check == "ALL": + return "ALL" + elif value_to_check == "NONE": + return None + elif value_to_check == 1: + # range + if value_to_check[0] == ":": + # [:N] + return slice(None, int(value_to_check[1:])) + elif value_to_check[-1] == ":": + # [N:] + return slice(int(value_to_check[:-1]), None) + else: + # [N:M] + return slice(int(value_to_check.split(":")[0]), int(value_to_check.split(":")[1])) + elif value_to_check.count(":") == 2: +# range with step + if value_to_check[0] == ":": + # [::M] + return slice(None, None, int(value_to_check[2:])) + elif value_to_check[-1] == ":": + # [N::] + return slice(int(value_to_check[:-2]), None, None) + else: + # [N::M] + return slice(int(value_to_check.split(":")[0]), None, int(value_to_check.split(":")[2])) + elif "," in value_to_check: + # list + return value_to_check.split(",") + else: + # value + return value_to_check + + + + @staticmethod def _check_relationship(relationships, level_to_check, value_to_check): """ Check if the current_job_value is included in the filter_value :param relationships: current filter level to check. - :param level_to_check: can be a date, member, chunk or split. + :param level_to_check: Can be date_from, member_from, chunk_from, split_from. :param value_to_check: Can be None, a date, a member, a chunk or a split. :return: """ filters = [] - for filter_range, filter_data in relationships.get(level_to_check, {}).items(): - if not value_to_check or str(filter_range).upper() in "ALL" or str(value_to_check).upper() in str(filter_range).upper(): - if filter_data: - if "?" in filter_range: - filter_data["OPTIONAL"] = True - else: - filter_data["OPTIONAL"] = relationships["OPTIONAL"] - if "!" in filter_range: - filter_data["CHECKPOINT"] = "!"+filter_range.split("!")[1] - #JobList._parse_checkpoint(filter_range.split("!")[1]) + relationship = relationships.get(level_to_check, {}) + status = relationship.pop("STATUS", relationships.get("STATUS", None)) + from_step = relationship.pop("FROM_STEP", relationships.get("FROM_STEP", None)) + for filter_range, filter_data in relationship.items(): + if not value_to_check or str(value_to_check).upper() in str(JobList._parse_filter_to_check(filter_range)).upper(): + if not filter_data.get("STATUS", None): + filter_data["STATUS"] = status + if not filter_data.get("FROM_STEP", None): + filter_data["FROM_STEP"] = from_step filters.append(filter_data) # Normalize the filter return if len(filters) == 0: @@ -478,7 +533,7 @@ class JobList(object): :param current_job: Current job to check. :return: filters_to_apply """ - optional = False + filters_to_apply = JobList._check_relationship(relationships, "DATES_FROM", date2str(current_job.date)) # there could be multiple filters that apply... per example # Current task date is 20020201, and member is fc2 @@ -503,29 +558,25 @@ class JobList(object): # [{MEMBERS_FROM{..},CHUNKS_FROM{...}},{MEMBERS_FROM{..},SPLITS_FROM{...}}] for i,filter in enumerate(filters_to_apply): # {MEMBERS_FROM{..},CHUNKS_FROM{...}} I want too look ALL filters not only one, but I want to go recursivily until get the _TO filter - optional = filter.pop("OPTIONAL", False) # This is not an if_else, because the current level ( dates ) could have two different filters. # Second case commented: ( date_from 20020201 ) # Will enter, go recursivily to the similar methods and in the end it will do: # Will enter members_from, and obtain [{DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", CHUNKS_FROM{...}] if "MEMBERS_FROM" in filter: - filters_to_apply_m = JobList._check_members({"MEMBERS_FROM": (filter.pop("MEMBERS_FROM")), "OPTIONAL":optional}, current_job) + filters_to_apply_m = JobList._check_members({"MEMBERS_FROM": (filter.pop("MEMBERS_FROM"))}, current_job) if len(filters_to_apply_m) > 0: filters_to_apply[i].update(filters_to_apply_m) # Will enter chunks_from, and obtain [{DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"] if "CHUNKS_FROM" in filter: - filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM")), "OPTIONAL":optional}, current_job) + filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM"))}, current_job) if len(filters_to_apply_c) > 0 and len(filters_to_apply_c[0]) > 0: filters_to_apply[i].update(filters_to_apply_c) # IGNORED if "SPLITS_FROM" in filter: - filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) + filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM"))}, current_job) if len(filters_to_apply_s) > 0: filters_to_apply[i].update(filters_to_apply_s) # Unify filters from all filters_from where the current job is included to have a single SET of filters_to - if optional: - for i in range(0, len(filters_to_apply)): - filters_to_apply[i]["OPTIONAL"] = True filters_to_apply = JobList._unify_to_filters(filters_to_apply) # {DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"} return filters_to_apply @@ -539,21 +590,15 @@ class JobList(object): :return: filters_to_apply """ filters_to_apply = JobList._check_relationship(relationships, "MEMBERS_FROM", current_job.member) - optional = False for i, filter_ in enumerate(filters_to_apply): - optional = filter_.pop("OPTIONAL", False) if "CHUNKS_FROM" in filter_: - filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter_.pop("CHUNKS_FROM")),"OPTIONAL":optional}, current_job) + filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter_.pop("CHUNKS_FROM"))}, current_job) if len(filters_to_apply_c) > 0: filters_to_apply[i].update(filters_to_apply_c) - if "SPLITS_FROM" in filter_: - filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter_.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) + filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter_.pop("SPLITS_FROM"))}, current_job) if len(filters_to_apply_s) > 0: filters_to_apply[i].update(filters_to_apply_s) - if optional: - for i in range(0, len(filters_to_apply) > 0): - filters_to_apply[i]["OPTIONAL"] = True filters_to_apply = JobList._unify_to_filters(filters_to_apply) return filters_to_apply @@ -565,17 +610,13 @@ class JobList(object): :param current_job: Current job to check. :return: filters_to_apply """ - optional = False + filters_to_apply = JobList._check_relationship(relationships, "CHUNKS_FROM", current_job.chunk) for i,filter in enumerate(filters_to_apply): - optional = filter.pop("OPTIONAL", False) if "SPLITS_FROM" in filter: - filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM")),"OPTIONAL":optional}, current_job) + filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM"))}, current_job) if len(filters_to_apply_s) > 0: filters_to_apply[i].update(filters_to_apply_s) - if optional: - for i,filter in enumerate(filters_to_apply): - filters_to_apply[i]["OPTIONAL"] = True filters_to_apply = JobList._unify_to_filters(filters_to_apply) return filters_to_apply @@ -607,7 +648,6 @@ class JobList(object): if aux: aux = aux.split(",") for element in aux: - # element is SECTION(alphanumeric) then ? or ! can figure and then ! or ? can figure # Get only the first alphanumeric part parsed_element = re.findall(r"[\w']+", element)[0].lower() # Get the rest @@ -615,14 +655,8 @@ class JobList(object): if parsed_element in ["natural", "none"] and len(unified_filter[filter_type]) > 0: continue else: - if filter_to.get("OPTIONAL", False) or "?" in data: - if "?" not in element: - element += "?" - if "!" in data: - element = parsed_element+data - elif filter_to.get("CHECKPOINT", None): - element = parsed_element+filter_to.get("CHECKPOINT", None) - + if "?" not in element: + element += data unified_filter[filter_type].add(element) @staticmethod def _normalize_to_filters(filter_to,filter_type): @@ -649,18 +683,20 @@ class JobList(object): """ unified_filter = {"DATES_TO": set(), "MEMBERS_TO": set(), "CHUNKS_TO": set(), "SPLITS_TO": set()} for filter_to in filter_to_apply: + if "STATUS" not in unified_filter and filter_to.get("STATUS", None): + unified_filter["STATUS"] = filter_to["STATUS"] + if "FROM_STEP" not in unified_filter and filter_to.get("FROM_STEP", None): + unified_filter["FROM_STEP"] = filter_to["FROM_STEP"] if len(filter_to) > 0: JobList._unify_to_filter(unified_filter,filter_to,"DATES_TO") JobList._unify_to_filter(unified_filter,filter_to,"MEMBERS_TO") JobList._unify_to_filter(unified_filter,filter_to,"CHUNKS_TO") JobList._unify_to_filter(unified_filter,filter_to,"SPLITS_TO") - filter_to.pop("OPTIONAL", None) JobList._normalize_to_filters(unified_filter,"DATES_TO") JobList._normalize_to_filters(unified_filter,"MEMBERS_TO") JobList._normalize_to_filters(unified_filter,"CHUNKS_TO") JobList._normalize_to_filters(unified_filter,"SPLITS_TO") - return unified_filter @staticmethod @@ -689,10 +725,6 @@ class JobList(object): filters_to_apply = {} # Check if filter_from-filter_to relationship is set if relationships is not None and len(relationships) > 0: - if "OPTIONAL" not in relationships: - relationships["OPTIONAL"] = False - if "CHECKPOINT" not in relationships: - relationships["CHECKPOINT"] = None # Look for a starting point, this can be if else becasue they're exclusive as a DATE_FROM can't be in a MEMBER_FROM and so on if "DATES_FROM" in relationships: filters_to_apply = JobList._check_dates(relationships, current_job) @@ -764,24 +796,22 @@ class JobList(object): valid_chunks = JobList._apply_filter(parent.chunk, chunks_to, associative_list["chunks"], "chunks") valid_splits = JobList._apply_filter(parent.split, splits_to, associative_list["splits"], "splits") if valid_dates and valid_members and valid_chunks and valid_splits: - for value in [dates_to, members_to, chunks_to, splits_to]: - if "?" in value: - return True, True - return True, False - return False,False + return True + return False - def _add_edge_info(self,job,special_variables): + def _add_edge_info(self,job,parent,special_status): """ Special relations to be check in the update_list method :param job: Current job :param parent: parent jobs to check :return: """ - if job.name not in self.jobs_edges: - self.jobs_edges[job] = special_variables.get("FROMSTEP", 0) + if job not in self.jobs_edges: + self.jobs_edges[job] = {} + if special_status not in self.jobs_edges[job]: + self.jobs_edges[job][special_status] = [] else: - if special_variables.get("FROMSTEP", 0) > self.jobs_edges[job]: - self.jobs_edges[job] = special_variables.get("FROMSTEP", 0) + self.jobs_edges[job][special_status].append(parent) @@ -830,24 +860,15 @@ class JobList(object): else: natural_relationship = False # Check if the current parent is a valid parent based on the dependencies set on expdef.conf - valid,optional = JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship,filters_to_apply) # If the parent is valid, add it to the graph - if valid: + if JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship,filters_to_apply): job.add_parent(parent) self._add_edge(graph, job, parent) # Could be more variables in the future - # todo, default to TRUE for testing propouses # Do parse checkpoint - checkpoint= {"status":Status.RUNNING,"from_step":2} - if optional and checkpoint: - self._add_edge_info(job,special_variables={"optional":True,"checkpoint":checkpoint}) - job.add_edge_info(parent.name,special_variables={"optional":True,"checkpoint":checkpoint}) - if optional and not checkpoint: - #JobList._add_edge_info(job) - job.add_edge_info(parent.name, special_variables={"optional": True}) - if not optional and checkpoint: - self._add_edge_info(job,parent) - job.add_edge_info(parent.name, special_variables={"checkpoint": True}) + if filters_to_apply.get("STATUS",None): + self._add_edge_info(job, parent, filters_to_apply["STATUS"]) + job.add_edge_info(parent.name,{filters_to_apply["STATUS"],filters_to_apply["FROM_STEP"]}) JobList.handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member, member_list, dependency.section, graph, other_parents) @@ -1957,6 +1978,7 @@ class JobList(object): Check if all parents of a job have the correct status for checkpointing :return: jobs that fullfill the special conditions """ jobs_to_check = [] + todo for job, checkpoint_step in self.jobs_edges.items(): if checkpoint_step > 0: max_step = job.get_checkpoint_files(checkpoint_step) diff --git a/autosubmit/platforms/platform.py b/autosubmit/platforms/platform.py index ba355ebb3..84012944b 100644 --- a/autosubmit/platforms/platform.py +++ b/autosubmit/platforms/platform.py @@ -524,16 +524,12 @@ class Platform(object): if job.current_checkpoint_step >= max_step: return job.current_checkpoint_step - local_checkpoint_path = Path(f"{self.get_files_path()}/CHECKPOINT_{job.current_checkpoint_step}") - self.get_file(local_checkpoint_path, False, ignore_log=True) - while self.check_file_exists(local_checkpoint_path) and job.current_checkpoint_step <= max_step: - self.remove_checkpoint_file(local_checkpoint_path) + remote_checkpoint_path = f"{self.get_files_path()}/CHECKPOINT_" + self.get_file(remote_checkpoint_path+job.current_checkpoint_step, False, ignore_log=True) + while self.check_file_exists(remote_checkpoint_path+job.current_checkpoint_step) and job.current_checkpoint_step < max_step: + self.remove_checkpoint_file(remote_checkpoint_path+job.current_checkpoint_step) job.current_checkpoint_step += 1 - local_checkpoint_path = Path(f"{self.get_files_path()}/CHECKPOINT_{job.current_checkpoint_step}") - self.get_file(local_checkpoint_path, False, ignore_log=True) - self.remove_checkpoint_file(local_checkpoint_path) - else: - self.remove_checkpoint_file(local_checkpoint_path) + self.get_file(remote_checkpoint_path+job.current_checkpoint_step, False, ignore_log=True) return job.current_checkpoint_step def get_completed_files(self, job_name, retries=0, recovery=False, wrapper_failed=False): """ -- GitLab From c50452d5ee4a11bd6ea153a8fb74fc8ae8124037 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 5 Jul 2023 16:52:10 +0200 Subject: [PATCH 39/68] all working, tODO clean old conditional code remake tests dependencies --- autosubmit/job/job.py | 18 ++++---- autosubmit/job/job_list.py | 71 ++++++++++++++++++++------------ autosubmit/platforms/platform.py | 18 ++++---- 3 files changed, 62 insertions(+), 45 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index d5da089d1..a22dbc930 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -224,6 +224,7 @@ class Job(object): self.exclusive = "" # internal self.current_checkpoint_step = 0 + self.max_checkpoint_step = 0 @property @autosubmit_parameter(name='tasktype') @@ -270,13 +271,13 @@ class Job(object): else: # bash self._checkpoint = "as_checkpoint" - def get_checkpoint_files(self,steps): + def get_checkpoint_files(self): """ Downloads checkpoint files from remote host. If they aren't already in local. :param steps: list of steps to download :return: the max step downloaded """ - return self.platform.get_checkpoint_files(self,steps) + return self.platform.get_checkpoint_files(self) @autosubmit_parameter(name='sdate') def sdate(self): """Current start date.""" @@ -715,20 +716,19 @@ class Job(object): """ self.children.add(new_child) - def add_edge_info(self, parent_name, special_variables): + def add_edge_info(self, parent, special_variables): """ Adds edge information to the job - :param parent_name: parent name - :type parent_name: str + :param parent: parent job + :type parent: Job :param special_variables: special variables :type special_variables: dict """ - if parent_name not in self.edge_info: - self.edge_info[parent_name] = special_variables + if special_variables["STATUS"] not in self.edge_info: + self.edge_info[special_variables["STATUS"]] = [(parent, special_variables.get("FROM_STEP", 0))] else: - #TODO - self.edge_info[parent_name].update(special_variables) + self.edge_info[special_variables["STATUS"]].append((parent, special_variables.get("FROM_STEP", 0))) def delete_parent(self, parent): """ diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 198002cb8..2f9f8d7b9 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -70,10 +70,7 @@ class JobList(object): self._persistence_file = "job_list_" + expid self._job_list = list() self._base_job_list = list() - # convert job_edges_structure function to lambda self.jobs_edges = {} - #"now str" - self._expid = expid self._config = config self.experiment_data = as_conf.experiment_data @@ -799,19 +796,19 @@ class JobList(object): return True return False - def _add_edge_info(self,job,parent,special_status): + def _add_edge_info(self,job,special_status): """ Special relations to be check in the update_list method :param job: Current job :param parent: parent jobs to check :return: """ - if job not in self.jobs_edges: - self.jobs_edges[job] = {} - if special_status not in self.jobs_edges[job]: - self.jobs_edges[job][special_status] = [] - else: - self.jobs_edges[job][special_status].append(parent) + if special_status not in self.jobs_edges: + self.jobs_edges[special_status] = set() + self.jobs_edges[special_status].add(job) + if "ALL" not in self.jobs_edges: + self.jobs_edges["ALL"] = set() + self.jobs_edges["ALL"].add(job) @@ -832,6 +829,7 @@ class JobList(object): parsed_date_list = [] for dat in date_list: parsed_date_list.append(date2str(dat)) + special_conditions = dict() for key in dependencies_keys: dependency = dependencies.get(key,None) if dependency is None: @@ -850,6 +848,12 @@ class JobList(object): all_parents = list(set(other_parents + parents_jobs)) # Get dates_to, members_to, chunks_to of the deepest level of the relationship. filters_to_apply = JobList._filter_current_job(job,copy.deepcopy(dependency.relationships)) + if "?" in [filters_to_apply.get("SPLITS_TO",""),filters_to_apply.get("DATES_TO",""),filters_to_apply.get("MEMBERS_TO",""),filters_to_apply.get("CHUNKS_TO","")]: + only_marked_status = True + else: + only_marked_status = False + special_conditions["STATUS"] = filters_to_apply.pop("STATUS", None) + special_conditions["FROM_STEP"] = filters_to_apply.pop("FROM_STEP", None) for parent in all_parents: # If splits is not None, the job is a list of jobs if parent.name == job.name: @@ -861,14 +865,25 @@ class JobList(object): natural_relationship = False # Check if the current parent is a valid parent based on the dependencies set on expdef.conf # If the parent is valid, add it to the graph + if JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship,filters_to_apply): job.add_parent(parent) self._add_edge(graph, job, parent) # Could be more variables in the future # Do parse checkpoint - if filters_to_apply.get("STATUS",None): - self._add_edge_info(job, parent, filters_to_apply["STATUS"]) - job.add_edge_info(parent.name,{filters_to_apply["STATUS"],filters_to_apply["FROM_STEP"]}) + if special_conditions.get("STATUS",None): + if only_marked_status: + if str(job.split)+"?" in filters_to_apply.get("SPLITS_TO","") or str(job.chunk)+"?" in filters_to_apply.get("CHUNKS_TO","") or str(job.member)+"?" in filters_to_apply.get("MEMBERS_TO","") or str(job.date)+"?" in filters_to_apply.get("DATES_TO",""): + selected = False + else: + selected = True + else: + selected = True + if selected: + job.max_checkpoint_step = int(special_conditions.get("FROM_STEP", 0)) if int(special_conditions.get("FROM_STEP", + 0)) > job.max_checkpoint_step else job.max_checkpoint_step + self._add_edge_info(job, special_conditions["STATUS"]) + job.add_edge_info(parent,special_conditions) JobList.handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member, member_list, dependency.section, graph, other_parents) @@ -1973,20 +1988,24 @@ class JobList(object): """ Check if a checkpoint step exists for this edge""" return job.get_checkpoint_files(parent.name) - def check_checkpoint_parent_status(self): + def check_special_status(self): """ Check if all parents of a job have the correct status for checkpointing :return: jobs that fullfill the special conditions """ jobs_to_check = [] - todo - for job, checkpoint_step in self.jobs_edges.items(): - if checkpoint_step > 0: - max_step = job.get_checkpoint_files(checkpoint_step) - else: - max_step = None - for parent in parent_to_check: #if checkpoint_info: - if parent.status != checkpoint_info["status"]: - jobs_to_check.append(job) + for status, sorted_job_list in self.jobs_edges.items(): + if status == "ALL": + continue + for job in sorted_job_list: + if status in ["RUNNING", "FAILED"]: + if job.platform.connected: # This will be true only when used under setstatus/run + job.get_checkpoint_files() + for parent in job.edge_info[status]: + if parent[0].status == Status.WAITING: + if status in ["RUNNING", "FAILED"] and int(parent[1]) >= job.current_checkpoint_step: + continue + else: + jobs_to_check.append(parent[0]) return jobs_to_check def update_list(self, as_conf, store_change=True, fromSetStatus=False, submitter=None, first_time=False): # type: (AutosubmitConfig, bool, bool, object, bool) -> bool @@ -2059,10 +2078,10 @@ class JobList(object): job.status = Status.FAILED job.packed = False save = True - # Check checkpoint jobs, the status can be Ready, Running, Queuing - for job in self.check_checkpoint_parent_status(): + # Check checkpoint jobs, the status can be Any + for job in self.check_special_status(): # Check if all jobs fullfill the conditions to a job be ready - tmp = [parent for parent in job.parents if parent.status == Status.COMPLETED or parent in self.jobs_edges[job] ] + tmp = [parent for parent in job.parents if parent.status == Status.COMPLETED or parent in self.jobs_edges["ALL"] ] if len(tmp) == len(job.parents): job.status = Status.READY job.id = None diff --git a/autosubmit/platforms/platform.py b/autosubmit/platforms/platform.py index 84012944b..059609cc6 100644 --- a/autosubmit/platforms/platform.py +++ b/autosubmit/platforms/platform.py @@ -513,7 +513,7 @@ class Platform(object): (job_out_filename, job_err_filename) = remote_logs self.get_files([job_out_filename, job_err_filename], False, 'LOG_{0}'.format(exp_id)) - def get_checkpoint_files(self, job, max_step): + def get_checkpoint_files(self, job): """ Get all the checkpoint files of a job :param job: Get the checkpoint files @@ -522,15 +522,13 @@ class Platform(object): :type max_step: int """ - if job.current_checkpoint_step >= max_step: - return job.current_checkpoint_step - remote_checkpoint_path = f"{self.get_files_path()}/CHECKPOINT_" - self.get_file(remote_checkpoint_path+job.current_checkpoint_step, False, ignore_log=True) - while self.check_file_exists(remote_checkpoint_path+job.current_checkpoint_step) and job.current_checkpoint_step < max_step: - self.remove_checkpoint_file(remote_checkpoint_path+job.current_checkpoint_step) - job.current_checkpoint_step += 1 - self.get_file(remote_checkpoint_path+job.current_checkpoint_step, False, ignore_log=True) - return job.current_checkpoint_step + if job.current_checkpoint_step < job.max_checkpoint_step: + remote_checkpoint_path = f"{self.get_files_path()}/CHECKPOINT_" + self.get_file(remote_checkpoint_path+str(job.current_checkpoint_step), False, ignore_log=True) + while self.check_file_exists(remote_checkpoint_path+str(job.current_checkpoint_step)) and job.current_checkpoint_step < job.max_checkpoint_step: + self.remove_checkpoint_file(remote_checkpoint_path+str(job.current_checkpoint_step)) + job.current_checkpoint_step += 1 + self.get_file(remote_checkpoint_path+str(job.current_checkpoint_step), False, ignore_log=True) def get_completed_files(self, job_name, retries=0, recovery=False, wrapper_failed=False): """ Get the COMPLETED file of the given job -- GitLab From 030fc64e97fcec21fc738730de145bc055a84511 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 5 Jul 2023 16:54:41 +0200 Subject: [PATCH 40/68] all working, tODO clean old conditional code remake tests dependencies --- autosubmit/job/job_list.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 2f9f8d7b9..0794d7dfb 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -874,9 +874,9 @@ class JobList(object): if special_conditions.get("STATUS",None): if only_marked_status: if str(job.split)+"?" in filters_to_apply.get("SPLITS_TO","") or str(job.chunk)+"?" in filters_to_apply.get("CHUNKS_TO","") or str(job.member)+"?" in filters_to_apply.get("MEMBERS_TO","") or str(job.date)+"?" in filters_to_apply.get("DATES_TO",""): - selected = False - else: selected = True + else: + selected = False else: selected = True if selected: -- GitLab From 7706fd831cb7c7888896529b81d46aeba3274aac Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 10 Jul 2023 08:34:38 +0200 Subject: [PATCH 41/68] refined some issues --- autosubmit/job/job.py | 6 +- autosubmit/job/job_list.py | 539 +++++++++++++++++++--------------- autosubmit/monitor/monitor.py | 59 +++- 3 files changed, 366 insertions(+), 238 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index a22dbc930..3637f0897 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -726,9 +726,9 @@ class Job(object): :type special_variables: dict """ if special_variables["STATUS"] not in self.edge_info: - self.edge_info[special_variables["STATUS"]] = [(parent, special_variables.get("FROM_STEP", 0))] - else: - self.edge_info[special_variables["STATUS"]].append((parent, special_variables.get("FROM_STEP", 0))) + self.edge_info[special_variables["STATUS"]] = {} + + self.edge_info[special_variables["STATUS"]][parent.name] = (parent,special_variables.get("FROM_STEP", 0)) def delete_parent(self, parent): """ diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 0794d7dfb..71b128daa 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -14,37 +14,36 @@ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# You should have received a copy of the GNU General Public License -# along with Autosubmit. If not, see . -import collections import copy -import re +import datetime +import math import os import pickle +# You should have received a copy of the GNU General Public License +# along with Autosubmit. If not, see . +import re import traceback -import math -import copy -from collections import defaultdict -from time import localtime, strftime, mktime +from bscearth.utils.date import date2str, parse_date +from networkx import DiGraph from shutil import move +from threading import Thread +from time import localtime, strftime, mktime +from typing import List, Dict + +import autosubmit.database.db_structure as DbStructure +from autosubmit.helpers.data_transfer import JobRow from autosubmit.job.job import Job -from autosubmit.job.job_package_persistence import JobPackagePersistence +from autosubmit.job.job_common import Status, bcolors from autosubmit.job.job_dict import DicJobs +from autosubmit.job.job_package_persistence import JobPackagePersistence from autosubmit.job.job_packages import JobPackageThread from autosubmit.job.job_utils import Dependency -from autosubmit.job.job_common import Status, bcolors -from bscearth.utils.date import date2str, parse_date -import autosubmit.database.db_structure as DbStructure -import datetime -from networkx import DiGraph from autosubmit.job.job_utils import transitive_reduction -from log.log import AutosubmitCritical, AutosubmitError, Log -from threading import Thread from autosubmitconfigparser.config.basicconfig import BasicConfig from autosubmitconfigparser.config.configcommon import AutosubmitConfig -from autosubmit.helpers.data_transfer import JobRow -from typing import List, Dict -import log.fd_show +from log.log import AutosubmitCritical, AutosubmitError, Log + + # Log.get_logger("Log.Autosubmit") @@ -54,15 +53,17 @@ def threaded(fn): thread.name = "data_processing" thread.start() return thread + return wrapper + class JobList(object): """ Class to manage the list of jobs to be run by autosubmit """ - def __init__(self, expid, config, parser_factory, job_list_persistence,as_conf): + def __init__(self, expid, config, parser_factory, job_list_persistence, as_conf): self._persistence_path = os.path.join( config.LOCAL_ROOT_DIR, expid, "pkl") self._update_file = "updated_list_" + expid + ".txt" @@ -93,6 +94,7 @@ class JobList(object): self._run_members = None self.jobs_to_run_first = list() self.rerun_job_list = list() + @property def expid(self): """ @@ -127,46 +129,56 @@ class JobList(object): @run_members.setter def run_members(self, value): - if value is not None and len(str(value)) > 0 : + if value is not None and len(str(value)) > 0: self._run_members = value - self._base_job_list = [job for job in self._job_list] + self._base_job_list = [job for job in self._job_list] found_member = False processed_job_list = [] - for job in self._job_list: # We are assuming that the jobs are sorted in topological order (which is the default) - if (job.member is None and found_member is False) or job.member in self._run_members or job.status not in [Status.WAITING, Status.READY]: + for job in self._job_list: # We are assuming that the jobs are sorted in topological order (which is the default) + if ( + job.member is None and found_member is False) or job.member in self._run_members or job.status not in [ + Status.WAITING, Status.READY]: processed_job_list.append(job) if job.member is not None and len(str(job.member)) > 0: found_member = True - self._job_list = processed_job_list + self._job_list = processed_job_list # Old implementation that also considered children of the members. # self._job_list = [job for job in old_job_list if len( # job.parents) == 0 or len(set(old_job_list_names).intersection(set([jobp.name for jobp in job.parents]))) == len(job.parents)] - def create_dictionary(self, date_list, member_list, num_chunks, chunk_ini, date_format, default_retrials, wrapper_jobs): + def create_dictionary(self, date_list, member_list, num_chunks, chunk_ini, date_format, default_retrials, + wrapper_jobs): chunk_list = list(range(chunk_ini, num_chunks + 1)) jobs_parser = self._get_jobs_parser() dic_jobs = DicJobs(self, date_list, member_list, - chunk_list, date_format, default_retrials,jobs_data={},experiment_data=self.experiment_data) + chunk_list, date_format, default_retrials, jobs_data={}, + experiment_data=self.experiment_data) self._dic_jobs = dic_jobs for wrapper_section in wrapper_jobs: if str(wrapper_jobs[wrapper_section]).lower() != 'none': - self._ordered_jobs_by_date_member[wrapper_section] = self._create_sorted_dict_jobs(wrapper_jobs[wrapper_section]) + self._ordered_jobs_by_date_member[wrapper_section] = self._create_sorted_dict_jobs( + wrapper_jobs[wrapper_section]) else: self._ordered_jobs_by_date_member[wrapper_section] = {} pass + def _delete_edgeless_jobs(self): jobs_to_delete = [] # indices to delete for i, job in enumerate(self._job_list): if job.dependencies is not None: - if ( ( len(job.dependencies) > 0 and not job.has_parents()) and not job.has_children()) and job.delete_when_edgeless in ["true",True,1]: + if (( + len(job.dependencies) > 0 and not job.has_parents()) and not job.has_children()) and job.delete_when_edgeless in [ + "true", True, 1]: jobs_to_delete.append(job) # delete jobs by indices for i in jobs_to_delete: self._job_list.remove(i) + def generate(self, date_list, member_list, num_chunks, chunk_ini, parameters, date_format, default_retrials, - default_job_type, wrapper_type=None, wrapper_jobs=dict(), new=True, notransitive=False, update_structure=False, run_only_members=[],show_log=True,jobs_data={},as_conf=""): + default_job_type, wrapper_type=None, wrapper_jobs=dict(), new=True, notransitive=False, + update_structure=False, run_only_members=[], show_log=True, jobs_data={}, as_conf=""): """ Creates all jobs needed for the current workflow @@ -204,8 +216,8 @@ class JobList(object): chunk_list = list(range(chunk_ini, num_chunks + 1)) self._chunk_list = chunk_list - - dic_jobs = DicJobs(self,date_list, member_list,chunk_list, date_format, default_retrials,jobs_data,experiment_data=self.experiment_data) + dic_jobs = DicJobs(self, date_list, member_list, chunk_list, date_format, default_retrials, jobs_data, + experiment_data=self.experiment_data) self._dic_jobs = dic_jobs priority = 0 if show_log: @@ -221,15 +233,15 @@ class JobList(object): except Exception as e: pass Log.info("Deleting previous pkl due being incompatible with current AS version") - if os.path.exists(os.path.join(self._persistence_path, self._persistence_file+".pkl")): - os.remove(os.path.join(self._persistence_path, self._persistence_file+".pkl")) - if os.path.exists(os.path.join(self._persistence_path, self._persistence_file+"_backup.pkl")): - os.remove(os.path.join(self._persistence_path, self._persistence_file+"_backup.pkl")) + if os.path.exists(os.path.join(self._persistence_path, self._persistence_file + ".pkl")): + os.remove(os.path.join(self._persistence_path, self._persistence_file + ".pkl")) + if os.path.exists(os.path.join(self._persistence_path, self._persistence_file + "_backup.pkl")): + os.remove(os.path.join(self._persistence_path, self._persistence_file + "_backup.pkl")) - self._create_jobs(dic_jobs, priority,default_job_type, jobs_data) + self._create_jobs(dic_jobs, priority, default_job_type, jobs_data) if show_log: Log.info("Adding dependencies...") - self._add_dependencies(date_list, member_list,chunk_list, dic_jobs, self.graph) + self._add_dependencies(date_list, member_list, chunk_list, dic_jobs, self.graph) if show_log: Log.info("Removing redundant dependencies...") @@ -237,7 +249,7 @@ class JobList(object): new, notransitive, update_structure=update_structure) for job in self._job_list: job.parameters = parameters - job_data = jobs_data.get(job.name,"none") + job_data = jobs_data.get(job.name, "none") try: if job_data != "none": job.wrapper_type = job_data[12] @@ -254,7 +266,9 @@ class JobList(object): str(run_only_members))) old_job_list = [job for job in self._job_list] self._job_list = [ - job for job in old_job_list if job.member is None or job.member in run_only_members or job.status not in [Status.WAITING, Status.READY]] + job for job in old_job_list if + job.member is None or job.member in run_only_members or job.status not in [Status.WAITING, + Status.READY]] for job in self._job_list: for jobp in job.parents: if jobp in self._job_list: @@ -268,22 +282,24 @@ class JobList(object): for wrapper_section in wrapper_jobs: try: if wrapper_jobs[wrapper_section] is not None and len(str(wrapper_jobs[wrapper_section])) > 0: - self._ordered_jobs_by_date_member[wrapper_section] = self._create_sorted_dict_jobs(wrapper_jobs[wrapper_section]) + self._ordered_jobs_by_date_member[wrapper_section] = self._create_sorted_dict_jobs( + wrapper_jobs[wrapper_section]) else: self._ordered_jobs_by_date_member[wrapper_section] = {} except BaseException as e: - raise AutosubmitCritical("Some section jobs of the wrapper:{0} are not in the current job_list defined in jobs.conf".format(wrapper_section),7014,str(e)) - + raise AutosubmitCritical( + "Some section jobs of the wrapper:{0} are not in the current job_list defined in jobs.conf".format( + wrapper_section), 7014, str(e)) - def _add_dependencies(self,date_list, member_list, chunk_list, dic_jobs, graph, option="DEPENDENCIES"): - jobs_data = dic_jobs._jobs_data.get("JOBS",{}) + def _add_dependencies(self, date_list, member_list, chunk_list, dic_jobs, graph, option="DEPENDENCIES"): + jobs_data = dic_jobs._jobs_data.get("JOBS", {}) for job_section in jobs_data.keys(): Log.debug("Adding dependencies for {0} jobs".format(job_section)) # If it does not have dependencies, do nothing if not (job_section, option): continue - dependencies_keys = jobs_data[job_section].get(option,{}) + dependencies_keys = jobs_data[job_section].get(option, {}) if type(dependencies_keys) is str: if "," in dependencies_keys: dependencies_list = dependencies_keys.split(",") @@ -303,10 +319,9 @@ class JobList(object): for i in range(num_jobs): _job = job[i] if num_jobs > 1 else job self._manage_job_dependencies(dic_jobs, _job, date_list, member_list, chunk_list, dependencies_keys, - dependencies, graph) + dependencies, graph) pass - @staticmethod def _manage_dependencies(dependencies_keys, dic_jobs, job_section): parameters = dic_jobs._jobs_data["JOBS"] @@ -334,7 +349,7 @@ class JobList(object): distance = int(key_split[1]) if '[' in section: - #Todo check what is this because we never enter this + # Todo check what is this because we never enter this try: section_name = section[0:section.find("[")] splits_section = int( @@ -344,13 +359,14 @@ class JobList(object): section = section_name except Exception as e: pass - if parameters.get(section,None) is None: + if parameters.get(section, None) is None: Log.printlog("WARNING: SECTION {0} is not defined in jobs.conf".format(section)) continue - #raise AutosubmitCritical("Section:{0} doesn't exists.".format(section),7014) + # raise AutosubmitCritical("Section:{0} doesn't exists.".format(section),7014) dependency_running_type = str(parameters[section].get('RUNNING', 'once')).lower() delay = int(parameters[section].get('DELAY', -1)) - dependency = Dependency(section, distance, dependency_running_type, sign, delay, splits,relationships=dependencies_keys[key]) + dependency = Dependency(section, distance, dependency_running_type, sign, delay, splits, + relationships=dependencies_keys[key]) dependencies[key] = dependency return dependencies @@ -371,13 +387,14 @@ class JobList(object): return splits @staticmethod - def _apply_filter(parent_value,filter_value,associative_list,filter_type="dates"): + def _apply_filter(parent_value, filter_value, associative_list, filter_type="dates"): """ Check if the current_job_value is included in the filter_value :param parent_value: :param filter_value: filter :param associative_list: dates, members, chunks. - :param is_chunk: True if the filter_value is a chunk. + :param filter_type: dates, members, chunks. + :return: boolean """ to_filter = [] @@ -408,11 +425,11 @@ class JobList(object): start = start_end[0].strip("[]") end = start_end[1].strip("[]") del start_end - if filter_type not in ["chunks", "splits"]: # chunk directly + if filter_type not in ["chunks", "splits"]: # chunk directly for value in range(int(start), int(end) + 1): to_filter.append(value) - else: # index - for value in range(int(start+1), int(end) + 1): + else: # index + for value in range(int(start + 1), int(end) + 1): to_filter.append(value) else: to_filter.append(filter_value) @@ -477,7 +494,7 @@ class JobList(object): # [N:M] return slice(int(value_to_check.split(":")[0]), int(value_to_check.split(":")[1])) elif value_to_check.count(":") == 2: -# range with step + # range with step if value_to_check[0] == ":": # [::M] return slice(None, None, int(value_to_check[2:])) @@ -494,9 +511,6 @@ class JobList(object): # value return value_to_check - - - @staticmethod def _check_relationship(relationships, level_to_check, value_to_check): """ @@ -511,7 +525,8 @@ class JobList(object): status = relationship.pop("STATUS", relationships.get("STATUS", None)) from_step = relationship.pop("FROM_STEP", relationships.get("FROM_STEP", None)) for filter_range, filter_data in relationship.items(): - if not value_to_check or str(value_to_check).upper() in str(JobList._parse_filter_to_check(filter_range)).upper(): + if not value_to_check or str(value_to_check).upper() in str( + JobList._parse_filter_to_check(filter_range)).upper(): if not filter_data.get("STATUS", None): filter_data["STATUS"] = status if not filter_data.get("FROM_STEP", None): @@ -530,30 +545,30 @@ class JobList(object): :param current_job: Current job to check. :return: filters_to_apply """ - + filters_to_apply = JobList._check_relationship(relationships, "DATES_FROM", date2str(current_job.date)) # there could be multiple filters that apply... per example # Current task date is 20020201, and member is fc2 # Dummy example, not specially usefull in a real case - #DATES_FROM: - #all: - #MEMBERS_FROM: - #ALL: ... - #CHUNKS_FROM: - #ALL: ... - #20020201: - #MEMBERS_FROM: - #fc2: - #DATES_TO: "20020201" - #MEMBERS_TO: "fc2" - #CHUNKS_TO: "ALL" - #SPLITS_FROM: - #ALL: - #SPLITS_TO: "1" + # DATES_FROM: + # all: + # MEMBERS_FROM: + # ALL: ... + # CHUNKS_FROM: + # ALL: ... + # 20020201: + # MEMBERS_FROM: + # fc2: + # DATES_TO: "20020201" + # MEMBERS_TO: "fc2" + # CHUNKS_TO: "ALL" + # SPLITS_FROM: + # ALL: + # SPLITS_TO: "1" # this "for" iterates for ALL and fc2 as current task is selected in both filters # The dict in this step is: # [{MEMBERS_FROM{..},CHUNKS_FROM{...}},{MEMBERS_FROM{..},SPLITS_FROM{...}}] - for i,filter in enumerate(filters_to_apply): + for i, filter in enumerate(filters_to_apply): # {MEMBERS_FROM{..},CHUNKS_FROM{...}} I want too look ALL filters not only one, but I want to go recursivily until get the _TO filter # This is not an if_else, because the current level ( dates ) could have two different filters. # Second case commented: ( date_from 20020201 ) @@ -607,9 +622,9 @@ class JobList(object): :param current_job: Current job to check. :return: filters_to_apply """ - + filters_to_apply = JobList._check_relationship(relationships, "CHUNKS_FROM", current_job.chunk) - for i,filter in enumerate(filters_to_apply): + for i, filter in enumerate(filters_to_apply): if "SPLITS_FROM" in filter: filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM"))}, current_job) if len(filters_to_apply_s) > 0: @@ -632,7 +647,7 @@ class JobList(object): return filters_to_apply @staticmethod - def _unify_to_filter(unified_filter,filter_to,filter_type): + def _unify_to_filter(unified_filter, filter_to, filter_type): """ Unify filter_to filters into a single dictionary :param unified_filter: Single dictionary with all filters_to @@ -655,8 +670,9 @@ class JobList(object): if "?" not in element: element += data unified_filter[filter_type].add(element) + @staticmethod - def _normalize_to_filters(filter_to,filter_type): + def _normalize_to_filters(filter_to, filter_type): """ Normalize filter_to filters to a single string or "all" :param filter_to: Unified filter_to dictionary @@ -685,19 +701,19 @@ class JobList(object): if "FROM_STEP" not in unified_filter and filter_to.get("FROM_STEP", None): unified_filter["FROM_STEP"] = filter_to["FROM_STEP"] if len(filter_to) > 0: - JobList._unify_to_filter(unified_filter,filter_to,"DATES_TO") - JobList._unify_to_filter(unified_filter,filter_to,"MEMBERS_TO") - JobList._unify_to_filter(unified_filter,filter_to,"CHUNKS_TO") - JobList._unify_to_filter(unified_filter,filter_to,"SPLITS_TO") - - JobList._normalize_to_filters(unified_filter,"DATES_TO") - JobList._normalize_to_filters(unified_filter,"MEMBERS_TO") - JobList._normalize_to_filters(unified_filter,"CHUNKS_TO") - JobList._normalize_to_filters(unified_filter,"SPLITS_TO") + JobList._unify_to_filter(unified_filter, filter_to, "DATES_TO") + JobList._unify_to_filter(unified_filter, filter_to, "MEMBERS_TO") + JobList._unify_to_filter(unified_filter, filter_to, "CHUNKS_TO") + JobList._unify_to_filter(unified_filter, filter_to, "SPLITS_TO") + + JobList._normalize_to_filters(unified_filter, "DATES_TO") + JobList._normalize_to_filters(unified_filter, "MEMBERS_TO") + JobList._normalize_to_filters(unified_filter, "CHUNKS_TO") + JobList._normalize_to_filters(unified_filter, "SPLITS_TO") return unified_filter @staticmethod - def _filter_current_job(current_job,relationships): + def _filter_current_job(current_job, relationships): ''' This function will filter the current job based on the relationships given :param current_job: Current job to filter :param relationships: Relationships to apply @@ -741,10 +757,8 @@ class JobList(object): filters_to_apply = relationships return filters_to_apply - - @staticmethod - def _valid_parent(parent,member_list,date_list,chunk_list,is_a_natural_relation,filter_): + def _valid_parent(parent, member_list, date_list, chunk_list, is_a_natural_relation, filter_): ''' Check if the parent is valid for the current job :param parent: job to check @@ -752,17 +766,16 @@ class JobList(object): :param date_list: list of dates :param chunk_list: list of chunks :param is_a_natural_relation: if the relation is natural or not - :param filters_to_apply: filters to apply :return: True if the parent is valid, False otherwise ''' - #check if current_parent is listed on dependency.relationships + # check if current_parent is listed on dependency.relationships associative_list = {} associative_list["dates"] = date_list associative_list["members"] = member_list associative_list["chunks"] = chunk_list if parent.splits is not None: - associative_list["splits"] = [ str(split) for split in range(1,int(parent.splits)+1) ] + associative_list["splits"] = [str(split) for split in range(1, int(parent.splits) + 1)] else: associative_list["splits"] = None dates_to = str(filter_.get("DATES_TO", "natural")).lower() @@ -788,15 +801,15 @@ class JobList(object): associative_list["splits"] = [parent.split] if parent.split is not None else parent.splits parsed_parent_date = date2str(parent.date) if parent.date is not None else None # Apply all filters to look if this parent is an appropriated candidate for the current_job - valid_dates = JobList._apply_filter(parsed_parent_date, dates_to, associative_list["dates"], "dates") + valid_dates = JobList._apply_filter(parsed_parent_date, dates_to, associative_list["dates"], "dates") valid_members = JobList._apply_filter(parent.member, members_to, associative_list["members"], "members") - valid_chunks = JobList._apply_filter(parent.chunk, chunks_to, associative_list["chunks"], "chunks") - valid_splits = JobList._apply_filter(parent.split, splits_to, associative_list["splits"], "splits") + valid_chunks = JobList._apply_filter(parent.chunk, chunks_to, associative_list["chunks"], "chunks") + valid_splits = JobList._apply_filter(parent.split, splits_to, associative_list["splits"], "splits") if valid_dates and valid_members and valid_chunks and valid_splits: return True return False - def _add_edge_info(self,job,special_status): + def _add_edge_info(self, job, special_status): """ Special relations to be check in the update_list method :param job: Current job @@ -810,9 +823,8 @@ class JobList(object): self.jobs_edges["ALL"] = set() self.jobs_edges["ALL"].add(job) - - - def _manage_job_dependencies(self,dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys, dependencies, + def _manage_job_dependencies(self, dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys, + dependencies, graph): ''' Manage the dependencies of a job @@ -831,9 +843,10 @@ class JobList(object): parsed_date_list.append(date2str(dat)) special_conditions = dict() for key in dependencies_keys: - dependency = dependencies.get(key,None) + dependency = dependencies.get(key, None) if dependency is None: - Log.printlog("WARNING: SECTION {0} is not defined in jobs.conf. Dependency skipped".format(key),Log.WARNING) + Log.printlog("WARNING: SECTION {0} is not defined in jobs.conf. Dependency skipped".format(key), + Log.WARNING) continue skip, (chunk, member, date) = JobList._calculate_dependency_metadata(job.chunk, chunk_list, job.member, member_list, @@ -847,8 +860,10 @@ class JobList(object): natural_jobs = dic_jobs.get_jobs(dependency.section, date, member, chunk) all_parents = list(set(other_parents + parents_jobs)) # Get dates_to, members_to, chunks_to of the deepest level of the relationship. - filters_to_apply = JobList._filter_current_job(job,copy.deepcopy(dependency.relationships)) - if "?" in [filters_to_apply.get("SPLITS_TO",""),filters_to_apply.get("DATES_TO",""),filters_to_apply.get("MEMBERS_TO",""),filters_to_apply.get("CHUNKS_TO","")]: + filters_to_apply = JobList._filter_current_job(job, copy.deepcopy(dependency.relationships)) + if "?" in filters_to_apply.get("SPLITS_TO", "") or "?" in filters_to_apply.get("DATES_TO", + "") or "?" in filters_to_apply.get( + "MEMBERS_TO", "") or "?" in filters_to_apply.get("CHUNKS_TO", ""): only_marked_status = True else: only_marked_status = False @@ -859,31 +874,37 @@ class JobList(object): if parent.name == job.name: continue # Check if it is a natural relation. The only difference is that a chunk can depend on a chunks <= than the current chunk - if parent in natural_jobs and (job.chunk is None or parent.chunk is None or parent.chunk <= job.chunk ): + if parent in natural_jobs and (job.chunk is None or parent.chunk is None or parent.chunk <= job.chunk): natural_relationship = True else: natural_relationship = False # Check if the current parent is a valid parent based on the dependencies set on expdef.conf # If the parent is valid, add it to the graph - if JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship,filters_to_apply): + if JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship, + filters_to_apply): job.add_parent(parent) self._add_edge(graph, job, parent) # Could be more variables in the future # Do parse checkpoint - if special_conditions.get("STATUS",None): + if special_conditions.get("STATUS", None): if only_marked_status: - if str(job.split)+"?" in filters_to_apply.get("SPLITS_TO","") or str(job.chunk)+"?" in filters_to_apply.get("CHUNKS_TO","") or str(job.member)+"?" in filters_to_apply.get("MEMBERS_TO","") or str(job.date)+"?" in filters_to_apply.get("DATES_TO",""): + if str(job.split) + "?" in filters_to_apply.get("SPLITS_TO", "") or str( + job.chunk) + "?" in filters_to_apply.get("CHUNKS_TO", "") or str( + job.member) + "?" in filters_to_apply.get("MEMBERS_TO", "") or str( + job.date) + "?" in filters_to_apply.get("DATES_TO", ""): selected = True else: selected = False else: selected = True if selected: - job.max_checkpoint_step = int(special_conditions.get("FROM_STEP", 0)) if int(special_conditions.get("FROM_STEP", - 0)) > job.max_checkpoint_step else job.max_checkpoint_step + if special_conditions.get("FROM_STEP", None): + job.max_checkpoint_step = int(special_conditions.get("FROM_STEP", 0)) if int( + special_conditions.get("FROM_STEP", + 0)) > job.max_checkpoint_step else job.max_checkpoint_step self._add_edge_info(job, special_conditions["STATUS"]) - job.add_edge_info(parent,special_conditions) + job.add_edge_info(parent, special_conditions) JobList.handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member, member_list, dependency.section, graph, other_parents) @@ -939,7 +960,7 @@ class JobList(object): @staticmethod def handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member, member_list, - section_name, graph,visited_parents): + section_name, graph, visited_parents): if job.wait and job.frequency > 1: if job.chunk is not None and len(str(job.chunk)) > 0: max_distance = (chunk_list.index(chunk) + 1) % job.frequency @@ -982,9 +1003,10 @@ class JobList(object): parent = parents[i] if isinstance(parents, list) else parents graph.add_edge(parent.name, job.name) pass + @staticmethod def _create_jobs(dic_jobs, priority, default_job_type, jobs_data=dict()): - for section in dic_jobs._jobs_data.get("JOBS",{}).keys(): + for section in dic_jobs._jobs_data.get("JOBS", {}).keys(): Log.debug("Creating {0} jobs".format(section)) dic_jobs.read_section(section, priority, default_job_type, jobs_data) priority += 1 @@ -1001,7 +1023,10 @@ class JobList(object): :return: Sorted Dictionary of List that represents the jobs included in the wrapping process. \n :rtype: Dictionary Key: date, Value: (Dictionary Key: Member, Value: List of jobs that belong to the date, member, and are ordered by chunk number if it is a chunk job otherwise num_chunks from JOB TYPE (section) """ + # Dictionary Key: date, Value: (Dictionary Key: Member, Value: List) + job = None + dict_jobs = dict() for date in self._date_list: dict_jobs[date] = dict() @@ -1009,7 +1034,6 @@ class JobList(object): dict_jobs[date][member] = list() num_chunks = len(self._chunk_list) - sections_running_type_map = dict() if wrapper_jobs is not None and len(str(wrapper_jobs)) > 0: if type(wrapper_jobs) is not list: @@ -1019,13 +1043,12 @@ class JobList(object): char = " " wrapper_jobs = wrapper_jobs.split(char) - for section in wrapper_jobs: # RUNNING = once, as default. This value comes from jobs_.yml try: sections_running_type_map[section] = str(self.jobs_data[section].get("RUNNING", 'once')) except BaseException as e: - raise AutosubmitCritical("Key {0} doesn't exists.".format(section),7014,str(e)) + raise AutosubmitCritical("Key {0} doesn't exists.".format(section), 7014, str(e)) # Select only relevant jobs, those belonging to the sections defined in the wrapper @@ -1043,20 +1066,19 @@ class JobList(object): for member in self._member_list: # Filter list of fake jobs according to date and member, result not sorted at this point sorted_jobs_list = list(filter(lambda job: job.name.split("_")[1] == str_date and - job.name.split("_")[2] == member, filtered_jobs_fake_date_member)) - #sorted_jobs_list = [job for job in filtered_jobs_fake_date_member if job.name.split("_")[1] == str_date and + job.name.split("_")[2] == member, + filtered_jobs_fake_date_member)) + # sorted_jobs_list = [job for job in filtered_jobs_fake_date_member if job.name.split("_")[1] == str_date and # job.name.split("_")[2] == member] - #There can be no jobs for this member when select chunk/member is enabled + # There can be no jobs for this member when select chunk/member is enabled if not sorted_jobs_list or len(sorted_jobs_list) == 0: continue - previous_job = sorted_jobs_list[0] # get RUNNING for this section section_running_type = sections_running_type_map[previous_job.section] - jobs_to_sort = [previous_job] previous_section_running_type = None # Index starts at 1 because 0 has been taken in a previous step @@ -1069,14 +1091,16 @@ class JobList(object): previous_section_running_type = section_running_type section_running_type = sections_running_type_map[job.section] # Test if RUNNING is different between sections, or if we have reached the last item in sorted_jobs_list - if (previous_section_running_type is not None and previous_section_running_type != section_running_type) \ + if ( + previous_section_running_type is not None and previous_section_running_type != section_running_type) \ or index == len(sorted_jobs_list): # Sorting by date, member, chunk number if it is a chunk job otherwise num_chunks from JOB TYPE (section) # Important to note that the only differentiating factor would be chunk OR num_chunks jobs_to_sort = sorted(jobs_to_sort, key=lambda k: (k.name.split('_')[1], (k.name.split('_')[2]), (int(k.name.split('_')[3]) - if len(k.name.split('_')) == 5 else num_chunks + 1))) + if len(k.name.split( + '_')) == 5 else num_chunks + 1))) # Bringing back original job if identified for idx in range(0, len(jobs_to_sort)): @@ -1089,7 +1113,7 @@ class JobList(object): # By adding to the result at this step, only those with the same RUNNING have been added. dict_jobs[date][member] += jobs_to_sort jobs_to_sort = [] - if len(sorted_jobs_list) > 1 : + if len(sorted_jobs_list) > 1: jobs_to_sort.append(job) previous_job = job @@ -1121,7 +1145,7 @@ class JobList(object): fake_job = copy.deepcopy(job) # Use previous values to modify name of fake job fake_job.name = fake_job.name.split('_', 1)[0] + "_" + self._get_date(date) + "_" \ - + member + "_" + fake_job.name.split("_", 1)[1] + + member + "_" + fake_job.name.split("_", 1)[1] # Filling list of fake jobs, only difference is the name filtered_jobs_fake_date_member.append(fake_job) # Mapping fake jobs to original ones @@ -1211,7 +1235,8 @@ class JobList(object): def copy_ordered_jobs_by_date_member(self): pass - def get_ordered_jobs_by_date_member(self,section): + + def get_ordered_jobs_by_date_member(self, section): """ Get the dictionary of jobs ordered according to wrapper's expression divided by date and member @@ -1250,7 +1275,8 @@ class JobList(object): :return: completed jobs :rtype: list """ - uncompleted_jobs = [job for job in self._job_list if (platform is None or job.platform.name == platform.name) and + uncompleted_jobs = [job for job in self._job_list if + (platform is None or job.platform.name == platform.name) and job.status != Status.COMPLETED] if wrapper: @@ -1362,7 +1388,8 @@ class JobList(object): :rtype: list """ unsubmitted = [job for job in self._job_list if (platform is None or job.platform.name == platform.name) and - (job.status != Status.SUBMITTED and job.status != Status.QUEUING and job.status == Status.RUNNING and job.status == Status.COMPLETED)] + ( + job.status != Status.SUBMITTED and job.status != Status.QUEUING and job.status == Status.RUNNING and job.status == Status.COMPLETED)] if wrapper: return [job for job in unsubmitted if job.packed is False] @@ -1386,7 +1413,7 @@ class JobList(object): else: return all_jobs - def get_job_names(self,lower_case=False): + def get_job_names(self, lower_case=False): """ Returns a list of all job names :param: lower_case: if true, returns lower case job names @@ -1407,21 +1434,24 @@ class JobList(object): def update_two_step_jobs(self): prev_jobs_to_run_first = self.jobs_to_run_first if len(self.jobs_to_run_first) > 0: - self.jobs_to_run_first = [ job for job in self.jobs_to_run_first if job.status != Status.COMPLETED ] + self.jobs_to_run_first = [job for job in self.jobs_to_run_first if job.status != Status.COMPLETED] keep_running = False for job in self.jobs_to_run_first: - running_parents = [parent for parent in job.parents if parent.status != Status.WAITING and parent.status != Status.FAILED ] #job is parent of itself + running_parents = [parent for parent in job.parents if + parent.status != Status.WAITING and parent.status != Status.FAILED] # job is parent of itself if len(running_parents) == len(job.parents): keep_running = True if len(self.jobs_to_run_first) > 0 and keep_running is False: - raise AutosubmitCritical("No more jobs to run first, there were still pending jobs but they're unable to run without their parents or there are failed jobs.",7014) + raise AutosubmitCritical( + "No more jobs to run first, there were still pending jobs but they're unable to run without their parents or there are failed jobs.", + 7014) - def parse_jobs_by_filter(self, unparsed_jobs,two_step_start = True): + def parse_jobs_by_filter(self, unparsed_jobs, two_step_start=True): jobs_to_run_first = list() - select_jobs_by_name = "" #job_name - select_all_jobs_by_section = "" # all + select_jobs_by_name = "" # job_name + select_all_jobs_by_section = "" # all filter_jobs_by_section = "" # Select, chunk / member - if "&" in unparsed_jobs: # If there are explicit jobs add them + if "&" in unparsed_jobs: # If there are explicit jobs add them jobs_to_check = unparsed_jobs.split("&") select_jobs_by_name = jobs_to_check[0] unparsed_jobs = jobs_to_check[1] @@ -1438,20 +1468,26 @@ class JobList(object): filter_jobs_by_section = aux[1] if two_step_start: try: - self.jobs_to_run_first = self.get_job_related(select_jobs_by_name=select_jobs_by_name,select_all_jobs_by_section=select_all_jobs_by_section,filter_jobs_by_section=filter_jobs_by_section) + self.jobs_to_run_first = self.get_job_related(select_jobs_by_name=select_jobs_by_name, + select_all_jobs_by_section=select_all_jobs_by_section, + filter_jobs_by_section=filter_jobs_by_section) except Exception as e: - raise AutosubmitCritical("Check the {0} format.\nFirst filter is optional ends with '&'.\nSecond filter ends with ';'.\nThird filter must contain '['. ".format(unparsed_jobs)) + raise AutosubmitCritical( + "Check the {0} format.\nFirst filter is optional ends with '&'.\nSecond filter ends with ';'.\nThird filter must contain '['. ".format( + unparsed_jobs)) else: try: self.rerun_job_list = self.get_job_related(select_jobs_by_name=select_jobs_by_name, - select_all_jobs_by_section=select_all_jobs_by_section, - filter_jobs_by_section=filter_jobs_by_section,two_step_start=two_step_start) + select_all_jobs_by_section=select_all_jobs_by_section, + filter_jobs_by_section=filter_jobs_by_section, + two_step_start=two_step_start) except Exception as e: raise AutosubmitCritical( "Check the {0} format.\nFirst filter is optional ends with '&'.\nSecond filter ends with ';'.\nThird filter must contain '['. ".format( unparsed_jobs)) - def get_job_related(self, select_jobs_by_name="",select_all_jobs_by_section="",filter_jobs_by_section="",two_step_start=True): + def get_job_related(self, select_jobs_by_name="", select_all_jobs_by_section="", filter_jobs_by_section="", + two_step_start=True): """ :param two_step_start: :param select_jobs_by_name: job name @@ -1465,25 +1501,29 @@ class JobList(object): jobs_date = [] # First Filter {select job by name} if select_jobs_by_name != "": - jobs_by_name = [ job for job in self._job_list if re.search("(^|[^0-9a-z_])"+job.name.lower()+"([^a-z0-9_]|$)",select_jobs_by_name.lower()) is not None ] - jobs_by_name_no_expid = [job for job in self._job_list if - re.search("(^|[^0-9a-z_])" + job.name.lower()[5:] + "([^a-z0-9_]|$)", + jobs_by_name = [job for job in self._job_list if + re.search("(^|[^0-9a-z_])" + job.name.lower() + "([^a-z0-9_]|$)", select_jobs_by_name.lower()) is not None] + jobs_by_name_no_expid = [job for job in self._job_list if + re.search("(^|[^0-9a-z_])" + job.name.lower()[5:] + "([^a-z0-9_]|$)", + select_jobs_by_name.lower()) is not None] ultimate_jobs_list.extend(jobs_by_name) ultimate_jobs_list.extend(jobs_by_name_no_expid) # Second Filter { select all } if select_all_jobs_by_section != "": - all_jobs_by_section = [ job for job in self._job_list if re.search("(^|[^0-9a-z_])"+job.section.upper()+"([^a-z0-9_]|$)",select_all_jobs_by_section.upper()) is not None ] + all_jobs_by_section = [job for job in self._job_list if + re.search("(^|[^0-9a-z_])" + job.section.upper() + "([^a-z0-9_]|$)", + select_all_jobs_by_section.upper()) is not None] ultimate_jobs_list.extend(all_jobs_by_section) # Third Filter N section { date , member? , chunk?} # Section[date[member][chunk]] # filter_jobs_by_section="SIM[20[C:000][M:1]],DA[20 21[M:000 001][C:1]]" if filter_jobs_by_section != "": - section_name="" - section_dates="" - section_chunks="" - section_members="" + section_name = "" + section_dates = "" + section_chunks = "" + section_members = "" jobs_final = list() for complete_filter_by_section in filter_jobs_by_section.split(','): section_list = complete_filter_by_section.split('[') @@ -1499,20 +1539,27 @@ class JobList(object): elif 'm' in section_list[3].lower(): section_members = section_list[3].strip('mM:[]') - if section_name != "": jobs_filtered = [job for job in self._job_list if - re.search("(^|[^0-9a-z_])" + job.section.upper() + "([^a-z0-9_]|$)", - section_name.upper()) is not None] + re.search("(^|[^0-9a-z_])" + job.section.upper() + "([^a-z0-9_]|$)", + section_name.upper()) is not None] if section_dates != "": - jobs_date = [ job for job in jobs_filtered if re.search("(^|[^0-9a-z_])" + date2str(job.date, job.date_format) + "([^a-z0-9_]|$)", section_dates.lower()) is not None or job.date is None ] + jobs_date = [job for job in jobs_filtered if + re.search("(^|[^0-9a-z_])" + date2str(job.date, job.date_format) + "([^a-z0-9_]|$)", + section_dates.lower()) is not None or job.date is None] if section_chunks != "" or section_members != "": - jobs_final = [job for job in jobs_date if ( section_chunks == "" or re.search("(^|[^0-9a-z_])" + str(job.chunk) + "([^a-z0-9_]|$)",section_chunks) is not None ) and ( section_members == "" or re.search("(^|[^0-9a-z_])" + str(job.member) + "([^a-z0-9_]|$)",section_members.lower()) is not None ) ] + jobs_final = [job for job in jobs_date if ( + section_chunks == "" or re.search("(^|[^0-9a-z_])" + str(job.chunk) + "([^a-z0-9_]|$)", + section_chunks) is not None) and ( + section_members == "" or re.search( + "(^|[^0-9a-z_])" + str(job.member) + "([^a-z0-9_]|$)", + section_members.lower()) is not None)] ultimate_jobs_list.extend(jobs_final) # Duplicates out ultimate_jobs_list = list(set(ultimate_jobs_list)) - Log.debug("List of jobs filtered by TWO_STEP_START parameter:\n{0}".format([job.name for job in ultimate_jobs_list])) + Log.debug( + "List of jobs filtered by TWO_STEP_START parameter:\n{0}".format([job.name for job in ultimate_jobs_list])) return ultimate_jobs_list def get_logs(self): @@ -1550,7 +1597,8 @@ class JobList(object): :return: ready jobs :rtype: list """ - ready = [job for job in self._job_list if ( platform is None or platform == "" or job.platform.name == platform.name ) and + ready = [job for job in self._job_list if + (platform is None or platform == "" or job.platform.name == platform.name) and job.status == Status.READY and job.hold is hold] if wrapper: @@ -1570,6 +1618,7 @@ class JobList(object): prepared = [job for job in self._job_list if (platform is None or job.platform.name == platform.name) and job.status == Status.PREPARED] return prepared + def get_delayed(self, platform=None): """ Returns a list of delayed jobs @@ -1580,8 +1629,9 @@ class JobList(object): :rtype: list """ delayed = [job for job in self._job_list if (platform is None or job.platform.name == platform.name) and - job.status == Status.DELAYED] + job.status == Status.DELAYED] return delayed + def get_skipped(self, platform=None): """ Returns a list of skipped jobs @@ -1622,7 +1672,7 @@ class JobList(object): """ waiting_jobs = [job for job in self._job_list if ( - job.platform.type == platform_type and job.status == Status.WAITING)] + job.platform.type == platform_type and job.status == Status.WAITING)] return waiting_jobs def get_held_jobs(self, platform=None): @@ -1734,13 +1784,15 @@ class JobList(object): """ active = self.get_in_queue(platform) + self.get_ready( - platform=platform, hold=True) + self.get_ready(platform=platform, hold=False) + self.get_delayed(platform=platform) + platform=platform, hold=True) + self.get_ready(platform=platform, hold=False) + self.get_delayed( + platform=platform) tmp = [job for job in active if job.hold and not (job.status == - Status.SUBMITTED or job.status == Status.READY or job.status == Status.DELAYED) ] + Status.SUBMITTED or job.status == Status.READY or job.status == Status.DELAYED)] if len(tmp) == len(active): # IF only held jobs left without dependencies satisfied if len(tmp) != 0 and len(active) != 0: raise AutosubmitCritical( - "Only Held Jobs active. Exiting Autosubmit (TIP: This can happen if suspended or/and Failed jobs are found on the workflow)", 7066) + "Only Held Jobs active. Exiting Autosubmit (TIP: This can happen if suspended or/and Failed jobs are found on the workflow)", + 7066) active = [] return active @@ -1756,6 +1808,7 @@ class JobList(object): for job in self._job_list: if job.name == name: return job + def get_jobs_by_section(self, section_list): """ Returns the job that its name matches parameter section @@ -1786,7 +1839,7 @@ class JobList(object): def get_in_ready_grouped_id(self, platform): jobs = [] [jobs.append(job) for job in jobs if ( - platform is None or job.platform.name is platform.name)] + platform is None or job.platform.name is platform.name)] jobs_by_id = dict() for job in jobs: @@ -1889,15 +1942,15 @@ class JobList(object): try: self._persistence.save(self._persistence_path, - self._persistence_file, self._job_list if self.run_members is None or job_list is None else job_list) + self._persistence_file, + self._job_list if self.run_members is None or job_list is None else job_list) pass except BaseException as e: - raise AutosubmitError(str(e),6040,"Failure while saving the job_list") + raise AutosubmitError(str(e), 6040, "Failure while saving the job_list") except AutosubmitError as e: raise except BaseException as e: - raise AutosubmitError(str(e),6040,"Unknown failure while saving the job_list") - + raise AutosubmitError(str(e), 6040, "Unknown failure while saving the job_list") def backup_save(self): """ @@ -1911,8 +1964,8 @@ class JobList(object): exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, self.expid) tmp_path = os.path.join(exp_path, BasicConfig.LOCAL_TMP_DIR) aslogs_path = os.path.join(tmp_path, BasicConfig.LOCAL_ASLOG_DIR) - Log.reset_status_file(os.path.join(aslogs_path,"jobs_active_status.log"),"status") - Log.reset_status_file(os.path.join(aslogs_path,"jobs_failed_status.log"),"status_failed") + Log.reset_status_file(os.path.join(aslogs_path, "jobs_active_status.log"), "status") + Log.reset_status_file(os.path.join(aslogs_path, "jobs_failed_status.log"), "status_failed") job_list = self.get_completed()[-5:] + self.get_in_queue() failed_job_list = self.get_failed() if len(job_list) > 0: @@ -1920,7 +1973,7 @@ class JobList(object): "Job Id", "Job Status", "Job Platform", "Job Queue") if len(failed_job_list) > 0: Log.status_failed("\n{0:<35}{1:<15}{2:<15}{3:<20}{4:<15}", "Job Name", - "Job Id", "Job Status", "Job Platform", "Job Queue") + "Job Id", "Job Status", "Job Platform", "Job Queue") for job in job_list: if len(job.queue) > 0 and str(job.platform.queue).lower() != "none": queue = job.queue @@ -1961,8 +2014,10 @@ class JobList(object): "_" + output_date)) def get_skippable_jobs(self, jobs_in_wrapper): - job_list_skip = [job for job in self.get_job_list() if job.skippable == "true" and (job.status == Status.QUEUING or job.status == - Status.RUNNING or job.status == Status.COMPLETED or job.status == Status.READY) and jobs_in_wrapper.find(job.section) == -1] + job_list_skip = [job for job in self.get_job_list() if + job.skippable == "true" and (job.status == Status.QUEUING or job.status == + Status.RUNNING or job.status == Status.COMPLETED or job.status == Status.READY) and jobs_in_wrapper.find( + job.section) == -1] skip_by_section = dict() for job in job_list_skip: if job.section not in skip_by_section: @@ -1998,15 +2053,16 @@ class JobList(object): continue for job in sorted_job_list: if status in ["RUNNING", "FAILED"]: - if job.platform.connected: # This will be true only when used under setstatus/run + if job.platform.connected: # This will be true only when used under setstatus/run job.get_checkpoint_files() - for parent in job.edge_info[status]: + for parent in job.edge_info[status].values(): if parent[0].status == Status.WAITING: - if status in ["RUNNING", "FAILED"] and int(parent[1]) >= job.current_checkpoint_step: + if status in ["RUNNING", "FAILED"] and parent[1] and int(parent[1]) >= job.current_checkpoint_step: continue else: jobs_to_check.append(parent[0]) return jobs_to_check + def update_list(self, as_conf, store_change=True, fromSetStatus=False, submitter=None, first_time=False): # type: (AutosubmitConfig, bool, bool, object, bool) -> bool """ @@ -2081,7 +2137,8 @@ class JobList(object): # Check checkpoint jobs, the status can be Any for job in self.check_special_status(): # Check if all jobs fullfill the conditions to a job be ready - tmp = [parent for parent in job.parents if parent.status == Status.COMPLETED or parent in self.jobs_edges["ALL"] ] + tmp = [parent for parent in job.parents if + parent.status == Status.COMPLETED or parent in self.jobs_edges["ALL"]] if len(tmp) == len(job.parents): job.status = Status.READY job.id = None @@ -2118,9 +2175,12 @@ class JobList(object): if datetime.datetime.now() >= job.delay_end: job.status = Status.READY for job in self.get_waiting(): - tmp = [parent for parent in job.parents if parent.status == Status.COMPLETED or parent.status == Status.SKIPPED] - tmp2 = [parent for parent in job.parents if parent.status == Status.COMPLETED or parent.status == Status.SKIPPED or parent.status == Status.FAILED] - tmp3 = [parent for parent in job.parents if parent.status == Status.SKIPPED or parent.status == Status.FAILED] + tmp = [parent for parent in job.parents if + parent.status == Status.COMPLETED or parent.status == Status.SKIPPED] + tmp2 = [parent for parent in job.parents if + parent.status == Status.COMPLETED or parent.status == Status.SKIPPED or parent.status == Status.FAILED] + tmp3 = [parent for parent in job.parents if + parent.status == Status.SKIPPED or parent.status == Status.FAILED] failed_ones = [parent for parent in job.parents if parent.status == Status.FAILED] if job.parents is None or len(tmp) == len(job.parents): job.status = Status.READY @@ -2138,14 +2198,15 @@ class JobList(object): if parent.name in job.edge_info and job.edge_info[parent.name].get('optional', False): weak_dependencies_failure = True elif parent.section in job.dependencies: - if parent.status not in [Status.COMPLETED,Status.SKIPPED]: + if parent.status not in [Status.COMPLETED, Status.SKIPPED]: strong_dependencies_failure = True break if not strong_dependencies_failure and weak_dependencies_failure: job.status = Status.READY job.hold = False Log.debug( - "Setting job: {0} status to: READY (conditional jobs are completed/failed)...".format(job.name)) + "Setting job: {0} status to: READY (conditional jobs are completed/failed)...".format( + job.name)) break if as_conf.get_remote_dependencies() == "true": all_parents_completed.append(job.name) @@ -2173,23 +2234,26 @@ class JobList(object): job.hold = False save = True Log.debug( - "A job in prepared status has all parent completed, job: {0} status set to: READY ...".format(job.name)) + "A job in prepared status has all parent completed, job: {0} status set to: READY ...".format( + job.name)) Log.debug('Updating WAITING jobs eligible for be prepared') # Setup job name should be a variable for job in self.get_waiting_remote_dependencies('slurm'): if job.name not in all_parents_completed: tmp = [parent for parent in job.parents if ( - (parent.status == Status.SKIPPED or parent.status == Status.COMPLETED or parent.status == Status.QUEUING or parent.status == Status.RUNNING) and "setup" not in parent.name.lower())] + ( + parent.status == Status.SKIPPED or parent.status == Status.COMPLETED or parent.status == Status.QUEUING or parent.status == Status.RUNNING) and "setup" not in parent.name.lower())] if len(tmp) == len(job.parents): job.status = Status.PREPARED job.hold = True Log.debug( - "Setting job: {0} status to: Prepared for be held (all parents queuing, running or completed)...".format(job.name)) + "Setting job: {0} status to: Prepared for be held (all parents queuing, running or completed)...".format( + job.name)) Log.debug('Updating Held jobs') if self.job_package_map: held_jobs = [job for job in self.get_held_jobs() if ( - job.id not in list(self.job_package_map.keys()))] + job.id not in list(self.job_package_map.keys()))] held_jobs += [wrapper_job for wrapper_job in list(self.job_package_map.values()) if wrapper_job.status == Status.HELD] else: @@ -2208,7 +2272,7 @@ class JobList(object): job.hold = hold_wrapper if not job.hold: for inner_job in job.job_list: - inner_job.hold = False + inner_job.hold = False Log.debug( "Setting job: {0} status to: Queuing (all parents completed)...".format( job.name)) @@ -2216,7 +2280,7 @@ class JobList(object): tmp = [ parent for parent in job.parents if parent.status == Status.COMPLETED] if len(tmp) == len(job.parents): - job.hold = False + job.hold = False Log.debug( "Setting job: {0} status to: Queuing (all parents completed)...".format( job.name)) @@ -2248,7 +2312,7 @@ class JobList(object): for related_job in jobs_to_skip[section]: if members.index(job.member) < members.index( related_job.member) and job.chunk == related_job.chunk and jobdate == date2str( - related_job.date, related_job.date_format): + related_job.date, related_job.date_format): try: if job.status == Status.QUEUING: job.platform.send_command(job.platform.cancel_cmd + " " + str(job.id), @@ -2257,7 +2321,7 @@ class JobList(object): pass # job_id finished already job.status = Status.SKIPPED save = True - #save = True + # save = True self.update_two_step_jobs() Log.debug('Update finished') return save @@ -2306,7 +2370,8 @@ class JobList(object): if m_time_job_conf: if m_time_job_conf > m_time_db: Log.info( - "File jobs_{0}.yml has been modified since the last time the structure persistence was saved.".format(self.expid)) + "File jobs_{0}.yml has been modified since the last time the structure persistence was saved.".format( + self.expid)) structure_valid = False else: Log.info( @@ -2329,7 +2394,7 @@ class JobList(object): if structure_valid is False: # Structure does not exist, or it is not be updated, attempt to create it. Log.info("Updating structure persistence...") - self.graph = transitive_reduction(self.graph) # add threads for large experiments? todo + self.graph = transitive_reduction(self.graph) # add threads for large experiments? todo if self.graph: for job in self._job_list: children_to_remove = [ @@ -2365,7 +2430,8 @@ class JobList(object): out = False return out - def save_wrappers(self,packages_to_save,failed_packages,as_conf,packages_persistence,hold=False,inspect=False): + def save_wrappers(self, packages_to_save, failed_packages, as_conf, packages_persistence, hold=False, + inspect=False): for package in packages_to_save: if package.jobs[0].id not in failed_packages: if hasattr(package, "name"): @@ -2380,6 +2446,7 @@ class JobList(object): # Saving only when it is a real multi job package packages_persistence.save( package.name, package.jobs, package._expid, inspect) + def check_scripts(self, as_conf): """ When we have created the scripts, all parameters should have been substituted. @@ -2437,7 +2504,7 @@ class JobList(object): self._job_list.remove(job) - def rerun(self, job_list_unparsed,as_conf, monitor=False): + def rerun(self, job_list_unparsed, as_conf, monitor=False): """ Updates job list to rerun the jobs specified by a job list :param job_list_unparsed: list of jobs to rerun @@ -2448,7 +2515,7 @@ class JobList(object): :type monitor: bool """ - self.parse_jobs_by_filter(job_list_unparsed,two_step_start=False) + self.parse_jobs_by_filter(job_list_unparsed, two_step_start=False) member_list = set() chunk_list = set() date_list = set() @@ -2477,8 +2544,8 @@ class JobList(object): for job_section in job_sections: Log.debug( "Reading rerun dependencies for {0} jobs".format(job_section)) - if as_conf.jobs_data[job_section].get('DEPENDENCIES',None) is not None: - dependencies_keys = as_conf.jobs_data[job_section].get('DEPENDENCIES',{}) + if as_conf.jobs_data[job_section].get('DEPENDENCIES', None) is not None: + dependencies_keys = as_conf.jobs_data[job_section].get('DEPENDENCIES', {}) if type(dependencies_keys) is str: dependencies_keys = dependencies_keys.upper().split() if dependencies_keys is None: @@ -2487,11 +2554,16 @@ class JobList(object): for job in self.get_jobs_by_section(job_section): for key in dependencies_keys: dependency = dependencies[key] - skip, (chunk, member, date) = JobList._calculate_dependency_metadata(job.chunk, self._chunk_list, job.member, self._member_list, job.date, self._date_list, dependency) + skip, (chunk, member, date) = JobList._calculate_dependency_metadata(job.chunk, + self._chunk_list, + job.member, + self._member_list, + job.date, self._date_list, + dependency) if skip: continue section_name = dependencies[key].section - for parent in self._dic_jobs.get_jobs(section_name, job.date, job.member,job.chunk): + for parent in self._dic_jobs.get_jobs(section_name, job.date, job.member, job.chunk): if not monitor: parent.status = Status.WAITING Log.debug("Parent: " + parent.name) @@ -2535,10 +2607,11 @@ class JobList(object): allJobs = self.get_all() if existingList is None else existingList # Header result = (bcolors.BOLD if nocolor is False else '') + \ - "## String representation of Job List [" + str(len(allJobs)) + "] " + "## String representation of Job List [" + str(len(allJobs)) + "] " if statusChange is not None and len(str(statusChange)) > 0: result += "with " + (bcolors.OKGREEN if nocolor is False else '') + str(len(list(statusChange.keys())) - ) + " Change(s) ##" + (bcolors.ENDC + bcolors.ENDC if nocolor is False else '') + ) + " Change(s) ##" + ( + bcolors.ENDC + bcolors.ENDC if nocolor is False else '') else: result += " ## " @@ -2549,7 +2622,7 @@ class JobList(object): if len(job.parents) == 0: roots.append(job) visited = list() - #print(root) + # print(root) # root exists for root in roots: if root is not None and len(str(root)) > 0: @@ -2610,17 +2683,17 @@ class JobList(object): prefix += "| " # Prefix + Job Name result = "\n" + prefix + \ - (bcolors.BOLD + bcolors.CODE_TO_COLOR[job.status] if nocolor is False else '') + \ - job.name + \ - (bcolors.ENDC + bcolors.ENDC if nocolor is False else '') + (bcolors.BOLD + bcolors.CODE_TO_COLOR[job.status] if nocolor is False else '') + \ + job.name + \ + (bcolors.ENDC + bcolors.ENDC if nocolor is False else '') if len(job._children) > 0: level += 1 children = job._children total_children = len(job._children) # Writes children number and status if color are not being showed result += " ~ [" + str(total_children) + (" children] " if total_children > 1 else " child] ") + \ - ("[" + Status.VALUE_TO_KEY[job.status] + - "] " if nocolor is True else "") + ("[" + Status.VALUE_TO_KEY[job.status] + + "] " if nocolor is True else "") if statusChange is not None and len(str(statusChange)) > 0: # Writes change if performed result += (bcolors.BOLD + @@ -2639,7 +2712,7 @@ class JobList(object): "] " if nocolor is True else "") return result - + @staticmethod def retrieve_packages(BasicConfig, expid, current_jobs=None): """ @@ -2704,7 +2777,8 @@ class JobList(object): return job_to_package, package_to_jobs, package_to_package_id, package_to_symbol @staticmethod - def retrieve_times(status_code, name, tmp_path, make_exception=False, job_times=None, seconds=False, job_data_collection=None): + def retrieve_times(status_code, name, tmp_path, make_exception=False, job_times=None, seconds=False, + job_data_collection=None): """ Retrieve job timestamps from database. :param job_data_collection: @@ -2765,10 +2839,13 @@ class JobList(object): if status_code in [Status.SUSPENDED]: t_submit = t_start = t_finish = 0 - return JobRow(job_data.job_name, int(queue_time), int(running_time), status, energy, JobList.ts_to_datetime(t_submit), JobList.ts_to_datetime(t_start), JobList.ts_to_datetime(t_finish), job_data.ncpus, job_data.run_id) + return JobRow(job_data.job_name, int(queue_time), int(running_time), status, energy, + JobList.ts_to_datetime(t_submit), JobList.ts_to_datetime(t_start), + JobList.ts_to_datetime(t_finish), job_data.ncpus, job_data.run_id) # Using standard procedure - if status_code in [Status.RUNNING, Status.SUBMITTED, Status.QUEUING, Status.FAILED] or make_exception is True: + if status_code in [Status.RUNNING, Status.SUBMITTED, Status.QUEUING, + Status.FAILED] or make_exception is True: # COMPLETED adds too much overhead so these values are now stored in a database and retrieved separately submit_time, start_time, finish_time, status = JobList._job_running_check( status_code, name, tmp_path) @@ -2781,7 +2858,7 @@ class JobList(object): Status.FAILED] else 0 else: queuing_for_min = ( - datetime.datetime.now() - submit_time) + datetime.datetime.now() - submit_time) running_for_min = datetime.datetime.now() - datetime.datetime.now() submit_time = mktime(submit_time.timetuple()) start_time = 0 @@ -2815,9 +2892,9 @@ class JobList(object): return seconds_queued = seconds_queued * \ - (-1) if seconds_queued < 0 else seconds_queued + (-1) if seconds_queued < 0 else seconds_queued seconds_running = seconds_running * \ - (-1) if seconds_running < 0 else seconds_running + (-1) if seconds_running < 0 else seconds_running if seconds is False: queue_time = math.ceil( seconds_queued / 60) if seconds_queued > 0 else 0 @@ -2827,17 +2904,17 @@ class JobList(object): queue_time = seconds_queued running_time = seconds_running - return JobRow(name, - int(queue_time), - int(running_time), - status, - energy, - JobList.ts_to_datetime(submit_time), - JobList.ts_to_datetime(start_time), - JobList.ts_to_datetime(finish_time), - 0, - 0) - + return JobRow(name, + int(queue_time), + int(running_time), + status, + energy, + JobList.ts_to_datetime(submit_time), + JobList.ts_to_datetime(start_time), + JobList.ts_to_datetime(finish_time), + 0, + 0) + @staticmethod def _job_running_check(status_code, name, tmp_path): """ @@ -2908,7 +2985,7 @@ class JobList(object): if len(values) > 3 and current_status != status_from_job and current_status != "NA": current_status = "SUSPICIOUS" return submit_time, start_time, finish_time, current_status - + @staticmethod def ts_to_datetime(timestamp): if timestamp and timestamp > 0: @@ -2916,4 +2993,4 @@ class JobList(object): # timestamp).strftime('%Y-%m-%d %H:%M:%S')) return datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S') else: - return None \ No newline at end of file + return None diff --git a/autosubmit/monitor/monitor.py b/autosubmit/monitor/monitor.py index 8b8bffc55..53293ef11 100644 --- a/autosubmit/monitor/monitor.py +++ b/autosubmit/monitor/monitor.py @@ -233,6 +233,48 @@ class Monitor: Log.debug('Graph definition finalized') return graph + def _check_final_status(self, job, child): + # order of self._table + # child.edge_info is a tuple, I want to get first element of each tuple with a lambda + label = None + if len(child.edge_info) > 0: + if job in child.edge_info.get("FAILED",{}): + color = self._table.get(Status.FAILED,None) + label = child.edge_info["FAILED"].get(job.name,0)[1] + elif job.name in child.edge_info.get("RUNNING",{}): + color = self._table.get(Status.RUNNING,None) + label = child.edge_info["RUNNING"].get(job.name,0)[1] + elif job.name in child.edge_info.get("QUEUING",{}): + color = self._table.get(Status.QUEUING,None) + elif job.name in child.edge_info.get("HELD",{}): + color = self._table.get(Status.HELD,None) + elif job.name in child.edge_info.get("DELAYED",{}): + color = self._table.get(Status.DELAYED,None) + elif job.name in child.edge_info.get("UNKNOWN",{}): + color = self._table.get(Status.UNKNOWN,None) + elif job.name in child.edge_info.get("SUSPENDED",{}): + color = self._table.get(Status.SUSPENDED,None) + elif job.name in child.edge_info.get("SKIPPED",{}): + color = self._table.get(Status.SKIPPED,None) + elif job.name in child.edge_info.get("WAITING",{}): + color = self._table.get(Status.WAITING,None) + elif job.name in child.edge_info.get("READY",{}): + color = self._table.get(Status.READY,None) + elif job.name in child.edge_info.get("SUBMITTED",{}): + color = self._table.get(Status.SUBMITTED,None) + else: + color = self._table.get(Status.COMPLETED,None) + if label and label == 0: + label = None + return color,label + else: + return None, None + + + + + + def _add_children(self, job, exp, node_job, groups, hide_groups): if job in self.nodes_plotted: return @@ -241,20 +283,29 @@ class Monitor: for child in sorted(job.children, key=lambda k: k.name): node_child, skip = self._check_node_exists( exp, child, groups, hide_groups) + color, label = self._check_final_status(job, child) if len(node_child) == 0 and not skip: node_child = self._create_node(child, groups, hide_groups) if node_child: exp.add_node(node_child) - if job.name in child.edge_info and child.edge_info[job.name].get('optional', False): - exp.add_edge(pydotplus.Edge(node_job, node_child,style="dashed")) + if color: + # label = None doesn't disable label, instead it sets it to nothing and complain about invalid syntax + if label: + exp.add_edge(pydotplus.Edge(node_job, node_child,style="dashed",color=color,label=label)) + else: + exp.add_edge(pydotplus.Edge(node_job, node_child,style="dashed",color=color)) else: exp.add_edge(pydotplus.Edge(node_job, node_child)) else: skip = True elif not skip: node_child = node_child[0] - if job.name in child.edge_info and child.edge_info[job.name].get('optional', False): - exp.add_edge(pydotplus.Edge(node_job, node_child,style="dashed")) + if color: + # label = None doesn't disable label, instead it sets it to nothing and complain about invalid syntax + if label: + exp.add_edge(pydotplus.Edge(node_job, node_child, style="dashed", color=color, label=label)) + else: + exp.add_edge(pydotplus.Edge(node_job, node_child, style="dashed", color=color)) else: exp.add_edge(pydotplus.Edge(node_job, node_child)) skip = True -- GitLab From 8c01e6deb2d33a28d2551abe34ed8eeab7304da0 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 10 Jul 2023 15:13:08 +0200 Subject: [PATCH 42/68] added some tests --- autosubmit/job/job_list.py | 168 +++++++++++++++++++++++---------- test/unit/test_dependencies.py | 64 ++++++++++--- 2 files changed, 168 insertions(+), 64 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 71b128daa..f29926ab4 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -439,94 +439,162 @@ class JobList(object): else: return False + + @staticmethod - def _parse_checkpoint(data): - checkpoint = {"STATUS": None, "FROM_STEP": None} - data = data.lower() - if data[0] == "r": - checkpoint["STATUS"] = Status.RUNNING - if len(data) > 1: - checkpoint["FROM_STEP"] = data[1:] - else: - checkpoint["FROM_STEP"] = "1" - elif data[0] == "f": - checkpoint["STATUS"] = Status.FAILED - if len(data) > 1: - checkpoint["FROM_STEP"] = data[1:] - else: - checkpoint["FROM_STEP"] = "1" - elif data[0] == "q": - checkpoint["STATUS"] = Status.QUEUING - elif data[0] == "s": - checkpoint["STATUS"] = Status.SUBMITTED - return checkpoint + def _parse_filters_to_check(list_of_values_to_check,value_list=[]): + final_values = [] + list_of_values_to_check = str(list_of_values_to_check).upper() + if list_of_values_to_check is None: + return None + elif list_of_values_to_check == "ALL": + return ["ALL"] + elif list_of_values_to_check == "NONE": + return None + elif list_of_values_to_check == "NATURAL": + return ["NATURAL"] + elif "," in list_of_values_to_check: + for value_to_check in list_of_values_to_check.split(","): + final_values.extend(JobList._parse_filter_to_check(value_to_check,value_list)) + else: + final_values = JobList._parse_filter_to_check(list_of_values_to_check,value_list) + return final_values + @staticmethod - def _parse_filter_to_check(value_to_check): + def _parse_filter_to_check(value_to_check,value_list=[]): """ Parse the filter to check and return the value to check. Selection process: value_to_check can be: a range: [0:], [:N], [0:N], [:-1], [0:N:M] ... - a value: N - a list of values : 0,2,4,5,7,10 ... + a value: N. a range with step: [0::M], [::2], [0::3], [::3] ... :param value_to_check: value to check. + :param value_list: list of values to check. Dates, members, chunks or splits. :return: parsed value to check. """ - # regex + value_to_check = str(value_to_check).upper() - if value_to_check is None: - return None - elif value_to_check == "ALL": - return "ALL" - elif value_to_check == "NONE": - return None - elif value_to_check == 1: + if value_to_check.count(":") == 1: # range - if value_to_check[0] == ":": + if value_to_check[1] == ":": # [:N] - return slice(None, int(value_to_check[1:])) - elif value_to_check[-1] == ":": + # Find N index in the list + start = None + end = value_to_check.split(":")[1].strip("[]") + # get index in the value_list + if len(value_list) > 0: + end = value_list.index(end) + else: + end = int(end) + elif value_to_check[-2] == ":": # [N:] - return slice(int(value_to_check[:-1]), None) + # Find N index in the list + start = value_to_check.split(":")[0].strip("[]") + end = None + # get index in the value_list + if len(value_list) > 0: + start = value_list.index(start) + else: + start = int(start) else: # [N:M] - return slice(int(value_to_check.split(":")[0]), int(value_to_check.split(":")[1])) + # Find N index in the list + start = value_to_check.split(":")[0].strip("[]") + end = value_to_check.split(":")[1].strip("[]") + step = None + # get index in the value_list + if len(value_list) > 0: + start = value_list.index(start) + end = value_list.index(end) + else: + start = int(start) + end = int(end) + if end is not None: + end+=1 + if len(value_list) > 0: + return value_list[slice(start, end)] + else: + return [ str(number_gen) for number_gen in range(start, end)] elif value_to_check.count(":") == 2: # range with step - if value_to_check[0] == ":": - # [::M] - return slice(None, None, int(value_to_check[2:])) - elif value_to_check[-1] == ":": - # [N::] - return slice(int(value_to_check[:-2]), None, None) + if value_to_check[-2] == ":" and value_to_check[-3] == ":": # [N::] + # Find N index in the list + start = value_to_check.split(":")[0].strip("[]") + end = None + step = None + # get index in the value_list + if len(value_list) > 0: + start = value_list.index(start) + else: + start = int(start) + elif value_to_check[1] == ":" and value_to_check[2] == ":": # [::S] + # Find N index in the list + start = None + end = None + step = value_to_check.split(":")[-1].strip("[]") + # get index in the value_list + step = int(step) + elif value_to_check[1] == ":" and value_to_check[-2] == ":": # [:M:] + # Find N index in the list + start = None + end = value_to_check.split(":")[1].strip("[]") + step = None + # get index in the value_list + if len(value_list) > 0: + end = value_list.index(end) + else: + end = int(end) + else: # [N:M:S] + # Find N index in the list + start = value_to_check.split(":")[0].strip("[]") + end = value_to_check.split(":")[1].strip("[]") + step = value_to_check.split(":")[2].strip("[]") + # get index in the value_list + if len(value_list) > 0: + start = value_list.index(start) + end = value_list.index(end) + else: + start = int(start) + end = int(end) + step = int(step) + if end is not None: + end+=1 + if len(value_list) > 0: + return value_list[slice(start, end, step)] else: - # [N::M] - return slice(int(value_to_check.split(":")[0]), None, int(value_to_check.split(":")[2])) - elif "," in value_to_check: - # list - return value_to_check.split(",") + return [str(number_gen) for number_gen in range(start, end, step)] else: # value - return value_to_check + return [value_to_check] + @staticmethod def _check_relationship(relationships, level_to_check, value_to_check): """ Check if the current_job_value is included in the filter_value :param relationships: current filter level to check. - :param level_to_check: Can be date_from, member_from, chunk_from, split_from. + :param level_to_check: Can be dates_from, members_from, chunks_from, splits_from. :param value_to_check: Can be None, a date, a member, a chunk or a split. :return: """ filters = [] + if level_to_check == "DATES_FROM": + values_list = self._date_list + elif level_to_check == "MEMBERS_FROM": + values_list = self._member_list + elif level_to_check == "CHUNKS_FROM": + values_list = self._chunk_list + else: + values_list = [] # need to obtain the MAX amount of splits set in the workflow + relationship = relationships.get(level_to_check, {}) status = relationship.pop("STATUS", relationships.get("STATUS", None)) from_step = relationship.pop("FROM_STEP", relationships.get("FROM_STEP", None)) for filter_range, filter_data in relationship.items(): if not value_to_check or str(value_to_check).upper() in str( - JobList._parse_filter_to_check(filter_range)).upper(): + JobList._parse_filters_to_check(filter_range)).upper(): if not filter_data.get("STATUS", None): filter_data["STATUS"] = status if not filter_data.get("FROM_STEP", None): diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 0af18081b..6b554f76a 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -1,17 +1,21 @@ -import unittest - import mock +import unittest from copy import deepcopy -from autosubmit.job.job_list import JobList +from datetime import datetime + from autosubmit.job.job import Job from autosubmit.job.job_common import Status -from datetime import datetime +from autosubmit.job.job_list import JobList + + class TestJobList(unittest.TestCase): def setUp(self): + self.date_list = ["20020201", "20020202", "20020203", "20020204", "20020205", "20020206", "20020207", "20020208", "20020209", "20020210"] + self.member_list = ["fc1", "fc2", "fc3", "fc4", "fc5", "fc6", "fc7", "fc8", "fc9", "fc10"] + self.chunk_list = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"] + self.split_list = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"] # Define common test case inputs here self.relationships_dates = { - "OPTIONAL": False, - "CHECKPOINT": None, "DATES_FROM": { "20020201": { "MEMBERS_FROM": { @@ -34,8 +38,6 @@ class TestJobList(unittest.TestCase): self.relationships_dates_optional["DATES_FROM"]["20020201"]["SPLITS_FROM"] = { "ALL": { "SPLITS_TO": "1?" } } self.relationships_members = { - "OPTIONAL": False, - "CHECKPOINT": None, "MEMBERS_FROM": { "fc2": { "SPLITS_FROM": { @@ -50,8 +52,6 @@ class TestJobList(unittest.TestCase): } } self.relationships_chunks = { - "OPTIONAL": False, - "CHECKPOINT": None, "CHUNKS_FROM": { "1": { "DATES_TO": "20020201", @@ -62,8 +62,6 @@ class TestJobList(unittest.TestCase): } } self.relationships_chunks2 = { - "OPTIONAL": False, - "CHECKPOINT": None, "CHUNKS_FROM": { "1": { "DATES_TO": "20020201", @@ -82,8 +80,6 @@ class TestJobList(unittest.TestCase): } self.relationships_splits = { - "OPTIONAL": False, - "CHECKPOINT": None, "SPLITS_FROM": { "1": { "DATES_TO": "20020201", @@ -170,6 +166,46 @@ class TestJobList(unittest.TestCase): "SPLITS_TO": "1?" } self.assertEqual(result, expected_output) + def test_parse_filters_to_check(self): + result = JobList._parse_filters_to_check("20020201,20020202,20020203",self.date_list) + expected_output = ["20020201","20020202","20020203"] + self.assertEqual(result, expected_output) + result = JobList._parse_filters_to_check("20020201,[20020203:20020205]",self.date_list) + + + def test_parse_filter_to_check(self): + # Call the function to get the result + # Value can have the following formats: + # a range: [0:], [:N], [0:N], [:-1], [0:N:M] ... + # a value: N + # a range with step: [0::M], [::2], [0::3], [::3] ... + result = JobList._parse_filter_to_check("20020201",self.date_list) + expected_output = ["20020201"] + self.assertEqual(result, expected_output) + result = JobList._parse_filter_to_check("[20020201:20020203]",self.date_list) + expected_output = ["20020201","20020202","20020203"] + self.assertEqual(result, expected_output) + result = JobList._parse_filter_to_check("[20020201:20020203:2]",self.date_list) + expected_output = ["20020201","20020203"] + self.assertEqual(result, expected_output) + result = JobList._parse_filter_to_check("[20020202:]",self.date_list) + expected_output = self.date_list[1:] + self.assertEqual(result, expected_output) + result = JobList._parse_filter_to_check("[:20020203]",self.date_list) + expected_output = self.date_list[:3] + self.assertEqual(result, expected_output) + result = JobList._parse_filter_to_check("[::2]",self.date_list) + expected_output = self.date_list[::2] + self.assertEqual(result, expected_output) + result = JobList._parse_filter_to_check("[20020203::]",self.date_list) + expected_output = self.date_list[2:] + self.assertEqual(result, expected_output) + result = JobList._parse_filter_to_check("[:20020203:]",self.date_list) + expected_output = self.date_list[:3] + self.assertEqual(result, expected_output) + + + def test_check_dates(self): # Call the function to get the result self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") -- GitLab From f8f02fce32481686f663b665b9eb1aa3ac8ff708 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 12 Jul 2023 08:56:42 +0200 Subject: [PATCH 43/68] pipeline passing --- autosubmit/job/job.py | 43 ++++----- autosubmit/job/job_list.py | 63 +++++++------ test/unit/test_dependencies.py | 157 ++++++++++++++++----------------- test/unit/test_job.py | 15 ++-- 4 files changed, 137 insertions(+), 141 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 3637f0897..cdaa45fd7 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -21,33 +21,33 @@ Main module for Autosubmit. Only contains an interface class to all functionality implemented on Autosubmit """ -import os -import re -import time -import json -import datetime -import textwrap from collections import OrderedDict -import copy +import copy +import datetime +import json import locale +import os +import re +import textwrap +import time +from bscearth.utils.date import date2str, parse_date, previous_day, chunk_end_date, chunk_start_date, Log, subs_dates +from functools import reduce +from threading import Thread +from time import sleep +from typing import List, Union -from autosubmitconfigparser.config.configcommon import AutosubmitConfig -from autosubmit.job.job_common import Status, Type, increase_wallclock_by_chunk +from autosubmit.helpers.parameters import autosubmit_parameter, autosubmit_parameters +from autosubmit.history.experiment_history import ExperimentHistory from autosubmit.job.job_common import StatisticsSnippetBash, StatisticsSnippetPython from autosubmit.job.job_common import StatisticsSnippetR, StatisticsSnippetEmpty +from autosubmit.job.job_common import Status, Type, increase_wallclock_by_chunk from autosubmit.job.job_utils import get_job_package_code -from autosubmitconfigparser.config.basicconfig import BasicConfig -from autosubmit.history.experiment_history import ExperimentHistory -from bscearth.utils.date import date2str, parse_date, previous_day, chunk_end_date, chunk_start_date, Log, subs_dates -from time import sleep -from threading import Thread from autosubmit.platforms.paramiko_submitter import ParamikoSubmitter -from log.log import Log, AutosubmitCritical, AutosubmitError -from typing import List, Union -from functools import reduce +from autosubmitconfigparser.config.basicconfig import BasicConfig +from autosubmitconfigparser.config.configcommon import AutosubmitConfig from autosubmitconfigparser.config.yamlparser import YAMLParserFactory -from autosubmit.helpers.parameters import autosubmit_parameter, autosubmit_parameters +from log.log import Log, AutosubmitCritical, AutosubmitError Log.get_logger("Autosubmit") @@ -261,6 +261,7 @@ class Job(object): def checkpoint(self): """Generates a checkpoint step for this job based on job.type""" return self._checkpoint + @checkpoint.setter def checkpoint(self): """Generates a checkpoint step for this job based on job.type""" @@ -273,11 +274,11 @@ class Job(object): def get_checkpoint_files(self): """ - Downloads checkpoint files from remote host. If they aren't already in local. - :param steps: list of steps to download - :return: the max step downloaded + Check if there is a file on the remote host that contains the checkpoint """ return self.platform.get_checkpoint_files(self) + + @property @autosubmit_parameter(name='sdate') def sdate(self): """Current start date.""" diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index f29926ab4..89d8238c9 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -569,9 +569,7 @@ class JobList(object): # value return [value_to_check] - - @staticmethod - def _check_relationship(relationships, level_to_check, value_to_check): + def _check_relationship(self, relationships, level_to_check, value_to_check): """ Check if the current_job_value is included in the filter_value :param relationships: current filter level to check. @@ -581,6 +579,10 @@ class JobList(object): """ filters = [] if level_to_check == "DATES_FROM": + try: + value_to_check = date2str(value_to_check, "%Y%m%d") + except: + pass values_list = self._date_list elif level_to_check == "MEMBERS_FROM": values_list = self._member_list @@ -593,8 +595,8 @@ class JobList(object): status = relationship.pop("STATUS", relationships.get("STATUS", None)) from_step = relationship.pop("FROM_STEP", relationships.get("FROM_STEP", None)) for filter_range, filter_data in relationship.items(): - if not value_to_check or str(value_to_check).upper() in str( - JobList._parse_filters_to_check(filter_range)).upper(): + if filter_range in ["ALL","NATURAL"] or ( not value_to_check or str(value_to_check).upper() in str( + JobList._parse_filters_to_check(filter_range)).upper()): if not filter_data.get("STATUS", None): filter_data["STATUS"] = status if not filter_data.get("FROM_STEP", None): @@ -605,8 +607,8 @@ class JobList(object): filters = [{}] return filters - @staticmethod - def _check_dates(relationships, current_job): + + def _check_dates(self, relationships, current_job): """ Check if the current_job_value is included in the filter_from and retrieve filter_to value :param relationships: Remaining filters to apply. @@ -614,7 +616,7 @@ class JobList(object): :return: filters_to_apply """ - filters_to_apply = JobList._check_relationship(relationships, "DATES_FROM", date2str(current_job.date)) + filters_to_apply = self._check_relationship(relationships, "DATES_FROM", date2str(current_job.date)) # there could be multiple filters that apply... per example # Current task date is 20020201, and member is fc2 # Dummy example, not specially usefull in a real case @@ -643,17 +645,17 @@ class JobList(object): # Will enter, go recursivily to the similar methods and in the end it will do: # Will enter members_from, and obtain [{DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", CHUNKS_FROM{...}] if "MEMBERS_FROM" in filter: - filters_to_apply_m = JobList._check_members({"MEMBERS_FROM": (filter.pop("MEMBERS_FROM"))}, current_job) + filters_to_apply_m = self._check_members({"MEMBERS_FROM": (filter.pop("MEMBERS_FROM"))}, current_job) if len(filters_to_apply_m) > 0: filters_to_apply[i].update(filters_to_apply_m) # Will enter chunks_from, and obtain [{DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"] if "CHUNKS_FROM" in filter: - filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM"))}, current_job) + filters_to_apply_c = self._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM"))}, current_job) if len(filters_to_apply_c) > 0 and len(filters_to_apply_c[0]) > 0: filters_to_apply[i].update(filters_to_apply_c) # IGNORED if "SPLITS_FROM" in filter: - filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM"))}, current_job) + filters_to_apply_s = self._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM"))}, current_job) if len(filters_to_apply_s) > 0: filters_to_apply[i].update(filters_to_apply_s) # Unify filters from all filters_from where the current job is included to have a single SET of filters_to @@ -661,29 +663,28 @@ class JobList(object): # {DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"} return filters_to_apply - @staticmethod - def _check_members(relationships, current_job): + + def _check_members(self,relationships, current_job): """ Check if the current_job_value is included in the filter_from and retrieve filter_to value :param relationships: Remaining filters to apply. :param current_job: Current job to check. :return: filters_to_apply """ - filters_to_apply = JobList._check_relationship(relationships, "MEMBERS_FROM", current_job.member) + filters_to_apply = self._check_relationship(relationships, "MEMBERS_FROM", current_job.member) for i, filter_ in enumerate(filters_to_apply): if "CHUNKS_FROM" in filter_: - filters_to_apply_c = JobList._check_chunks({"CHUNKS_FROM": (filter_.pop("CHUNKS_FROM"))}, current_job) + filters_to_apply_c = self._check_chunks({"CHUNKS_FROM": (filter_.pop("CHUNKS_FROM"))}, current_job) if len(filters_to_apply_c) > 0: filters_to_apply[i].update(filters_to_apply_c) if "SPLITS_FROM" in filter_: - filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter_.pop("SPLITS_FROM"))}, current_job) + filters_to_apply_s = self._check_splits({"SPLITS_FROM": (filter_.pop("SPLITS_FROM"))}, current_job) if len(filters_to_apply_s) > 0: filters_to_apply[i].update(filters_to_apply_s) filters_to_apply = JobList._unify_to_filters(filters_to_apply) return filters_to_apply - @staticmethod - def _check_chunks(relationships, current_job): + def _check_chunks(self,relationships, current_job): """ Check if the current_job_value is included in the filter_from and retrieve filter_to value :param relationships: Remaining filters to apply. @@ -691,17 +692,16 @@ class JobList(object): :return: filters_to_apply """ - filters_to_apply = JobList._check_relationship(relationships, "CHUNKS_FROM", current_job.chunk) + filters_to_apply = self._check_relationship(relationships, "CHUNKS_FROM", current_job.chunk) for i, filter in enumerate(filters_to_apply): if "SPLITS_FROM" in filter: - filters_to_apply_s = JobList._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM"))}, current_job) + filters_to_apply_s = self._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM"))}, current_job) if len(filters_to_apply_s) > 0: filters_to_apply[i].update(filters_to_apply_s) filters_to_apply = JobList._unify_to_filters(filters_to_apply) return filters_to_apply - @staticmethod - def _check_splits(relationships, current_job): + def _check_splits(self,relationships, current_job): """ Check if the current_job_value is included in the filter_from and retrieve filter_to value :param relationships: Remaining filters to apply. @@ -709,7 +709,7 @@ class JobList(object): :return: filters_to_apply """ - filters_to_apply = JobList._check_relationship(relationships, "SPLITS_FROM", current_job.split) + filters_to_apply = self._check_relationship(relationships, "SPLITS_FROM", current_job.split) # No more FROM sections to check, unify _to FILTERS and return filters_to_apply = JobList._unify_to_filters(filters_to_apply) return filters_to_apply @@ -780,8 +780,7 @@ class JobList(object): JobList._normalize_to_filters(unified_filter, "SPLITS_TO") return unified_filter - @staticmethod - def _filter_current_job(current_job, relationships): + def _filter_current_job(self,current_job, relationships): ''' This function will filter the current job based on the relationships given :param current_job: Current job to filter :param relationships: Relationships to apply @@ -808,13 +807,13 @@ class JobList(object): if relationships is not None and len(relationships) > 0: # Look for a starting point, this can be if else becasue they're exclusive as a DATE_FROM can't be in a MEMBER_FROM and so on if "DATES_FROM" in relationships: - filters_to_apply = JobList._check_dates(relationships, current_job) + filters_to_apply = self._check_dates(relationships, current_job) elif "MEMBERS_FROM" in relationships: - filters_to_apply = JobList._check_members(relationships, current_job) + filters_to_apply = self._check_members(relationships, current_job) elif "CHUNKS_FROM" in relationships: - filters_to_apply = JobList._check_chunks(relationships, current_job) + filters_to_apply = self._check_chunks(relationships, current_job) elif "SPLITS_FROM" in relationships: - filters_to_apply = JobList._check_splits(relationships, current_job) + filters_to_apply = self._check_splits(relationships, current_job) else: relationships.pop("OPTIONAL", None) relationships.pop("CHECKPOINT", None) @@ -928,10 +927,8 @@ class JobList(object): natural_jobs = dic_jobs.get_jobs(dependency.section, date, member, chunk) all_parents = list(set(other_parents + parents_jobs)) # Get dates_to, members_to, chunks_to of the deepest level of the relationship. - filters_to_apply = JobList._filter_current_job(job, copy.deepcopy(dependency.relationships)) - if "?" in filters_to_apply.get("SPLITS_TO", "") or "?" in filters_to_apply.get("DATES_TO", - "") or "?" in filters_to_apply.get( - "MEMBERS_TO", "") or "?" in filters_to_apply.get("CHUNKS_TO", ""): + filters_to_apply = self._filter_current_job(job, copy.deepcopy(dependency.relationships)) + if "?" in filters_to_apply.get("SPLITS_TO", "") or "?" in filters_to_apply.get("DATES_TO","") or "?" in filters_to_apply.get("MEMBERS_TO", "") or "?" in filters_to_apply.get("CHUNKS_TO", ""): only_marked_status = True else: only_marked_status = False diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 6b554f76a..007aac369 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -1,4 +1,6 @@ +import inspect import mock +import tempfile import unittest from copy import deepcopy from datetime import datetime @@ -6,14 +8,50 @@ from datetime import datetime from autosubmit.job.job import Job from autosubmit.job.job_common import Status from autosubmit.job.job_list import JobList +from autosubmit.job.job_list_persistence import JobListPersistenceDb +from autosubmitconfigparser.config.yamlparser import YAMLParserFactory +class FakeBasicConfig: + def __init__(self): + pass + def props(self): + pr = {} + for name in dir(self): + value = getattr(self, name) + if not name.startswith('__') and not inspect.ismethod(value) and not inspect.isfunction(value): + pr[name] = value + return pr + DB_DIR = '/dummy/db/dir' + DB_FILE = '/dummy/db/file' + DB_PATH = '/dummy/db/path' + LOCAL_ROOT_DIR = '/dummy/local/root/dir' + LOCAL_TMP_DIR = '/dummy/local/temp/dir' + LOCAL_PROJ_DIR = '/dummy/local/proj/dir' + DEFAULT_PLATFORMS_CONF = '' + DEFAULT_JOBS_CONF = '' + class TestJobList(unittest.TestCase): def setUp(self): + self.experiment_id = 'random-id' + self.as_conf = mock.Mock() + self.as_conf.experiment_data = dict() + self.as_conf.experiment_data["JOBS"] = dict() + self.as_conf.jobs_data = self.as_conf.experiment_data["JOBS"] + self.as_conf.experiment_data["PLATFORMS"] = dict() + self.temp_directory = tempfile.mkdtemp() + self.JobList = JobList(self.experiment_id, FakeBasicConfig, YAMLParserFactory(), + JobListPersistenceDb(self.temp_directory, 'db'), self.as_conf) self.date_list = ["20020201", "20020202", "20020203", "20020204", "20020205", "20020206", "20020207", "20020208", "20020209", "20020210"] self.member_list = ["fc1", "fc2", "fc3", "fc4", "fc5", "fc6", "fc7", "fc8", "fc9", "fc10"] self.chunk_list = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"] self.split_list = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"] + self.JobList._date_list = self.date_list + self.JobList._member_list = self.member_list + self.JobList._chunk_list = self.chunk_list + self.JobList._split_list = self.split_list + + # Define common test case inputs here self.relationships_dates = { "DATES_FROM": { @@ -109,69 +147,30 @@ class TestJobList(unittest.TestCase): self.mock_job.chunk = None self.mock_job.split = None - def test_parse_checkpoint(self): - data = "r2" - correct = {"FROM_STEP": '2', "STATUS":Status.RUNNING} - result = JobList._parse_checkpoint(data) - self.assertEqual(result, correct) - data = "r" - correct = {"FROM_STEP": '1', "STATUS":Status.RUNNING} - result = JobList._parse_checkpoint(data) - self.assertEqual(result, correct) - data = "f2" - correct = {"FROM_STEP": '2', "STATUS":Status.FAILED} - result = JobList._parse_checkpoint(data) - self.assertEqual(result, correct) - data = "f" - correct = {"FROM_STEP": '1', "STATUS":Status.FAILED} - result = JobList._parse_checkpoint(data) - self.assertEqual(result, correct) - data = "s" - correct = {"FROM_STEP": None, "STATUS":Status.SUBMITTED} - result = JobList._parse_checkpoint(data) - self.assertEqual(result, correct) - data = "s2" - correct = {"FROM_STEP": None, "STATUS":Status.SUBMITTED} - result = JobList._parse_checkpoint(data) - self.assertEqual(result, correct) - data = "q" - correct = {"FROM_STEP": None, "STATUS":Status.QUEUING} - result = JobList._parse_checkpoint(data) - self.assertEqual(result, correct) - data = "q2" - correct = {"FROM_STEP": None, "STATUS":Status.QUEUING} - result = JobList._parse_checkpoint(data) - self.assertEqual(result, correct) - - def test_simple_dependency(self): - result_d = JobList._check_dates({}, self.mock_job) - result_m = JobList._check_members({}, self.mock_job) - result_c = JobList._check_chunks({}, self.mock_job) - result_s = JobList._check_splits({}, self.mock_job) + result_d = self.JobList._check_dates({}, self.mock_job) + result_m = self.JobList._check_members({}, self.mock_job) + result_c = self.JobList._check_chunks({}, self.mock_job) + result_s = self.JobList._check_splits({}, self.mock_job) self.assertEqual(result_d, {}) self.assertEqual(result_m, {}) self.assertEqual(result_c, {}) self.assertEqual(result_s, {}) - def test_check_dates_optional(self): - self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") - self.mock_job.member = "fc2" - self.mock_job.chunk = 1 - self.mock_job.split = 1 - result = JobList._check_dates(self.relationships_dates_optional, self.mock_job) - expected_output = { - "DATES_TO": "20020201?", - "MEMBERS_TO": "fc2?", - "CHUNKS_TO": "ALL?", - "SPLITS_TO": "1?" - } - self.assertEqual(result, expected_output) + def test_parse_filters_to_check(self): - result = JobList._parse_filters_to_check("20020201,20020202,20020203",self.date_list) + """Test the _parse_filters_to_check function""" + result = self.JobList._parse_filters_to_check("20020201,20020202,20020203",self.date_list) expected_output = ["20020201","20020202","20020203"] self.assertEqual(result, expected_output) - result = JobList._parse_filters_to_check("20020201,[20020203:20020205]",self.date_list) - + result = self.JobList._parse_filters_to_check("20020201,[20020203:20020205]",self.date_list) + expected_output = ["20020201","20020203","20020204","20020205"] + self.assertEqual(result, expected_output) + result = self.JobList._parse_filters_to_check("[20020201:20020203],[20020205:20020207]",self.date_list) + expected_output = ["20020201","20020202","20020203","20020205","20020206","20020207"] + self.assertEqual(result, expected_output) + result = self.JobList._parse_filters_to_check("20020201",self.date_list) + expected_output = ["20020201"] + self.assertEqual(result, expected_output) def test_parse_filter_to_check(self): # Call the function to get the result @@ -179,40 +178,38 @@ class TestJobList(unittest.TestCase): # a range: [0:], [:N], [0:N], [:-1], [0:N:M] ... # a value: N # a range with step: [0::M], [::2], [0::3], [::3] ... - result = JobList._parse_filter_to_check("20020201",self.date_list) + result = self.JobList._parse_filter_to_check("20020201",self.date_list) expected_output = ["20020201"] self.assertEqual(result, expected_output) - result = JobList._parse_filter_to_check("[20020201:20020203]",self.date_list) + result = self.JobList._parse_filter_to_check("[20020201:20020203]",self.date_list) expected_output = ["20020201","20020202","20020203"] self.assertEqual(result, expected_output) - result = JobList._parse_filter_to_check("[20020201:20020203:2]",self.date_list) + result = self.JobList._parse_filter_to_check("[20020201:20020203:2]",self.date_list) expected_output = ["20020201","20020203"] self.assertEqual(result, expected_output) - result = JobList._parse_filter_to_check("[20020202:]",self.date_list) + result = self.JobList._parse_filter_to_check("[20020202:]",self.date_list) expected_output = self.date_list[1:] self.assertEqual(result, expected_output) - result = JobList._parse_filter_to_check("[:20020203]",self.date_list) + result = self.JobList._parse_filter_to_check("[:20020203]",self.date_list) expected_output = self.date_list[:3] self.assertEqual(result, expected_output) - result = JobList._parse_filter_to_check("[::2]",self.date_list) + result = self.JobList._parse_filter_to_check("[::2]",self.date_list) expected_output = self.date_list[::2] self.assertEqual(result, expected_output) - result = JobList._parse_filter_to_check("[20020203::]",self.date_list) + result = self.JobList._parse_filter_to_check("[20020203::]",self.date_list) expected_output = self.date_list[2:] self.assertEqual(result, expected_output) - result = JobList._parse_filter_to_check("[:20020203:]",self.date_list) + result = self.JobList._parse_filter_to_check("[:20020203:]",self.date_list) expected_output = self.date_list[:3] self.assertEqual(result, expected_output) - - def test_check_dates(self): # Call the function to get the result self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") self.mock_job.member = "fc2" self.mock_job.chunk = 1 self.mock_job.split = 1 - result = JobList._check_dates(self.relationships_dates, self.mock_job) + result = self.JobList._check_dates(self.relationships_dates, self.mock_job) expected_output = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", @@ -221,14 +218,14 @@ class TestJobList(unittest.TestCase): } self.assertEqual(result, expected_output) self.mock_job.date = datetime.strptime("20020202", "%Y%m%d") - result = JobList._check_dates(self.relationships_dates, self.mock_job) + result = self.JobList._check_dates(self.relationships_dates, self.mock_job) self.assertEqual(result, {}) def test_check_members(self): # Call the function to get the result self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") self.mock_job.member = "fc2" - result = JobList._check_members(self.relationships_members, self.mock_job) + result = self.JobList._check_members(self.relationships_members, self.mock_job) expected_output = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", @@ -237,14 +234,14 @@ class TestJobList(unittest.TestCase): } self.assertEqual(result, expected_output) self.mock_job.member = "fc3" - result = JobList._check_members(self.relationships_members, self.mock_job) + result = self.JobList._check_members(self.relationships_members, self.mock_job) self.assertEqual(result, {}) def test_check_splits(self): # Call the function to get the result self.mock_job.split = 1 - result = JobList._check_splits(self.relationships_splits, self.mock_job) + result = self.JobList._check_splits(self.relationships_splits, self.mock_job) expected_output = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", @@ -253,13 +250,13 @@ class TestJobList(unittest.TestCase): } self.assertEqual(result, expected_output) self.mock_job.split = 2 - result = JobList._check_splits(self.relationships_splits, self.mock_job) + result = self.JobList._check_splits(self.relationships_splits, self.mock_job) self.assertEqual(result, {}) def test_check_chunks(self): # Call the function to get the result self.mock_job.chunk = 1 - result = JobList._check_chunks(self.relationships_chunks, self.mock_job) + result = self.JobList._check_chunks(self.relationships_chunks, self.mock_job) expected_output = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", @@ -268,17 +265,17 @@ class TestJobList(unittest.TestCase): } self.assertEqual(result, expected_output) self.mock_job.chunk = 2 - result = JobList._check_chunks(self.relationships_chunks, self.mock_job) + result = self.JobList._check_chunks(self.relationships_chunks, self.mock_job) self.assertEqual(result, {}) # test splits_from self.mock_job.split = 5 - result = JobList._check_chunks(self.relationships_chunks2, self.mock_job) + result = self.JobList._check_chunks(self.relationships_chunks2, self.mock_job) expected_output2 = { "SPLITS_TO": "2" } self.assertEqual(result, expected_output2) self.mock_job.split = 1 - result = JobList._check_chunks(self.relationships_chunks2, self.mock_job) + result = self.JobList._check_chunks(self.relationships_chunks2, self.mock_job) self.assertEqual(result, {}) def test_check_general(self): @@ -288,7 +285,7 @@ class TestJobList(unittest.TestCase): self.mock_job.member = "fc2" self.mock_job.chunk = 1 self.mock_job.split = 1 - result = JobList._filter_current_job(self.mock_job,self.relationships_general) + result = self.JobList._filter_current_job(self.mock_job,self.relationships_general) expected_output = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", @@ -317,17 +314,17 @@ class TestJobList(unittest.TestCase): self.mock_job.member = "fc2" self.mock_job.chunk = 1 self.mock_job.split = 1 - result = JobList._valid_parent(self.mock_job, date_list, member_list, chunk_list, is_a_natural_relation, filter_) + result = self.JobList._valid_parent(self.mock_job, date_list, member_list, chunk_list, is_a_natural_relation, filter_) # it returns a tuple, the first element is the result, the second is the optional flag - self.assertEqual(result, (True,False)) + self.assertEqual(result, True) filter_ = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", "CHUNKS_TO": "ALL", "SPLITS_TO": "1?" } - result = JobList._valid_parent(self.mock_job, date_list, member_list, chunk_list, is_a_natural_relation, filter_) - self.assertEqual(result, (True,True)) + result = self.JobList._valid_parent(self.mock_job, date_list, member_list, chunk_list, is_a_natural_relation, filter_) + self.assertEqual(result, True) if __name__ == '__main__': diff --git a/test/unit/test_job.py b/test/unit/test_job.py index caaf9c60a..f1bfbcbac 100644 --- a/test/unit/test_job.py +++ b/test/unit/test_job.py @@ -1,17 +1,19 @@ from unittest import TestCase + +import datetime +import inspect import os import sys -from autosubmitconfigparser.config.configcommon import AutosubmitConfig -from autosubmit.job.job_common import Status -from autosubmit.job.job import Job -from autosubmit.platforms.platform import Platform from mock import Mock, MagicMock from mock import patch -import datetime - # compatibility with both versions (2 & 3) from sys import version_info +from autosubmit.job.job import Job +from autosubmit.job.job_common import Status +from autosubmit.platforms.platform import Platform +from autosubmitconfigparser.config.configcommon import AutosubmitConfig + if version_info.major == 2: import builtins as builtins else: @@ -362,7 +364,6 @@ class TestJob(TestCase): self.job.date_format = test[1] self.assertEquals(test[2], self.job.sdate) -import inspect class FakeBasicConfig: def __init__(self): pass -- GitLab From 50384a2d550613fc7f8855df5dfd1b8a79406efa Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 12 Jul 2023 09:04:24 +0200 Subject: [PATCH 44/68] comment change --- autosubmit/job/job_list.py | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 89d8238c9..f15ae29e6 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -615,35 +615,9 @@ class JobList(object): :param current_job: Current job to check. :return: filters_to_apply """ - + # Check the test_dependencies.py to see how to use this function filters_to_apply = self._check_relationship(relationships, "DATES_FROM", date2str(current_job.date)) - # there could be multiple filters that apply... per example - # Current task date is 20020201, and member is fc2 - # Dummy example, not specially usefull in a real case - # DATES_FROM: - # all: - # MEMBERS_FROM: - # ALL: ... - # CHUNKS_FROM: - # ALL: ... - # 20020201: - # MEMBERS_FROM: - # fc2: - # DATES_TO: "20020201" - # MEMBERS_TO: "fc2" - # CHUNKS_TO: "ALL" - # SPLITS_FROM: - # ALL: - # SPLITS_TO: "1" - # this "for" iterates for ALL and fc2 as current task is selected in both filters - # The dict in this step is: - # [{MEMBERS_FROM{..},CHUNKS_FROM{...}},{MEMBERS_FROM{..},SPLITS_FROM{...}}] for i, filter in enumerate(filters_to_apply): - # {MEMBERS_FROM{..},CHUNKS_FROM{...}} I want too look ALL filters not only one, but I want to go recursivily until get the _TO filter - # This is not an if_else, because the current level ( dates ) could have two different filters. - # Second case commented: ( date_from 20020201 ) - # Will enter, go recursivily to the similar methods and in the end it will do: - # Will enter members_from, and obtain [{DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", CHUNKS_FROM{...}] if "MEMBERS_FROM" in filter: filters_to_apply_m = self._check_members({"MEMBERS_FROM": (filter.pop("MEMBERS_FROM"))}, current_job) if len(filters_to_apply_m) > 0: -- GitLab From 71e52c23cc0f86dd761cb7ee97c1aa13546a8856 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 13 Jul 2023 08:54:12 +0200 Subject: [PATCH 45/68] apply_parent changed to use the same function than in the other place --- autosubmit/job/job_list.py | 40 ++++++---------------------------- test/unit/test_dependencies.py | 5 ++--- 2 files changed, 9 insertions(+), 36 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index f15ae29e6..2bbba5fb0 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -397,44 +397,18 @@ class JobList(object): :return: boolean """ - to_filter = [] - # strip special chars if any filter_value = filter_value.strip("?") - if str(parent_value).lower().find("none") != -1: + if "NONE" in str(parent_value).upper(): return True - if filter_value.lower().find("all") != -1: + to_filter = JobList._parse_filter_to_check(filter_value,associative_list) + if "ALL" in to_filter: return True - elif filter_value.lower().find("natural") != -1: + elif "NATURAL" in to_filter: if parent_value is None or parent_value in associative_list: return True - elif filter_value.lower().find("none") != -1: + elif "NONE" in to_filter: return False - elif filter_value.find(",") != -1: - aux_filter = filter_value.split(",") - if filter_type not in ["chunks", "splits"]: - for value in aux_filter: - if str(value).isdigit(): - to_filter.append(associative_list[int(value)]) - else: - to_filter.append(value) - else: - to_filter = aux_filter - del aux_filter - elif filter_value.find(":") != -1: - start_end = filter_value.split(":") - start = start_end[0].strip("[]") - end = start_end[1].strip("[]") - del start_end - if filter_type not in ["chunks", "splits"]: # chunk directly - for value in range(int(start), int(end) + 1): - to_filter.append(value) - else: # index - for value in range(int(start + 1), int(end) + 1): - to_filter.append(value) - else: - to_filter.append(filter_value) - - if str(parent_value).upper() in str(to_filter).upper(): + elif str(parent_value).upper() in to_filter: return True else: return False @@ -596,7 +570,7 @@ class JobList(object): from_step = relationship.pop("FROM_STEP", relationships.get("FROM_STEP", None)) for filter_range, filter_data in relationship.items(): if filter_range in ["ALL","NATURAL"] or ( not value_to_check or str(value_to_check).upper() in str( - JobList._parse_filters_to_check(filter_range)).upper()): + JobList._parse_filters_to_check(filter_range,values_list)).upper()): if not filter_data.get("STATUS", None): filter_data["STATUS"] = status if not filter_data.get("FROM_STEP", None): diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 007aac369..4b4c19f83 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -297,7 +297,6 @@ class TestJobList(unittest.TestCase): def test_valid_parent(self): # Call the function to get the result - date_list = ["20020201"] member_list = ["fc1", "fc2", "fc3"] chunk_list = [1, 2, 3] @@ -314,7 +313,7 @@ class TestJobList(unittest.TestCase): self.mock_job.member = "fc2" self.mock_job.chunk = 1 self.mock_job.split = 1 - result = self.JobList._valid_parent(self.mock_job, date_list, member_list, chunk_list, is_a_natural_relation, filter_) + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) # it returns a tuple, the first element is the result, the second is the optional flag self.assertEqual(result, True) filter_ = { @@ -323,7 +322,7 @@ class TestJobList(unittest.TestCase): "CHUNKS_TO": "ALL", "SPLITS_TO": "1?" } - result = self.JobList._valid_parent(self.mock_job, date_list, member_list, chunk_list, is_a_natural_relation, filter_) + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) self.assertEqual(result, True) -- GitLab From c47870895be4cb4ad0a31587dbb08203b5d95cfd Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 13 Jul 2023 09:01:20 +0200 Subject: [PATCH 46/68] added additional tests --- autosubmit/job/job_list.py | 15 ++++++++++++--- test/unit/test_dependencies.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 2bbba5fb0..dea212622 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -481,7 +481,10 @@ class JobList(object): # get index in the value_list if len(value_list) > 0: start = value_list.index(start) - end = value_list.index(end) + try: + end = value_list.index(end) + except ValueError: + end = len(value_list)-1 else: start = int(start) end = int(end) @@ -517,7 +520,10 @@ class JobList(object): step = None # get index in the value_list if len(value_list) > 0: - end = value_list.index(end) + try: + end = value_list.index(end) + except ValueError: + end = len(value_list)-1 else: end = int(end) else: # [N:M:S] @@ -528,7 +534,10 @@ class JobList(object): # get index in the value_list if len(value_list) > 0: start = value_list.index(start) - end = value_list.index(end) + try: + end = value_list.index(end) + except ValueError: + end = len(value_list)-1 else: start = int(start) end = int(end) diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 4b4c19f83..6ef7d5161 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -324,6 +324,34 @@ class TestJobList(unittest.TestCase): } result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) self.assertEqual(result, True) + filter_ = { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1?" + } + self.mock_job.split = 2 + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) + self.assertEqual(result, False) + filter_ = { + "DATES_TO": "[20020201:20020205]", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + self.mock_job.split = 1 + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) + self.assertEqual(result, True) + filter_ = { + "DATES_TO": "[20020201:20020205]", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "ALL", + "SPLITS_TO": "1" + } + self.mock_job.date = datetime.strptime("20020206", "%Y%m%d") + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) + self.assertEqual(result, False) + if __name__ == '__main__': -- GitLab From b34d1d0595fc29b388bbb3fc5ad8c89a4a8bcd55 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 13 Jul 2023 12:50:03 +0200 Subject: [PATCH 47/68] added additional tests --- autosubmit/job/job.py | 7 +++++-- autosubmit/job/job_list.py | 27 ++++++++++++++++----------- test/unit/test_dependencies.py | 11 +++++++++++ 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index cdaa45fd7..d301d9759 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1373,7 +1373,8 @@ class Job(object): return parameters def update_job_parameters(self,as_conf, parameters): - parameters["AS_CHECKPOINT"] = self.checkpoint + if self.checkpoint: # To activate placeholder sustitution per in the template + parameters["AS_CHECKPOINT"] = self.checkpoint parameters['JOBNAME'] = self.name parameters['FAIL_COUNT'] = str(self.fail_count) parameters['SDATE'] = self.sdate @@ -1455,6 +1456,8 @@ class Job(object): parameters['EXPORT'] = self.export parameters['PROJECT_TYPE'] = as_conf.get_project_type() self.wchunkinc = as_conf.get_wchunkinc(self.section) + for key,value in as_conf.jobs_data.get(self.section,{}).items(): + parameters["CURRENT_"+key.upper()] = value return parameters def update_parameters(self, as_conf, parameters, @@ -1528,7 +1531,7 @@ class Job(object): template_file.close() else: if self.type == Type.BASH: - template = '%AS_CHECKPOINT%;sleep 320;%AS_CHECKPOINT%;sleep 320' + template = '%CURRENT_TESTNAME%;%AS_CHECKPOINT%;sleep 320;%AS_CHECKPOINT%;sleep 320' elif self.type == Type.PYTHON2: template = 'time.sleep(5)' + "\n" elif self.type == Type.PYTHON3 or self.type == Type.PYTHON: diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index dea212622..d78496883 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -401,14 +401,16 @@ class JobList(object): if "NONE" in str(parent_value).upper(): return True to_filter = JobList._parse_filter_to_check(filter_value,associative_list) - if "ALL" in to_filter: + if len(to_filter) == 0: + return False + elif "ALL".casefold() == str(to_filter[0]).casefold(): return True - elif "NATURAL" in to_filter: + elif "NATURAL" == str(to_filter[0]).casefold(): if parent_value is None or parent_value in associative_list: return True - elif "NONE" in to_filter: + elif "NONE" == str(to_filter[0]).casefold(): return False - elif str(parent_value).upper() in to_filter: + elif str(parent_value).casefold() in ( str(filter_).casefold() for filter_ in to_filter): return True else: return False @@ -448,8 +450,8 @@ class JobList(object): :param value_list: list of values to check. Dates, members, chunks or splits. :return: parsed value to check. """ - - value_to_check = str(value_to_check).upper() + if len(value_list) > 0 and type(value_list[0]) == str: # We dont want to cast split or chunk values + value_to_check = str(value_to_check).upper() if value_to_check.count(":") == 1: # range if value_to_check[1] == ":": @@ -563,16 +565,19 @@ class JobList(object): filters = [] if level_to_check == "DATES_FROM": try: - value_to_check = date2str(value_to_check, "%Y%m%d") + value_to_check = date2str(value_to_check, "%Y%m%d") # need to convert in some cases except: pass - values_list = self._date_list + try: + values_list = [date2str(date_, "%Y%m%d") for date_ in self._date_list] # need to convert in some cases + except: + values_list = self._date_list elif level_to_check == "MEMBERS_FROM": - values_list = self._member_list + values_list = self._member_list # Str list elif level_to_check == "CHUNKS_FROM": - values_list = self._chunk_list + values_list = self._chunk_list # int list else: - values_list = [] # need to obtain the MAX amount of splits set in the workflow + values_list = [] # splits, int list ( artificially generated later ) relationship = relationships.get(level_to_check, {}) status = relationship.pop("STATUS", relationships.get("STATUS", None)) diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 6ef7d5161..9f3fe855c 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -351,6 +351,17 @@ class TestJobList(unittest.TestCase): self.mock_job.date = datetime.strptime("20020206", "%Y%m%d") result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) self.assertEqual(result, False) + filter_ = { + "DATES_TO": "[20020201:20020205]", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "[2:4]", + "SPLITS_TO": "[1:5]" + } + self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") + self.mock_job.chunk = 2 + self.mock_job.split = 1 + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) + self.assertEqual(result, True) -- GitLab From 7191e02508054339e4a53b9dd8eb1583c900905a Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 13 Jul 2023 15:08:00 +0200 Subject: [PATCH 48/68] fix --- test/unit/test_dependencies.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 9f3fe855c..a1d6fbe45 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -300,6 +300,7 @@ class TestJobList(unittest.TestCase): date_list = ["20020201"] member_list = ["fc1", "fc2", "fc3"] chunk_list = [1, 2, 3] + self.mock_job.splits = 10 is_a_natural_relation = False # Filter_to values filter_ = { @@ -331,6 +332,7 @@ class TestJobList(unittest.TestCase): "SPLITS_TO": "1?" } self.mock_job.split = 2 + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) self.assertEqual(result, False) filter_ = { -- GitLab From a72bc815b3a0c9fde1a60de1087b09e9f9e9791d Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 19 Jul 2023 14:56:44 +0200 Subject: [PATCH 49/68] Some rework, may be finally ready to review --- autosubmit/job/job_list.py | 212 +++++++++++++++++---------------- test/unit/test_checkpoints.py | 148 +++++++++++++++++++++++ test/unit/test_dependencies.py | 124 +++++++++++++------ test/unit/test_job_list.py | 1 - 4 files changed, 346 insertions(+), 139 deletions(-) create mode 100644 test/unit/test_checkpoints.py diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index d78496883..90f2448e0 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -387,7 +387,7 @@ class JobList(object): return splits @staticmethod - def _apply_filter(parent_value, filter_value, associative_list, filter_type="dates"): + def _apply_filter(parent_value, filter_value, associative_list, level_to_check="DATES_FROM"): """ Check if the current_job_value is included in the filter_value :param parent_value: @@ -398,17 +398,19 @@ class JobList(object): :return: boolean """ filter_value = filter_value.strip("?") - if "NONE" in str(parent_value).upper(): + if "NONE".casefold() in str(parent_value).casefold(): return True - to_filter = JobList._parse_filter_to_check(filter_value,associative_list) - if len(to_filter) == 0: + to_filter = JobList._parse_filters_to_check(filter_value,associative_list,level_to_check) + if to_filter is None: + return False + elif len(to_filter) == 0: return False elif "ALL".casefold() == str(to_filter[0]).casefold(): return True - elif "NATURAL" == str(to_filter[0]).casefold(): + elif "NATURAL".casefold() == str(to_filter[0]).casefold(): if parent_value is None or parent_value in associative_list: return True - elif "NONE" == str(to_filter[0]).casefold(): + elif "NONE".casefold() == str(to_filter[0]).casefold(): return False elif str(parent_value).casefold() in ( str(filter_).casefold() for filter_ in to_filter): return True @@ -418,27 +420,27 @@ class JobList(object): @staticmethod - def _parse_filters_to_check(list_of_values_to_check,value_list=[]): + def _parse_filters_to_check(list_of_values_to_check,value_list=[],level_to_check="DATES_FROM"): final_values = [] list_of_values_to_check = str(list_of_values_to_check).upper() if list_of_values_to_check is None: return None - elif list_of_values_to_check == "ALL": + elif list_of_values_to_check.casefold() == "ALL".casefold() : return ["ALL"] - elif list_of_values_to_check == "NONE": - return None - elif list_of_values_to_check == "NATURAL": + elif list_of_values_to_check.casefold() == "NONE".casefold(): + return ["NONE"] + elif list_of_values_to_check.casefold() == "NATURAL".casefold(): return ["NATURAL"] elif "," in list_of_values_to_check: for value_to_check in list_of_values_to_check.split(","): - final_values.extend(JobList._parse_filter_to_check(value_to_check,value_list)) + final_values.extend(JobList._parse_filter_to_check(value_to_check,value_list,level_to_check)) else: - final_values = JobList._parse_filter_to_check(list_of_values_to_check,value_list) + final_values = JobList._parse_filter_to_check(list_of_values_to_check,value_list,level_to_check) return final_values @staticmethod - def _parse_filter_to_check(value_to_check,value_list=[]): + def _parse_filter_to_check(value_to_check,value_list=[],level_to_check="DATES_FROM"): """ Parse the filter to check and return the value to check. Selection process: @@ -450,8 +452,7 @@ class JobList(object): :param value_list: list of values to check. Dates, members, chunks or splits. :return: parsed value to check. """ - if len(value_list) > 0 and type(value_list[0]) == str: # We dont want to cast split or chunk values - value_to_check = str(value_to_check).upper() + step = 1 if value_to_check.count(":") == 1: # range if value_to_check[1] == ":": @@ -459,54 +460,32 @@ class JobList(object): # Find N index in the list start = None end = value_to_check.split(":")[1].strip("[]") - # get index in the value_list - if len(value_list) > 0: - end = value_list.index(end) - else: + if level_to_check in ["CHUNKS_FROM","SPLITS_FROM"]: end = int(end) elif value_to_check[-2] == ":": # [N:] # Find N index in the list start = value_to_check.split(":")[0].strip("[]") - end = None - # get index in the value_list - if len(value_list) > 0: - start = value_list.index(start) - else: + if level_to_check in ["CHUNKS_FROM","SPLITS_FROM"]: start = int(start) + end = None else: # [N:M] # Find N index in the list start = value_to_check.split(":")[0].strip("[]") end = value_to_check.split(":")[1].strip("[]") - step = None - # get index in the value_list - if len(value_list) > 0: - start = value_list.index(start) - try: - end = value_list.index(end) - except ValueError: - end = len(value_list)-1 - else: + step = 1 + if level_to_check in ["CHUNKS_FROM","SPLITS_FROM"]: start = int(start) end = int(end) - if end is not None: - end+=1 - if len(value_list) > 0: - return value_list[slice(start, end)] - else: - return [ str(number_gen) for number_gen in range(start, end)] elif value_to_check.count(":") == 2: # range with step if value_to_check[-2] == ":" and value_to_check[-3] == ":": # [N::] # Find N index in the list start = value_to_check.split(":")[0].strip("[]") end = None - step = None - # get index in the value_list - if len(value_list) > 0: - start = value_list.index(start) - else: + step = 1 + if level_to_check in ["CHUNKS_FROM","SPLITS_FROM"]: start = int(start) elif value_to_check[1] == ":" and value_to_check[2] == ":": # [::S] # Find N index in the list @@ -519,40 +498,37 @@ class JobList(object): # Find N index in the list start = None end = value_to_check.split(":")[1].strip("[]") - step = None - # get index in the value_list - if len(value_list) > 0: - try: - end = value_list.index(end) - except ValueError: - end = len(value_list)-1 - else: + if level_to_check in ["CHUNKS_FROM","SPLITS_FROM"]: end = int(end) + step = 1 else: # [N:M:S] # Find N index in the list start = value_to_check.split(":")[0].strip("[]") end = value_to_check.split(":")[1].strip("[]") step = value_to_check.split(":")[2].strip("[]") - # get index in the value_list - if len(value_list) > 0: - start = value_list.index(start) - try: - end = value_list.index(end) - except ValueError: - end = len(value_list)-1 - else: + step = int(step) + if level_to_check in ["CHUNKS_FROM","SPLITS_FROM"]: start = int(start) end = int(end) - step = int(step) - if end is not None: - end+=1 - if len(value_list) > 0: - return value_list[slice(start, end, step)] - else: - return [str(number_gen) for number_gen in range(start, end, step)] else: # value return [value_to_check] + ## values to return + if len(value_list) > 0: + if start is None: + start = value_list[0] + if end is None: + end = value_list[-1] + try: + return value_list[slice(value_list.index(start), value_list.index(end)+1, int(step))] + except ValueError: + return value_list[slice(0,len(value_list)-1,int(step))] + else: + if not start: + start = 0 + if end is None: + return [] + return [number_gen for number_gen in range(int(start), int(end)+1, int(step))] def _check_relationship(self, relationships, level_to_check, value_to_check): """ @@ -583,8 +559,8 @@ class JobList(object): status = relationship.pop("STATUS", relationships.get("STATUS", None)) from_step = relationship.pop("FROM_STEP", relationships.get("FROM_STEP", None)) for filter_range, filter_data in relationship.items(): - if filter_range in ["ALL","NATURAL"] or ( not value_to_check or str(value_to_check).upper() in str( - JobList._parse_filters_to_check(filter_range,values_list)).upper()): + if filter_range.casefold() in ["ALL".casefold(),"NATURAL".casefold()] or ( not value_to_check or str(value_to_check).upper() in str( + JobList._parse_filters_to_check(filter_range,values_list,level_to_check)).upper()): if not filter_data.get("STATUS", None): filter_data["STATUS"] = status if not filter_data.get("FROM_STEP", None): @@ -621,7 +597,7 @@ class JobList(object): if len(filters_to_apply_s) > 0: filters_to_apply[i].update(filters_to_apply_s) # Unify filters from all filters_from where the current job is included to have a single SET of filters_to - filters_to_apply = JobList._unify_to_filters(filters_to_apply) + filters_to_apply = self._unify_to_filters(filters_to_apply) # {DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"} return filters_to_apply @@ -643,7 +619,7 @@ class JobList(object): filters_to_apply_s = self._check_splits({"SPLITS_FROM": (filter_.pop("SPLITS_FROM"))}, current_job) if len(filters_to_apply_s) > 0: filters_to_apply[i].update(filters_to_apply_s) - filters_to_apply = JobList._unify_to_filters(filters_to_apply) + filters_to_apply = self._unify_to_filters(filters_to_apply) return filters_to_apply def _check_chunks(self,relationships, current_job): @@ -660,7 +636,7 @@ class JobList(object): filters_to_apply_s = self._check_splits({"SPLITS_FROM": (filter.pop("SPLITS_FROM"))}, current_job) if len(filters_to_apply_s) > 0: filters_to_apply[i].update(filters_to_apply_s) - filters_to_apply = JobList._unify_to_filters(filters_to_apply) + filters_to_apply = self._unify_to_filters(filters_to_apply) return filters_to_apply def _check_splits(self,relationships, current_job): @@ -673,11 +649,10 @@ class JobList(object): filters_to_apply = self._check_relationship(relationships, "SPLITS_FROM", current_job.split) # No more FROM sections to check, unify _to FILTERS and return - filters_to_apply = JobList._unify_to_filters(filters_to_apply) + filters_to_apply = self._unify_to_filters(filters_to_apply) return filters_to_apply - @staticmethod - def _unify_to_filter(unified_filter, filter_to, filter_type): + def _unify_to_filter(self,unified_filter, filter_to, filter_type): """ Unify filter_to filters into a single dictionary :param unified_filter: Single dictionary with all filters_to @@ -685,21 +660,49 @@ class JobList(object): :param filter_type: "DATES_TO", "MEMBERS_TO", "CHUNKS_TO", "SPLITS_TO" :return: unified_filter """ - if "all" not in unified_filter[filter_type]: + if filter_type == "DATES_TO": + value_list = self._date_list + level_to_check = "DATES_FROM" + elif filter_type == "MEMBERS_TO": + value_list = self._member_list + level_to_check = "MEMBERS_FROM" + elif filter_type == "CHUNKS_TO": + value_list = self._chunk_list + level_to_check = "CHUNKS_FROM" + elif filter_type == "SPLITS_TO": + value_list = self._split_list + level_to_check = "SPLITS_FROM" + if "all".casefold() not in unified_filter[filter_type].casefold(): aux = filter_to.pop(filter_type, None) if aux: aux = aux.split(",") for element in aux: - # Get only the first alphanumeric part - parsed_element = re.findall(r"[\w']+", element)[0].lower() - # Get the rest - data = element[len(parsed_element):] - if parsed_element in ["natural", "none"] and len(unified_filter[filter_type]) > 0: + if element == "": + continue + # Get only the first alphanumeric part and [:] chars + parsed_element = re.findall(r"([\[:\]a-zA-Z0-9]+)", element)[0].lower() + extra_data = element[len(parsed_element):] + parsed_element = JobList._parse_filter_to_check(parsed_element, value_list = value_list, level_to_check = filter_type) + # convert list to str + skip = False + if isinstance(parsed_element, list): + # check if any element is natural or none + for ele in parsed_element: + if ele.lower() in ["natural", "none"]: + skip = True + else: + if parsed_element.lower() in ["natural", "none"]: + skip = True + if skip and len(unified_filter[filter_type]) > 0: continue else: - if "?" not in element: - element += data - unified_filter[filter_type].add(element) + for ele in parsed_element: + if ele not in unified_filter[filter_type]: + if len(unified_filter[filter_type]) > 0 and unified_filter[filter_type][-1] == ",": + unified_filter[filter_type] += ele + extra_data + else: + unified_filter[filter_type] += "," + ele + extra_data + "," + return unified_filter @staticmethod def _normalize_to_filters(filter_to, filter_type): @@ -709,32 +712,35 @@ class JobList(object): :param filter_type: "DATES_TO", "MEMBERS_TO", "CHUNKS_TO", "SPLITS_TO" :return: """ - if len(filter_to[filter_type]) == 0: + if len(filter_to[filter_type]) == 0 or ("," in filter_to[filter_type] and len(filter_to[filter_type]) == 1): filter_to.pop(filter_type, None) - elif "all" in filter_to[filter_type]: + elif "all".casefold() in filter_to[filter_type]: filter_to[filter_type] = "all" else: - # transform to str separated by commas if multiple elements - filter_to[filter_type] = ",".join(filter_to[filter_type]) + # delete last comma + if "," in filter_to[filter_type][-1]: + filter_to[filter_type] = filter_to[filter_type][:-1] + # delete first comma + if "," in filter_to[filter_type][0]: + filter_to[filter_type] = filter_to[filter_type][1:] - @staticmethod - def _unify_to_filters(filter_to_apply): + def _unify_to_filters(self,filter_to_apply): """ Unify all filter_to filters into a single dictionary ( of current selection ) :param filter_to_apply: Filters to apply :return: Single dictionary with all filters_to """ - unified_filter = {"DATES_TO": set(), "MEMBERS_TO": set(), "CHUNKS_TO": set(), "SPLITS_TO": set()} + unified_filter = {"DATES_TO": "", "MEMBERS_TO": "", "CHUNKS_TO": "", "SPLITS_TO": ""} for filter_to in filter_to_apply: if "STATUS" not in unified_filter and filter_to.get("STATUS", None): unified_filter["STATUS"] = filter_to["STATUS"] if "FROM_STEP" not in unified_filter and filter_to.get("FROM_STEP", None): unified_filter["FROM_STEP"] = filter_to["FROM_STEP"] if len(filter_to) > 0: - JobList._unify_to_filter(unified_filter, filter_to, "DATES_TO") - JobList._unify_to_filter(unified_filter, filter_to, "MEMBERS_TO") - JobList._unify_to_filter(unified_filter, filter_to, "CHUNKS_TO") - JobList._unify_to_filter(unified_filter, filter_to, "SPLITS_TO") + self._unify_to_filter(unified_filter, filter_to, "DATES_TO") + self._unify_to_filter(unified_filter, filter_to, "MEMBERS_TO") + self._unify_to_filter(unified_filter, filter_to, "CHUNKS_TO") + self._unify_to_filter(unified_filter, filter_to, "SPLITS_TO") JobList._normalize_to_filters(unified_filter, "DATES_TO") JobList._normalize_to_filters(unified_filter, "MEMBERS_TO") @@ -777,8 +783,6 @@ class JobList(object): elif "SPLITS_FROM" in relationships: filters_to_apply = self._check_splits(relationships, current_job) else: - relationships.pop("OPTIONAL", None) - relationships.pop("CHECKPOINT", None) relationships.pop("CHUNKS_FROM", None) relationships.pop("MEMBERS_FROM", None) relationships.pop("DATES_FROM", None) @@ -2079,17 +2083,19 @@ class JobList(object): if status == "ALL": continue for job in sorted_job_list: + if job.status != Status.WAITING: + continue if status in ["RUNNING", "FAILED"]: if job.platform.connected: # This will be true only when used under setstatus/run job.get_checkpoint_files() for parent in job.edge_info[status].values(): - if parent[0].status == Status.WAITING: - if status in ["RUNNING", "FAILED"] and parent[1] and int(parent[1]) >= job.current_checkpoint_step: - continue - else: - jobs_to_check.append(parent[0]) + if status in ["RUNNING", "FAILED"] and parent[1] and int(parent[1]) >= job.current_checkpoint_step: + continue + else: + jobs_to_check.append(parent[0]) return jobs_to_check + def update_list(self, as_conf, store_change=True, fromSetStatus=False, submitter=None, first_time=False): # type: (AutosubmitConfig, bool, bool, object, bool) -> bool """ diff --git a/test/unit/test_checkpoints.py b/test/unit/test_checkpoints.py new file mode 100644 index 000000000..8772e3ae2 --- /dev/null +++ b/test/unit/test_checkpoints.py @@ -0,0 +1,148 @@ +from unittest import TestCase + +import inspect +import shutil +import tempfile +from mock import Mock +from random import randrange + +from autosubmit.job.job import Job +from autosubmit.job.job_common import Status +from autosubmit.job.job_list import JobList +from autosubmit.job.job_list_persistence import JobListPersistenceDb +from autosubmitconfigparser.config.yamlparser import YAMLParserFactory + + +class TestJobList(TestCase): + def setUp(self): + self.experiment_id = 'random-id' + self.as_conf = Mock() + self.as_conf.experiment_data = dict() + self.as_conf.experiment_data["JOBS"] = dict() + self.as_conf.jobs_data = self.as_conf.experiment_data["JOBS"] + self.as_conf.experiment_data["PLATFORMS"] = dict() + self.temp_directory = tempfile.mkdtemp() + self.job_list = JobList(self.experiment_id, FakeBasicConfig, YAMLParserFactory(), + JobListPersistenceDb(self.temp_directory, 'db'), self.as_conf) + + # creating jobs for self list + self.completed_job = self._createDummyJobWithStatus(Status.COMPLETED) + self.completed_job2 = self._createDummyJobWithStatus(Status.COMPLETED) + self.completed_job3 = self._createDummyJobWithStatus(Status.COMPLETED) + self.completed_job4 = self._createDummyJobWithStatus(Status.COMPLETED) + + self.submitted_job = self._createDummyJobWithStatus(Status.SUBMITTED) + self.submitted_job2 = self._createDummyJobWithStatus(Status.SUBMITTED) + self.submitted_job3 = self._createDummyJobWithStatus(Status.SUBMITTED) + + self.running_job = self._createDummyJobWithStatus(Status.RUNNING) + self.running_job2 = self._createDummyJobWithStatus(Status.RUNNING) + + self.queuing_job = self._createDummyJobWithStatus(Status.QUEUING) + + self.failed_job = self._createDummyJobWithStatus(Status.FAILED) + self.failed_job2 = self._createDummyJobWithStatus(Status.FAILED) + self.failed_job3 = self._createDummyJobWithStatus(Status.FAILED) + self.failed_job4 = self._createDummyJobWithStatus(Status.FAILED) + + self.ready_job = self._createDummyJobWithStatus(Status.READY) + self.ready_job2 = self._createDummyJobWithStatus(Status.READY) + self.ready_job3 = self._createDummyJobWithStatus(Status.READY) + + self.waiting_job = self._createDummyJobWithStatus(Status.WAITING) + self.waiting_job2 = self._createDummyJobWithStatus(Status.WAITING) + + self.unknown_job = self._createDummyJobWithStatus(Status.UNKNOWN) + + + self.job_list._job_list = [self.completed_job, self.completed_job2, self.completed_job3, self.completed_job4, + self.submitted_job, self.submitted_job2, self.submitted_job3, self.running_job, + self.running_job2, self.queuing_job, self.failed_job, self.failed_job2, + self.failed_job3, self.failed_job4, self.ready_job, self.ready_job2, + self.ready_job3, self.waiting_job, self.waiting_job2, self.unknown_job] + self.waiting_job.parents.add(self.ready_job) + self.waiting_job.parents.add(self.completed_job) + self.waiting_job.parents.add(self.failed_job) + self.waiting_job.parents.add(self.submitted_job) + self.waiting_job.parents.add(self.running_job) + self.waiting_job.parents.add(self.queuing_job) + + def tearDown(self) -> None: + shutil.rmtree(self.temp_directory) + + def test_add_edge_job(self): + special_variables = dict() + special_variables["STATUS"] = Status.COMPLETED + special_variables["FROM_STEP"] = 0 + for p in self.waiting_job.parents: + self.waiting_job.add_edge_info(p, special_variables) + for parent in self.waiting_job.parents: + self.assertEqual(self.waiting_job.edge_info[special_variables["STATUS"]][parent.name], + (parent, special_variables.get("FROM_STEP", 0))) + + + def test_add_edge_info_joblist(self): + special_conditions = dict() + special_conditions["STATUS"] = Status.COMPLETED + special_conditions["FROM_STEP"] = 0 + self.job_list._add_edge_info(self.waiting_job, special_conditions["STATUS"]) + self.assertEqual(len(self.job_list.jobs_edges.get(Status.COMPLETED,[])),1) + self.job_list._add_edge_info(self.waiting_job2, special_conditions["STATUS"]) + self.assertEqual(len(self.job_list.jobs_edges.get(Status.COMPLETED,[])),2) + + def test_check_special_status(self): + self.waiting_job.edge_info = dict() + + self.job_list.jobs_edges = dict() + # Adds edge info for waiting_job in the list + self.job_list._add_edge_info(self.waiting_job, Status.COMPLETED) + self.job_list._add_edge_info(self.waiting_job, Status.READY) + self.job_list._add_edge_info(self.waiting_job, Status.RUNNING) + self.job_list._add_edge_info(self.waiting_job, Status.SUBMITTED) + self.job_list._add_edge_info(self.waiting_job, Status.QUEUING) + self.job_list._add_edge_info(self.waiting_job, Status.FAILED) + # Adds edge info for waiting_job + special_variables = dict() + for p in self.waiting_job.parents: + special_variables["STATUS"] = p.status + special_variables["FROM_STEP"] = 0 + self.waiting_job.add_edge_info(p,special_variables) + # call to special status + jobs_to_check = self.job_list.check_special_status() + for job in jobs_to_check: + tmp = [parent for parent in job.parents if + parent.status == Status.COMPLETED or parent in self.jobs_edges["ALL"]] + assert len(tmp) == len(job.parents) + self.waiting_job.add_parent(self.waiting_job2) + for job in jobs_to_check: + tmp = [parent for parent in job.parents if + parent.status == Status.COMPLETED or parent in self.jobs_edges["ALL"]] + assert len(tmp) == len(job.parents) + + + + def _createDummyJobWithStatus(self, status): + job_name = str(randrange(999999, 999999999)) + job_id = randrange(1, 999) + job = Job(job_name, job_id, status, 0) + job.type = randrange(0, 2) + return job + +class FakeBasicConfig: + def __init__(self): + pass + def props(self): + pr = {} + for name in dir(self): + value = getattr(self, name) + if not name.startswith('__') and not inspect.ismethod(value) and not inspect.isfunction(value): + pr[name] = value + return pr + DB_DIR = '/dummy/db/dir' + DB_FILE = '/dummy/db/file' + DB_PATH = '/dummy/db/path' + LOCAL_ROOT_DIR = '/dummy/local/root/dir' + LOCAL_TMP_DIR = '/dummy/local/temp/dir' + LOCAL_PROJ_DIR = '/dummy/local/proj/dir' + DEFAULT_PLATFORMS_CONF = '' + DEFAULT_JOBS_CONF = '' diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index a1d6fbe45..085700ac6 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -44,8 +44,8 @@ class TestJobList(unittest.TestCase): JobListPersistenceDb(self.temp_directory, 'db'), self.as_conf) self.date_list = ["20020201", "20020202", "20020203", "20020204", "20020205", "20020206", "20020207", "20020208", "20020209", "20020210"] self.member_list = ["fc1", "fc2", "fc3", "fc4", "fc5", "fc6", "fc7", "fc8", "fc9", "fc10"] - self.chunk_list = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"] - self.split_list = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"] + self.chunk_list = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + self.split_list = [1, 2, 3, 4, 5] self.JobList._date_list = self.date_list self.JobList._member_list = self.member_list self.JobList._chunk_list = self.chunk_list @@ -58,9 +58,9 @@ class TestJobList(unittest.TestCase): "20020201": { "MEMBERS_FROM": { "fc2": { - "DATES_TO": "20020201", + "DATES_TO": "[20020201:20020202]*,20020203", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL" + "CHUNKS_TO": "all" } }, "SPLITS_FROM": { @@ -72,7 +72,7 @@ class TestJobList(unittest.TestCase): } } self.relationships_dates_optional = deepcopy(self.relationships_dates) - self.relationships_dates_optional["DATES_FROM"]["20020201"]["MEMBERS_FROM"] = { "fc2?": { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", "CHUNKS_TO": "ALL", "SPLITS_TO": "5" } } + self.relationships_dates_optional["DATES_FROM"]["20020201"]["MEMBERS_FROM"] = { "fc2?": { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", "CHUNKS_TO": "all", "SPLITS_TO": "5" } } self.relationships_dates_optional["DATES_FROM"]["20020201"]["SPLITS_FROM"] = { "ALL": { "SPLITS_TO": "1?" } } self.relationships_members = { @@ -82,7 +82,7 @@ class TestJobList(unittest.TestCase): "ALL": { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" } } @@ -94,7 +94,7 @@ class TestJobList(unittest.TestCase): "1": { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" } } @@ -104,7 +104,7 @@ class TestJobList(unittest.TestCase): "1": { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" }, "2": { @@ -122,7 +122,7 @@ class TestJobList(unittest.TestCase): "1": { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" } } @@ -131,7 +131,7 @@ class TestJobList(unittest.TestCase): self.relationships_general = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" } # Create a mock Job object @@ -147,6 +147,36 @@ class TestJobList(unittest.TestCase): self.mock_job.chunk = None self.mock_job.split = None + def test_unify_to_filter(self): + """Test the _unify_to_fitler function""" + # :param unified_filter: Single dictionary with all filters_to + # :param filter_to: Current dictionary that contains the filters_to + # :param filter_type: "DATES_TO", "MEMBERS_TO", "CHUNKS_TO", "SPLITS_TO" + # :return: unified_filter + unified_filter = \ + { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "all", + "SPLITS_TO": "1" + } + filter_to = \ + { + "DATES_TO": "20020205,[20020207:20020208]", + "MEMBERS_TO": "fc2,fc3", + "CHUNKS_TO": "all" + } + filter_type = "DATES_TO" + result = self.JobList._unify_to_filter(unified_filter, filter_to, filter_type) + expected_output = \ + { + "DATES_TO": "20020201,20020205,20020207,20020208,", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "all", + "SPLITS_TO": "1" + } + self.assertEqual(result, expected_output) + def test_simple_dependency(self): result_d = self.JobList._check_dates({}, self.mock_job) result_m = self.JobList._check_members({}, self.mock_job) @@ -202,6 +232,19 @@ class TestJobList(unittest.TestCase): result = self.JobList._parse_filter_to_check("[:20020203:]",self.date_list) expected_output = self.date_list[:3] self.assertEqual(result, expected_output) + # test with a member N:N + result = self.JobList._parse_filter_to_check("[fc2:fc3]",self.member_list) + expected_output = ["fc2","fc3"] + self.assertEqual(result, expected_output) + # test with a chunk + result = self.JobList._parse_filter_to_check("[1:2]",self.chunk_list,level_to_check="CHUNKS_FROM") + expected_output = [1,2] + self.assertEqual(result, expected_output) + # test with a split + result = self.JobList._parse_filter_to_check("[1:2]",self.split_list,level_to_check="SPLITS_FROM") + expected_output = [1,2] + self.assertEqual(result, expected_output) + def test_check_dates(self): # Call the function to get the result @@ -211,15 +254,13 @@ class TestJobList(unittest.TestCase): self.mock_job.split = 1 result = self.JobList._check_dates(self.relationships_dates, self.mock_job) expected_output = { - "DATES_TO": "20020201", + "DATES_TO": "20020201*,20020202*,20020203", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" } self.assertEqual(result, expected_output) - self.mock_job.date = datetime.strptime("20020202", "%Y%m%d") - result = self.JobList._check_dates(self.relationships_dates, self.mock_job) - self.assertEqual(result, {}) + def test_check_members(self): # Call the function to get the result self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") @@ -229,7 +270,7 @@ class TestJobList(unittest.TestCase): expected_output = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" } self.assertEqual(result, expected_output) @@ -237,6 +278,7 @@ class TestJobList(unittest.TestCase): result = self.JobList._check_members(self.relationships_members, self.mock_job) self.assertEqual(result, {}) + def test_check_splits(self): # Call the function to get the result @@ -245,13 +287,14 @@ class TestJobList(unittest.TestCase): expected_output = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" } self.assertEqual(result, expected_output) self.mock_job.split = 2 result = self.JobList._check_splits(self.relationships_splits, self.mock_job) self.assertEqual(result, {}) + def test_check_chunks(self): # Call the function to get the result @@ -260,23 +303,23 @@ class TestJobList(unittest.TestCase): expected_output = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" } self.assertEqual(result, expected_output) self.mock_job.chunk = 2 result = self.JobList._check_chunks(self.relationships_chunks, self.mock_job) self.assertEqual(result, {}) - # test splits_from - self.mock_job.split = 5 - result = self.JobList._check_chunks(self.relationships_chunks2, self.mock_job) - expected_output2 = { - "SPLITS_TO": "2" - } - self.assertEqual(result, expected_output2) - self.mock_job.split = 1 - result = self.JobList._check_chunks(self.relationships_chunks2, self.mock_job) - self.assertEqual(result, {}) + # # test splits_from + # self.mock_job.split = 5 + # result = self.JobList._check_chunks(self.relationships_chunks2, self.mock_job) + # expected_output2 = { + # "SPLITS_TO": "2" + # } + # self.assertEqual(result, expected_output2) + # self.mock_job.split = 1 + # result = self.JobList._check_chunks(self.relationships_chunks2, self.mock_job) + # self.assertEqual(result, {}) def test_check_general(self): # Call the function to get the result @@ -289,7 +332,7 @@ class TestJobList(unittest.TestCase): expected_output = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" } self.assertEqual(result, expected_output) @@ -297,7 +340,7 @@ class TestJobList(unittest.TestCase): def test_valid_parent(self): # Call the function to get the result - date_list = ["20020201"] + date_list = ["20020201", "20020202", "20020203", "20020204", "20020205", "20020206", "20020207", "20020208", "20020209", "20020210"] member_list = ["fc1", "fc2", "fc3"] chunk_list = [1, 2, 3] self.mock_job.splits = 10 @@ -306,7 +349,7 @@ class TestJobList(unittest.TestCase): filter_ = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" } # PArent job values @@ -320,7 +363,7 @@ class TestJobList(unittest.TestCase): filter_ = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1?" } result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) @@ -328,7 +371,7 @@ class TestJobList(unittest.TestCase): filter_ = { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1?" } self.mock_job.split = 2 @@ -338,7 +381,7 @@ class TestJobList(unittest.TestCase): filter_ = { "DATES_TO": "[20020201:20020205]", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" } self.mock_job.split = 1 @@ -347,7 +390,7 @@ class TestJobList(unittest.TestCase): filter_ = { "DATES_TO": "[20020201:20020205]", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "ALL", + "CHUNKS_TO": "all", "SPLITS_TO": "1" } self.mock_job.date = datetime.strptime("20020206", "%Y%m%d") @@ -364,6 +407,17 @@ class TestJobList(unittest.TestCase): self.mock_job.split = 1 result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) self.assertEqual(result, True) + filter_ = { + "DATES_TO": "[20020201:20020202],20020203,20020204,20020205", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "[2:4]*,5", + "SPLITS_TO": "[1:5],6" + } + self.mock_job.date = datetime.strptime("20020204", "%Y%m%d") + self.mock_job.chunk = 5 + self.mock_job.split = 6 + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) + self.assertEqual(result, True) diff --git a/test/unit/test_job_list.py b/test/unit/test_job_list.py index 0a3f6b3b4..e546b764d 100644 --- a/test/unit/test_job_list.py +++ b/test/unit/test_job_list.py @@ -275,7 +275,6 @@ class TestJobList(TestCase): job.type = randrange(0, 2) return job -import inspect class FakeBasicConfig: def __init__(self): pass -- GitLab From cd54874f9317c7b01610500d1952553392fdc6f9 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 19 Jul 2023 15:56:56 +0200 Subject: [PATCH 50/68] Added failure tests --- test/unit/test_dependencies.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 085700ac6..175a6e3e9 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -260,6 +260,11 @@ class TestJobList(unittest.TestCase): "SPLITS_TO": "1" } self.assertEqual(result, expected_output) + # failure + self.mock_job.date = datetime.strptime("20020301", "%Y%m%d") + result = self.JobList._check_dates(self.relationships_dates, self.mock_job) + self.assertEqual(result, {}) + def test_check_members(self): # Call the function to get the result @@ -277,6 +282,10 @@ class TestJobList(unittest.TestCase): self.mock_job.member = "fc3" result = self.JobList._check_members(self.relationships_members, self.mock_job) self.assertEqual(result, {}) + # FAILURE + self.mock_job.member = "fc99" + result = self.JobList._check_members(self.relationships_members, self.mock_job) + self.assertEqual(result, {}) def test_check_splits(self): @@ -294,6 +303,10 @@ class TestJobList(unittest.TestCase): self.mock_job.split = 2 result = self.JobList._check_splits(self.relationships_splits, self.mock_job) self.assertEqual(result, {}) + # failure + self.mock_job.split = 99 + result = self.JobList._check_splits(self.relationships_splits, self.mock_job) + self.assertEqual(result, {}) def test_check_chunks(self): # Call the function to get the result @@ -310,16 +323,13 @@ class TestJobList(unittest.TestCase): self.mock_job.chunk = 2 result = self.JobList._check_chunks(self.relationships_chunks, self.mock_job) self.assertEqual(result, {}) - # # test splits_from - # self.mock_job.split = 5 - # result = self.JobList._check_chunks(self.relationships_chunks2, self.mock_job) - # expected_output2 = { - # "SPLITS_TO": "2" - # } - # self.assertEqual(result, expected_output2) - # self.mock_job.split = 1 - # result = self.JobList._check_chunks(self.relationships_chunks2, self.mock_job) - # self.assertEqual(result, {}) + # failure + self.mock_job.chunk = 99 + result = self.JobList._check_chunks(self.relationships_chunks, self.mock_job) + self.assertEqual(result, {}) + + + def test_check_general(self): # Call the function to get the result -- GitLab From 82c0cc719437d667c9bb1bc2f9287395b99e13d3 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 20 Jul 2023 08:46:24 +0200 Subject: [PATCH 51/68] Ready --- autosubmit/job/job_list.py | 29 ++++++++++++++++------ test/unit/test_dependencies.py | 45 +++++++++++++++++++++++++--------- 2 files changed, 56 insertions(+), 18 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 90f2448e0..b7918dfcd 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -412,7 +412,7 @@ class JobList(object): return True elif "NONE".casefold() == str(to_filter[0]).casefold(): return False - elif str(parent_value).casefold() in ( str(filter_).casefold() for filter_ in to_filter): + elif len( [ filter_ for filter_ in to_filter if str(parent_value).casefold() in str(filter_).casefold() ] )>0: return True else: return False @@ -791,7 +791,7 @@ class JobList(object): return filters_to_apply @staticmethod - def _valid_parent(parent, member_list, date_list, chunk_list, is_a_natural_relation, filter_): + def _valid_parent(parent, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child): ''' Check if the parent is valid for the current job :param parent: job to check @@ -815,6 +815,7 @@ class JobList(object): members_to = str(filter_.get("MEMBERS_TO", "natural")).lower() chunks_to = str(filter_.get("CHUNKS_TO", "natural")).lower() splits_to = str(filter_.get("SPLITS_TO", "natural")).lower() + if not is_a_natural_relation: if dates_to == "natural": dates_to = "none" @@ -824,15 +825,29 @@ class JobList(object): chunks_to = "none" if splits_to == "natural": splits_to = "none" - if dates_to == "natural": + if "natural" in dates_to: associative_list["dates"] = [date2str(parent.date)] if parent.date is not None else date_list - if members_to == "natural": + if "natural" in members_to: associative_list["members"] = [parent.member] if parent.member is not None else member_list - if chunks_to == "natural": + if "natural" in chunks_to: associative_list["chunks"] = [parent.chunk] if parent.chunk is not None else chunk_list - if splits_to == "natural": + if "natural" in splits_to: associative_list["splits"] = [parent.split] if parent.split is not None else parent.splits parsed_parent_date = date2str(parent.date) if parent.date is not None else None + # Check for each * char in the filters + # Get all the dates that match * in the filter in a list separated by , + if "*" in dates_to: + dates_to = [ dat for dat in date_list.split(",") if dat is not None and "*" not in dat or ("*" in dat and date2str(child.date,"%Y%m%d") in dat) ] + dates_to = ",".join(dates_to) + if "*" in members_to: + members_to = [ mem for mem in member_list.split(",") if mem is not None and "*" not in mem or ("*" in mem and str(child.member) in mem) ] + members_to = ",".join(members_to) + if "*" in chunks_to: + chunks_to = [ chu for chu in chunk_list.split(",") if chu is not None and "*" not in chu or ("*" in chu and str(child.chunk) in chu) ] + chunks_to = ",".join(chunks_to) + if "*" in splits_to: + splits_to = [ spl for spl in splits_to.split(",") if child.split is None or spl is None or "*" not in spl or ("*" in spl and str(child.split) in spl) ] + splits_to = ",".join(splits_to) # Apply all filters to look if this parent is an appropriated candidate for the current_job valid_dates = JobList._apply_filter(parsed_parent_date, dates_to, associative_list["dates"], "dates") valid_members = JobList._apply_filter(parent.member, members_to, associative_list["members"], "members") @@ -913,7 +928,7 @@ class JobList(object): # If the parent is valid, add it to the graph if JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship, - filters_to_apply): + filters_to_apply,child): job.add_parent(parent) self._add_edge(graph, job, parent) # Could be more variables in the future diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 175a6e3e9..dbf93565e 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -1,3 +1,4 @@ +import copy import inspect import mock import tempfile @@ -134,6 +135,12 @@ class TestJobList(unittest.TestCase): "CHUNKS_TO": "all", "SPLITS_TO": "1" } + self.relationships_general_1_to_1 = { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "all", + "SPLITS_TO": "1*,2*,3*,4*,5*" + } # Create a mock Job object self.mock_job = mock.MagicMock(spec=Job) @@ -349,6 +356,7 @@ class TestJobList(unittest.TestCase): def test_valid_parent(self): + child = copy.deepcopy(self.mock_job) # Call the function to get the result date_list = ["20020201", "20020202", "20020203", "20020204", "20020205", "20020206", "20020207", "20020208", "20020209", "20020210"] member_list = ["fc1", "fc2", "fc3"] @@ -367,7 +375,7 @@ class TestJobList(unittest.TestCase): self.mock_job.member = "fc2" self.mock_job.chunk = 1 self.mock_job.split = 1 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) # it returns a tuple, the first element is the result, the second is the optional flag self.assertEqual(result, True) filter_ = { @@ -376,7 +384,7 @@ class TestJobList(unittest.TestCase): "CHUNKS_TO": "all", "SPLITS_TO": "1?" } - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) self.assertEqual(result, True) filter_ = { "DATES_TO": "20020201", @@ -386,7 +394,7 @@ class TestJobList(unittest.TestCase): } self.mock_job.split = 2 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) self.assertEqual(result, False) filter_ = { "DATES_TO": "[20020201:20020205]", @@ -395,7 +403,7 @@ class TestJobList(unittest.TestCase): "SPLITS_TO": "1" } self.mock_job.split = 1 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) self.assertEqual(result, True) filter_ = { "DATES_TO": "[20020201:20020205]", @@ -404,7 +412,7 @@ class TestJobList(unittest.TestCase): "SPLITS_TO": "1" } self.mock_job.date = datetime.strptime("20020206", "%Y%m%d") - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) self.assertEqual(result, False) filter_ = { "DATES_TO": "[20020201:20020205]", @@ -415,20 +423,35 @@ class TestJobList(unittest.TestCase): self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") self.mock_job.chunk = 2 self.mock_job.split = 1 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) self.assertEqual(result, True) + + + def test_valid_parent_1_to_1(self): + child = copy.deepcopy(self.mock_job) + + date_list = ["20020201", "20020202", "20020203", "20020204", "20020205", "20020206", "20020207", "20020208", "20020209", "20020210"] + member_list = ["fc1", "fc2", "fc3"] + chunk_list = [1, 2, 3] + is_a_natural_relation = False + + # Test 1_to_1 filter_ = { "DATES_TO": "[20020201:20020202],20020203,20020204,20020205", "MEMBERS_TO": "fc2", - "CHUNKS_TO": "[2:4]*,5", - "SPLITS_TO": "[1:5],6" + "CHUNKS_TO": "1,2,3,4,5,6", + "SPLITS_TO": "1*,2*,3*,4*,5*,6" } + self.mock_job.split = 1 self.mock_job.date = datetime.strptime("20020204", "%Y%m%d") self.mock_job.chunk = 5 - self.mock_job.split = 6 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_) + child.split = 1 + self.mock_job.split = 1 + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) self.assertEqual(result, True) - + child.split = 2 + result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) + self.assertEqual(result, False) if __name__ == '__main__': -- GitLab From 43cec285501504b7f5316d13b0fee4b3854e6bd1 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 20 Jul 2023 08:52:53 +0200 Subject: [PATCH 52/68] redo the split section TODO image --- .../userguide/defining_workflows/index.rst | 67 ++++++++++++++++--- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/docs/source/userguide/defining_workflows/index.rst b/docs/source/userguide/defining_workflows/index.rst index 35b4e3859..5ca0d27df 100644 --- a/docs/source/userguide/defining_workflows/index.rst +++ b/docs/source/userguide/defining_workflows/index.rst @@ -204,6 +204,47 @@ For the new format, consider that the priority is hierarchy and goes like this D * You can define a MEMBERS_FROM inside the DEPENDENCY and DEPENDENCY.DATES_FROM. * You can define a CHUNKS_FROM inside the DEPENDENCY, DEPENDENCY.DATES_FROM, DEPENDENCY.MEMBERS_FROM, DEPENDENCY.DATES_FROM.MEMBERS_FROM +Start conditions +~~~~~~~~~~~~~~~~ + +Sometimes you want to run a job only when a certain condition is met. For example, you may want to run a job only when a certain task is running. +This can be achieved using the START_CONDITIONS feature based on the dependencies rework. + +Start conditions are achieved by adding the keyword `STATUS` and optionally `FROM_STEP` keywords into any dependency that you want. + +The `STATUS` keyword can be used to select the status of the dependency that you want to check. The possible values are: + +* `running`: The dependency must be running. +* `completed`: The dependency must have completed. +* `failed`: The dependency must have failed. +* `queuing`: The dependency must be queuing. +* `submitted`: The dependency must have been submitted. +* `ready`: The dependency must be ready to be submitted. + +The `FROM_STEP` keyword can be used to select the step of the dependency that you want to check. The possible value is an integer. +Additionally, the target dependency, must call to %AS_CHECKPOINT% inside their scripts. This will create a checkpoint that will be used to check the amount of steps processed. + +.. code-block:: yaml + + ini: + FILE: ini.sh + RUNNING: member + + sim: + FILE: sim.sh + DEPENDENCIES: ini sim-1 + RUNNING: chunk + + postprocess: + FILE: postprocess.sh + DEPENDENCIES: + SIM: + STATUS: "RUNNING" + FROM_STEP: 0 + RUNNING: chunk + + + Job frequency ~~~~~~~~~~~~~ @@ -318,14 +359,12 @@ The resulting workflow of setting SYNCHRONIZE parameter to 'date' can be seen in Job split ~~~~~~~~~ -For jobs running at chunk level, it may be useful to split each chunk into different parts. +For jobs running at any level, it may be useful to split each task into different parts. This behaviour can be achieved using the SPLITS attribute to specify the number of parts. -It is possible to define dependencies to specific splits within [], as well as to a list/range of splits, -in the format [1:3,7,10] or [1,2,3] +It is also possible to specify the splits for each task using the SPLITS_FROM and SPLITS_TO attributes. -.. hint:: - This job parameter works with jobs with RUNNING parameter equals to 'chunk'. +There is also an special character '*' that can be used to specify that the split is 1-to-1 dependency. .. code-block:: yaml @@ -347,15 +386,27 @@ in the format [1:3,7,10] or [1,2,3] post: FILE: post.sh RUNNING: chunk - DEPENDENCIES: asim1: asim1:+1 + DEPENDENCIES: + asim: + SPLITS_FROM: + 2,3: + splits_to: 1,2*,3* + SPLITS: 2 + +In this example: + +Post job will be split into 2 parts. +Each part will depend on the 1st part of the asim job. +The 2nd part of the post job will depend on the 2nd part of the asim job. +The 3rd part of the post job will depend on the 3rd part of the asim job. The resulting workflow can be seen in Figure :numref:`split` -.. figure:: fig/split.png +.. figure:: fig/split_todo.png :name: split :width: 100% :align: center - :alt: simple workflow plot + :alt: TODO Example showing the job ASIM divided into 3 parts for each chunk. -- GitLab From 44f6cc154dce8988245e1d90a0431463f3de0d27 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 24 Jul 2023 16:04:57 +0200 Subject: [PATCH 53/68] fix offset --- autosubmit/job/job.py | 2 +- autosubmit/job/job_list.py | 25 +++++++++++++------------ autosubmit/platforms/slurmplatform.py | 10 ++++------ 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index d301d9759..051d6b0d5 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1531,7 +1531,7 @@ class Job(object): template_file.close() else: if self.type == Type.BASH: - template = '%CURRENT_TESTNAME%;%AS_CHECKPOINT%;sleep 320;%AS_CHECKPOINT%;sleep 320' + template = 'sleep 5' elif self.type == Type.PYTHON2: template = 'time.sleep(5)' + "\n" elif self.type == Type.PYTHON3 or self.type == Type.PYTHON: diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index b7918dfcd..374c5619d 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -398,6 +398,7 @@ class JobList(object): :return: boolean """ filter_value = filter_value.strip("?") + filter_value = filter_value.strip("*") if "NONE".casefold() in str(parent_value).casefold(): return True to_filter = JobList._parse_filters_to_check(filter_value,associative_list,level_to_check) @@ -412,7 +413,7 @@ class JobList(object): return True elif "NONE".casefold() == str(to_filter[0]).casefold(): return False - elif len( [ filter_ for filter_ in to_filter if str(parent_value).casefold() in str(filter_).casefold() ] )>0: + elif len( [ filter_ for filter_ in to_filter if str(parent_value).casefold() == str(filter_).casefold() ] )>0: return True else: return False @@ -670,7 +671,7 @@ class JobList(object): value_list = self._chunk_list level_to_check = "CHUNKS_FROM" elif filter_type == "SPLITS_TO": - value_list = self._split_list + value_list = [] level_to_check = "SPLITS_FROM" if "all".casefold() not in unified_filter[filter_type].casefold(): aux = filter_to.pop(filter_type, None) @@ -688,20 +689,20 @@ class JobList(object): if isinstance(parsed_element, list): # check if any element is natural or none for ele in parsed_element: - if ele.lower() in ["natural", "none"]: + if type(ele) is str and ele.lower() in ["natural", "none"]: skip = True else: - if parsed_element.lower() in ["natural", "none"]: + if type(parsed_element) is str and parsed_element.lower() in ["natural", "none"]: skip = True if skip and len(unified_filter[filter_type]) > 0: continue else: for ele in parsed_element: - if ele not in unified_filter[filter_type]: + if str(ele) not in unified_filter[filter_type]: if len(unified_filter[filter_type]) > 0 and unified_filter[filter_type][-1] == ",": - unified_filter[filter_type] += ele + extra_data + unified_filter[filter_type] += str(ele) + extra_data else: - unified_filter[filter_type] += "," + ele + extra_data + "," + unified_filter[filter_type] += "," + str(ele) + extra_data + "," return unified_filter @staticmethod @@ -837,16 +838,16 @@ class JobList(object): # Check for each * char in the filters # Get all the dates that match * in the filter in a list separated by , if "*" in dates_to: - dates_to = [ dat for dat in date_list.split(",") if dat is not None and "*" not in dat or ("*" in dat and date2str(child.date,"%Y%m%d") in dat) ] + dates_to = [ dat for dat in date_list.split(",") if dat is not None and "*" not in dat or ("*" in dat and date2str(child.date,"%Y%m%d") == dat.split("*")[0]) ] dates_to = ",".join(dates_to) if "*" in members_to: - members_to = [ mem for mem in member_list.split(",") if mem is not None and "*" not in mem or ("*" in mem and str(child.member) in mem) ] + members_to = [ mem for mem in member_list.split(",") if mem is not None and "*" not in mem or ("*" in mem and str(child.member) == mem.split("*")[0]) ] members_to = ",".join(members_to) if "*" in chunks_to: - chunks_to = [ chu for chu in chunk_list.split(",") if chu is not None and "*" not in chu or ("*" in chu and str(child.chunk) in chu) ] + chunks_to = [ chu for chu in chunk_list.split(",") if chu is not None and "*" not in chu or ("*" in chu and str(child.chunk) == chu.split("*")[0]) ] chunks_to = ",".join(chunks_to) if "*" in splits_to: - splits_to = [ spl for spl in splits_to.split(",") if child.split is None or spl is None or "*" not in spl or ("*" in spl and str(child.split) in spl) ] + splits_to = [ spl for spl in splits_to.split(",") if child.split is None or spl is None or "*" not in spl or ("*" in spl and str(child.split) == spl.split("*")[0]) ] splits_to = ",".join(splits_to) # Apply all filters to look if this parent is an appropriated candidate for the current_job valid_dates = JobList._apply_filter(parsed_parent_date, dates_to, associative_list["dates"], "dates") @@ -928,7 +929,7 @@ class JobList(object): # If the parent is valid, add it to the graph if JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship, - filters_to_apply,child): + filters_to_apply,job): job.add_parent(parent) self._add_edge(graph, job, parent) # Could be more variables in the future diff --git a/autosubmit/platforms/slurmplatform.py b/autosubmit/platforms/slurmplatform.py index e867ff062..a64d386e8 100644 --- a/autosubmit/platforms/slurmplatform.py +++ b/autosubmit/platforms/slurmplatform.py @@ -18,18 +18,16 @@ # along with Autosubmit. If not, see . import locale import os -from contextlib import suppress -from time import sleep +from datetime import datetime from time import mktime +from time import sleep from time import time -from datetime import datetime from typing import List, Union - from xml.dom.minidom import parseString from autosubmit.job.job_common import Status, parse_output_number -from autosubmit.platforms.paramiko_platform import ParamikoPlatform from autosubmit.platforms.headers.slurm_header import SlurmHeader +from autosubmit.platforms.paramiko_platform import ParamikoPlatform from autosubmit.platforms.wrappers.wrapper_factory import SlurmWrapperFactory from log.log import AutosubmitCritical, AutosubmitError, Log @@ -88,8 +86,8 @@ class SlurmPlatform(ParamikoPlatform): try: jobs_id = self.submit_Script(hold=hold) except AutosubmitError as e: - Log.error(f'TRACE:{e.trace}\n{e.message}') jobnames = [job.name for job in valid_packages_to_submit[0].jobs] + Log.error(f'TRACE:{e.trace}\n{e.message} JOBS:{jobnames}') for jobname in jobnames: jobid = self.get_jobid_by_jobname(jobname) #cancel bad submitted job if jobid is encountered -- GitLab From 1093285b4df5cb3b4d1fda7bfc13c8a87ab9778d Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 25 Jul 2023 16:01:50 +0200 Subject: [PATCH 54/68] Fixed few things , fixed docs, fixed and modify some tests --- autosubmit/job/job_common.py | 6 +- autosubmit/job/job_list.py | 33 ++++++----- .../defining_workflows/fig/splits_1_to_1.png | Bin 0 -> 15019 bytes .../userguide/defining_workflows/index.rst | 45 +++++++++------ test/unit/test_checkpoints.py | 52 +++++++++++++----- 5 files changed, 85 insertions(+), 51 deletions(-) create mode 100644 docs/source/userguide/defining_workflows/fig/splits_1_to_1.png diff --git a/autosubmit/job/job_common.py b/autosubmit/job/job_common.py index 042c6e330..f6d34ccff 100644 --- a/autosubmit/job/job_common.py +++ b/autosubmit/job/job_common.py @@ -13,11 +13,10 @@ # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. - # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . -import textwrap import datetime +import textwrap class Status: @@ -41,6 +40,7 @@ class Status: # Note: any change on constants must be applied on the dict below!!! VALUE_TO_KEY = {-3: 'SUSPENDED', -2: 'UNKNOWN', -1: 'FAILED', 0: 'WAITING', 1: 'READY', 2: 'SUBMITTED', 3: 'QUEUING', 4: 'RUNNING', 5: 'COMPLETED', 6: 'HELD', 7: 'PREPARED', 8: 'SKIPPED', 9: 'DELAYED'} + LOGICAL_ORDER = ["WAITING", "DELAYED", "PREPARED", "READY", "SUBMITTED", "HELD", "QUEUING", "RUNNING", "SKIPPED", "FAILED", "UNKNOWN", "COMPLETED", "SUSPENDED"] def retval(self, value): return getattr(self, value) @@ -131,7 +131,7 @@ class StatisticsSnippetBash: ################### # AS CHECKPOINT FUNCTION ################### - # Creates a new checkpoint file upton call based on the current numbers of calls to the function + # Creates a new checkpoint file upon call based on the current numbers of calls to the function AS_CHECKPOINT_CALLS=0 function as_checkpoint { diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 374c5619d..b642fcc2b 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -13,14 +13,13 @@ # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. - +# You should have received a copy of the GNU General Public License +# along with Autosubmit. If not, see . import copy import datetime import math import os import pickle -# You should have received a copy of the GNU General Public License -# along with Autosubmit. If not, see . import re import traceback from bscearth.utils.date import date2str, parse_date @@ -413,7 +412,7 @@ class JobList(object): return True elif "NONE".casefold() == str(to_filter[0]).casefold(): return False - elif len( [ filter_ for filter_ in to_filter if str(parent_value).casefold() == str(filter_).casefold() ] )>0: + elif len( [ filter_ for filter_ in to_filter if str(parent_value).casefold() == str(filter_).strip("*").strip("?").casefold() ] )>0: return True else: return False @@ -2102,13 +2101,21 @@ class JobList(object): if job.status != Status.WAITING: continue if status in ["RUNNING", "FAILED"]: + # check checkpoint if any if job.platform.connected: # This will be true only when used under setstatus/run job.get_checkpoint_files() + non_completed_parents_current = 0 + completed_parents = len([parent for parent in job.parents if parent.status == Status.COMPLETED]) for parent in job.edge_info[status].values(): if status in ["RUNNING", "FAILED"] and parent[1] and int(parent[1]) >= job.current_checkpoint_step: continue else: - jobs_to_check.append(parent[0]) + status_str = Status.VALUE_TO_KEY[parent[0].status] + if Status.LOGICAL_ORDER.index(status_str) >= Status.LOGICAL_ORDER.index(status): + non_completed_parents_current += 1 + if ( non_completed_parents_current + completed_parents ) == len(job.parents): + jobs_to_check.append(job) + return jobs_to_check @@ -2185,16 +2192,12 @@ class JobList(object): save = True # Check checkpoint jobs, the status can be Any for job in self.check_special_status(): - # Check if all jobs fullfill the conditions to a job be ready - tmp = [parent for parent in job.parents if - parent.status == Status.COMPLETED or parent in self.jobs_edges["ALL"]] - if len(tmp) == len(job.parents): - job.status = Status.READY - job.id = None - job.packed = False - job.wrapper_type = None - save = True - Log.debug(f"Special condition fullfilled for job {job.name}") + job.status = Status.READY + job.id = None + job.packed = False + job.wrapper_type = None + save = True + Log.debug(f"Special condition fullfilled for job {job.name}") # if waiting jobs has all parents completed change its State to READY for job in self.get_completed(): if job.synchronize is not None and len(str(job.synchronize)) > 0: diff --git a/docs/source/userguide/defining_workflows/fig/splits_1_to_1.png b/docs/source/userguide/defining_workflows/fig/splits_1_to_1.png new file mode 100644 index 0000000000000000000000000000000000000000..0e85db4e25fbbf3ba36316155235d02377299630 GIT binary patch literal 15019 zcmbWe2{e{#zdx>0nL-;8GBi*~AtF;GLYaj!hmPz0ZE%^Lx)eYn}Cf*4le}?8p6F_jP@T&+xqil@w(5ky(FbVK(HeL z|L73y#BV-iY%bzIyKE(8)rg3QJ`X8=#al|di<)+-Rwj0i1~$e7s;2gKcE&b0ygm{W z5U>!)N=c|W^-X`drA;+9aCVNEl8CVSngH#2!d@f$w4FP@k*Hmi%8<*ymLZy-S9nt0 zRYWSpgIw)(ZN8mEttCl*Hvu;b2@z4idyWTpubx-lv4iS~&xkWmYivZ!F~4K1Z)ex# zvdE$vm*Q8ApEPSz-#dSvfFMYhWPB&V`Q{x2ETdSREdfD*5CK6ABLRUSIROFB-v4}i zH)_|vK8^qT-{1bnAOHQ<`IQehUG2AT-z>kB@K1`5zPR8|O+Dwk±NaOYyRX}-nh zf;5e+zTV#8Zi{=eJ6B85f-@kWV{-La)!Jd4}!lK-Ms{1jg z#-rioz1V@cmMujYe8pcXZL(6!YY$osz3f%TX*X&9Q1IVXn#5OAy!^aPLdU&p~6Z@ z(@kH`lsotzi{GBTH5+;&&Bb!nJ=XjOd`icHXik} z$Gd$SA9tGjTGigJ@@N;O?dio&<_=m~y?M7f#_dYd(#({V_e)UXP7?0CN>9H`N$CG> zBJE3Or`_uO_g&WsE?%s4UtgY>oZMW9-!eBhm%50xT)3>GBg(_0EF&}a{xK(RS%fKM zWzg0@M@MJmj5c@Vvz#mkFBYGh9eeIp=`#vi-)%9O?alRfQ$(Ol>2JhvkXC~6JDmYG@5V9MmfH?-hB< zksT+TwJ&cberCO`q!gN*d=o3y^W2FT_lOa)eH9ls@ajs;^6a3Mg~gY)wmaWyi0$o{ zXgIHyEp=Z{R*c_V>e1^pU+&Jh-ZGAuV1f}Q3PZ_-Qoaw2kE>KFz{rSQs zw~BAM{hIkQ)syvLvoTW7IVgkiDz5!#^h{CtciS_4MM)Y)NsrN9g?D$;!%3EWO-KdVs7^Q4w|O z^fNlLI=_7kA1PS;tzXjoQ|0=*HvLy2+kP9mt0>w2k-VZJtyr(y;>46h96>p`lgnwcV@TG9hD&;SdUm@TD4xaqM}_%!K}5;> z`%C|!n*Lo7|DW>o??3*JYMSJ8n3Cut>1lm)f$h9Qf{%DUxPPDRa@6zGR9Q+NFR$tX zi_hKZTHd-|Loa=}?^ZqUn84V^>9W@!MRD33Qk9#L$UUfj>ondKp^{%+-0|yQ` zI5>@Gy~KGcM=@^TvL&DN66wzfAeOSkRF5wDLv zWgB?sNK8yjO+DeV2PYu6vbq|mo1>ws`Z@0wRmpDr$_0UcQl~^sc4AAf$jZu! zf{5sRAv%flsjaU+rJ4Qwv(>oK5e^PiB~nf_kUA1R zK0a(&Z|kq&5b*<>=~8uP&YV$l9H;rvQWff9!7=oJ>K_j6KN{p;Cw{Ma&Z$f$gy{vd zzSP@80dsn{ZrwU~@F4nzpP!#uPbN)A+|#Eeo?DwMD^BNPxkvXEYtKw{B%9s7t){O2 zxv!7wn#ae-LqkIk2|J*^)YaAo-g30H6~6P$XC>$e(P0jb z`|pN+d?|Ir2T`+xgoI*aV@F3vIW;o5I5>Fu_+o?|xqT(y6c*|gSsN%Tx238kyNF$$ zsP-k@y?b{_Ny)&#K!YF-wcAa z{4R-&J+x|dF1F%EJ^B3L8Tw%J{R*isUJQNxs)W1_4?ny{j*zact+hM%J2;F4HEau6 zFDEZA>^P&1NVs{EU4qiy-X6J#C5{l{UyS7Jr%q*MIS9nWhKA17-K0)SNs*F~>FMkY z-HH{w^KGOtV*RO8U*0YBy~^LqF{U>pCA~sIDCG782#xQ(cb<}wvA@6n$JCT78VOp+ zenMq+^^24~bNAkj9N|byOLMiTMECuxRW*z))V7)}jJ9A6F#;C%0|O_<$61J~Yilny zF9NP)zJ6WezAmt;Cl((c7dJCBbno>JL-MtIEhdqVA76QR>CQ8CPtMkT;Z+04z?>T(r$Pp2d{_gJX zuCB)-l_sX9$NQ6y5GA~P>B~ZNP%JLvu96z2s7Z2;il2g zZ?#XKKCRqbn<{txm7bp7@PG!WV%#6eo1LAF5G1BzBVsto#YIL&*1&hiNJKeRjg~fY z`+0$nBiN@V zCc^esj{M{etRD^wCX`rNU+3iH1X>~8%RV(Zc{qz`7unsz5sgx(g0`EMp6BnS$*we% zBiGpT8X9Mh9s7>JM~5dTCr?dHZE0y?Vqy{y5I`z?|Ni~hv133y{T1%Q&NG93eI?h| zn<7u$xN!qfG+|eQHXqeihK#bcv3cLrG&MC95fL%_rL^#7Yus3Ce0^OVLR5gCpNEc2 zRkZ`qxTvUTW5W%rprWFp5AH^Wqd4oIMn-V!mPN(HtWS1v;_mJY*Dcv!Jj~DERZ~M4 zY5aoyz4g9Ua+P*$Lvr%3v3Sq>2a_K+eJ>~|K&(&_wsm$MJ!gF3!Y*G)R=vsPg7Sg~ zJ?-t)`lp}9#F&|x<>%!A_#CIc|43}5!hIdNF_*tjbSSR-O@Q1>l3i)n>GA!``4ob8 z4~A7r;bM4674AYcKgtvWq|`GdzqYnkq(Fkk@l%jdp|DiR_W+kB%j%yEnkD&P4Z3yC z`;UK-hX1{wzHPW0Q!bs1FGb1ZSqk%2wAp*UIoJkBLhhxXkI$l;EDPn_s#_}gqhpjr zNx!6u&w3Dwi;Jfdd|Y?gm8k3BP=IX?3~YQUADMJ6Q8t3@jV9Mbxu=fK7aT0AYjt(? zIbmUCtx&=bEx|T8(WMwP)YI({pCMG5!A%S3!%Gkkkb`;h8q;8qu@#EtQt>UeX zjbUyLRbgJzi9FHGUS3{6VG7Z_-`8g=>r>e7Mlo;}X^m?$Vh!o&Ya2ee@Ljz4a+Z2-d1z{WiaMm&)YY{#Q11Gq|XFLm3@>;`W% z@(Brw%8zYRDgUi+#KXrYf9aAS_|@4#H@+KnLH+&Oi^K@Cs7H^e!T!XzHu#WlY;4jR z8no{15%KYZ-g_xVTjOU(M-MPE2F2*^w>`}(;=Xp~`0<9;*6l+90kEq4EwCm*DzSHA0@fHe;K=*Kx^Z>_iBY=zGIAmoH!b$o~6!8x3`Vd9TW~Yp*jhEX>TBjCh5G zh1*UB2!)aYxS`YLnqWOo2yw>FO$6 zvKzb)r2t0^tTH`3tg5D#XFVjj7}#uTW+rp-;xTsi`uh6My}df64*T4_qP$rTFfa%k zzkjs;Yj&{Q)sbt^RlE$Pra^e?_wU~TgNWcZc2tm^D76Uat+{s;_@w8;=oJPAhQ-Fy z2`Z^0v0GVsl?6FD_k)5)7FNd(fU{9L&myeL!@`uo9G{%LZarK6Eb7cH73}Th%Vf@D zTd~59A93CBLdXe%!om^WYPB^r;}a7v6i0%wQ3eJ>&xCBXb#(5GiKA~1_2%Bh?N(G& zq+QM6;Nn`GpO=hpICk9RIgs`uH0P8Lib-?T^04%pxeHG-I1}e)@Z()2nBx%=7%96XZiUr#s?)$OO1T+ z#4WN#G=8_)4w#iKLBK@g_5KCgXY`+CI;k7)xZ6`}|1YxM#em>s7#-&B!YKpo%n3lqX$X-JzkOxVSg~ zJrTE+({Te_vxF>%`1$!cucm+O?A%!XY~?I6VEQ`>c<|b_Ywj63$qKRbwa2(QITz;V zdC!~?u^lD5>g4R_-7QfJD$y#<6heFO;4Ne0uC_Lw$=UK`l~golHIDqA$c^8>W$rAU zJb4nO_}#mAjz2yPO-yKAyVhs>w7ObSWSrR5_Qb7`6XGfLdlVVz>gozI)Mh2j@z`a3*}`0`h>(ZZZEYeMW$)0TLkXpPz$pv@ z=5)Eg22hO@qfSdK*7R#`IzRXItp!T>V3K=rFU9arC)`OY1fv5-T?2|Yg;0P&1W!G^ zLnvWgj1=wX@pv~3ij9`m=$ZEVwTp|(!`@7!Iq05hA-fpFX!feeghH0POGcXmsW}lqH|uj;QhRw?i@Y|UFcJR z1%tq`nrF{kmVZQtgoKQ1ISlPjLuK~zlDvNXGs30bdm2af(?!Q5Ccs2t*0@Og+D=RC5#)iBHwdpX;D<}}Ku~k)3!FA^6 z=H5OzM$6OF-!Jmymy*wiXF?HSVG?`#YXYgzP}r_KJ#(x5=(*T|tY2C=MvX;xhW8YZ ziEqq#Kl{}V=1{RQ-vqVr{Zl1^1h;afr8JVgITFkmpAw9?vngla5^Qp~ZX<#oj)#lb#FhTA7+mE5(_{h({qY!V7 z7q9$$WmcQ}7a$$V9i&Ob0HF8n+qVJNv;LdVt660I0#wY+Z37(@V2SqRwa)3-n5R!A zFI-4fn*@{k^5x6-1!vpxW#_EvKcEW|3B-i+SqJ<7z+_V#QbWXAvB4K^_kM9 zCnsb0%??B~p5fywb(ns7R6hLCqa>7x+)XM4xxiRfxP;sHv%uxQvGmt>+(gr;)sT zvV_J^swY9uH`FK*;s_BLSzc}~A3y&-a`G7pjZT$n$&v5hm%6xRSvOrP663x57S#jVG#*sZ zN`?nZf|V#LDhkSAH-^@qQ(j=S-Fc5iP_U@DxVNY07gPi!F&O*n*Y{b7_IR_Bd3dp) zCAkP()zA=kSu}-si&F4I;Oq6XUnHlYpoNo#MaIWN9k#%USXp`g{C2S|Qow3JTSKGp zR>vz~W1GuDw6wH5R{1y#WDBYvDMddx$M>;2o8hzxD3CqNrH-?5DgpQQs|F}v-CpQ? zuw>D7$K@Z}h)T12q*Nh3Xb;nCpkC{6D2?;&T|_1>6{hJNM8S! zp8xyl1Q5>kYu5nhx9U!8A)>+F{vdqgwmJ{B0fSR=W{MOg1U0y8QdKFw6!eimIk~%u&S<9OTBqgo5YD{0( zhEih#t7~dT-rn1V({h6bfi~tPd1k^4QOr$+YpCOSXdjntTIt*jkdAzvB)Ol++H`a z%A_Qp`r$XdKOkn^y?YmVZSxeE9f8D47Zevf62|2mw-kKyI>Z&gm;slm3`p7G;o)e% zXa>vtcc2sER81d1e$LHZfD+Wz-7QTi1;FF%Ts+tHm!1?6nc1*-l;UiIC(%OZ;GiDb z5s2#b>oMl=q7G-h+IOmv6ueR5g_4U!WHx0r1$W~kD=wd?Yj&J90V&%D9Ukn}I!0<0 z@pWMp)a{%9Lk2Tn%9jmBytQQ;$YvZ%CB!>-otn0;t}C*##o$#JFZz0WUqHXIwarDu z2M1pPOMy2hE8B<;21<~bl41(GC-+T7g_wXqK}pF|h2-m+nu|+(zAGy$XdVd(3HSjo zA0Hp&Z;04hO3&_6gc?dlJD;&^7@{vvkW* zLO{cwtwdzKdGls-bMp_~rP*63nCJym9?z<%y;uU`@|{-aUmo>kF*Y%&tFOoPCMPC> znZ+xz0*{^WdH7Jl#KffanuzD7b{^%cSFfNSTu^xcNGT*J*bGLX?U4E}E8P?r;ti_6gK+nK{R&enqj~B_$>B2$0ZECXa{02$X zz`4Y}aNGaVbDYm+me;KXf*EpfuO=X%7v7!)V4Rzqn15C2d;X-f3o9L&-(3Ci<34sb zEQx+PmEkOP_1b%SW*zn?>rw3@;h7}8cP%I=3f}D(%P^{I+4>TNq|?8j(kTTf>k)Mt z8r3j*S8dxkXV~^o8+Ov}%pWI>Vr0Lv9VEV8X8}39l5}Ni&m49^o)#LjCKkizW=EV_%|9ciMl+hHQZ<7sKRvo z_;K^zoR@idYQXw+b!;Lc+IEZwoEfJ!U=>wsIjCuEk>TnM zLHuglW&yeC@94-ySnT0DL>fuICchlX<|LvE@%Zs$b;t-{0oK-Yb4;hdJ*U~5p(6D{ z2{H=owB*at-@bm$2J2m3b~w>uc3J24_$e*jr;xR*ZEVijj9h|*^`Mqs++!o#)9uf% zowT#FL)Qq8h|uEd23>$7DUdfSZgPr$F*5Jel`@wl4naYufihRXZHo5#RiFijI8zWHfzQ2g}iIk5K&^9A_eja8Kkl|8dek!*gqMFWF_V|k6=L)@Gh zC}%i$@Px*%p#>)qWZJ}+(+{E;I`(FKQA+K^1}Uk3*Vm$~aQh4>%B&AsmgZ~_!# z1QOW4f9OwlRq*cFs+WFM5Qq-pYS<*U2qF7PwF?*CVNeM2Q^OdODs<`wCny-zUkBZ(^h>ml{65|zo9%?FS8}ms7yQYO9X%Qq9Uc6r)wf&-TSJPjZo>e6 zHj5z=2<8?$BkBEiIo~pDOSlMpJ@S%x=xht3LFU`peTJE?Vl1P6?hKFz*8tf?MMUa? z8MawAssp?Ueg4PC`1$i;1@fD9n|~>Rl%S@pd)qS^tNDFU-j_hyy$xABc?jqB; zPq|P{zv-tKhVWAQtC`wRa3@iGzkO@Z65f(H4hOQTYA8A01AP!pg!wHL@bB$}hM5fa zx%S>XIG?|a4#Y^FjA;?j=94EaAP7K;9knT@COO8-tHNX{e)-a+m6hQTH1x!T1iz7? z!$7sn9xDb&Je+GuEH85==?JkQOtDZJQ z>)cN$@lply3tks{bkOGv^tSO8AP2NDUq3(if8;~L0E*j^78VHhja+0}R)nJkvY9gP z8TPJeu*BZXDKfZq40^q^qGD@D$3m6&-V5uaP^$2&SD-Gi0sP8Iwr3MgH#~ausNs*} zaHNkP=h7|HJY5j{H43)$S}a=LJjfNkqGQ^ubm{um!3Yo)+z~5w6N;dn@)^)0DYY9)>TqE?Ye9Z zN;Xj&x^hjg%;_iiW=?7~rn_9f$iHkeh4lkVkL&1BAL&GUaq?s(quh{pCb{@)VlP;8 z>DfHu;+~i`;$8Qvw3`SA%1%tY0mn6)2YW2}f{uZq(rr~`Pm>a0%Ysw|lxzS9<6Qb5 z1LZL+0e9~bw6**A`C--wo7(i8%ExskK~D%*hoFOyz~r95@2^u+pK~cOgIQjFi07;q z4K3}j^t9aET=P*T+^-*(ca_u6Z@Y_crKY58L!{zjEtA-f78TmuJ(v{$?#9d-Mj4vE zzt=yA9^O0M8Iq7-C?Ro=%;Yyj@9^Z$scN^Oo}FjWf1?Qo0Ur{MYmPQUBk6c`)-y%1 zRouGracT@o!EQas^%h4wFnv8!z}Ehdh_mg*`h?;9Fm zimyZoyRBr8?~#6?1ag=p+JTZ~N&DmRqem5MlW7LMc~Y%Zzt;!1W(xG-NFY(em(!6m z7d=BD>tGDY<5p|lix&o{$O#MT%F3{^;2t&3Wk!Inp%f+fK5sL{`010X==YZO8IVmP zS|*fhp4oNv^gPHM2Q6=p8%~r7LD?w0aR$8|Mzeu9aSjHQ;>KR2s=XeM7CsRz?pcwo zc$``eBv$Qa{qRpHb1oj5$Bw;ZtxPy~H)|k{^i@Ja4yH!6m4VwU9!SCv!yF&Z z1sr$cvNg(h7M(dfcdBMlr?RMGB3|QfIksF7T2J+iVNUOL7}$L z)wc!Q4jKq8d3lR?i^-jPm@ryx`eR7d50b9%M9W(HOL;;GD3h{Q2LQ>Bcm}!WA7ym&|KRGi4_vSSkA%qir{)Lj8n_H5j=9QVN z=yKrWKkCDdmM7cS-iPW$W5dvGE>QaVH4aH`<4yBjdk2SbnM8!Y-MQF5%@5fXYtptG zPf(D7;`cf6>`~e+77rL9$VeX3GHNZj^WNL$<{*V+`3cV4q@)@E4;V^vvwASpE$gZ| zh&;QGIMaBn4Ea|D8Rw$JU}>A1>Ttcac4JcayUji_vYKZ`bV*==VU-hMgRT>Mb$mU- z7J2qSjX;k;;@EtKgAO@FQO;^}q>;+e@fUEQ+38>6%)$=lp^DETgys|$#sm54TN-*o z>;oW8>Qu%zVGzYoL63Z%+zn_RJ*LVn`jC)~QKp!IZR)PArhpql(8c z@1H}w!SHe6^{KEUGD$8YVsE345LFKhc+B({V;S4ztzF^0(&~^u^PVlX<>h63LrV2s zckr;p3KBrNZ7XYo>s3n1ajg%Qiz1KYJ``s1>{JiktI`+4R80MG_y1{_RlD-1lgRuK z$z6`;Bq`NOVQZop0C|{|!7=^U=;PqOM<00#r7PT4F@Mi1F0O~zz|1>5clRq=YHDbR zYFb(-?w7=FLL9i$7=Ele`O?Db1|}xwGmrv$l^*4WwIqA@?i~__K?#}&xdiSoEHuEA z*Qu!o>F73qM(xh&$jSz`m>`6KE|@~!}3@O|Oo3~Sa-2}*N;bqsdl#4x&T$aaAy zNn?9Z-l}4i?bInn4Gl*ql-ql!LgR9>3aT7HyNnb~5`!&3Q{cmgnHGJ;NJ!xI+}zM*FUZYBZ^zmymuJ4fr|Ac` z0B}LkMume~VpH5N>bh)t`!>i>jN3;`n31U2^pup@xZ(LjU}JzI_)~eyF39hoybb_0 zrLz_CnJS+PEL7FiMXkRQNKoQI1&k}&6xr`{rfJAH3>+vbs76c_b&Q3`FiB0KWN*e_PZk^%MzU@{RxmeU&6ncHVk zIA)RUImS)6$66y^m>`)trEDfHZTU4g?7_jxq9Q?2(b#ltsFOG2-P!t;kTGIw3yq!2aQN{_g3>2CS}RL$Tf7IhvnjfVMH6``DJjj(&)eA9Ju=#%wELFY zi^Rkur%!*xcy?#!8F(MS1JGFI)W`D)kED=XIPm{?Hxoa893@IX!OzK2S-AxR2i^@x zs^0Vn?L}z3QG4lQqN961ek7SULzVSAcex}Se0}<;x)P>kJ36R7w0-!H zsn^(^#j$v8rx(jeG06qyfAnm;+(OTISFt36$FLA>pKaDMipumr`TEx?Z$s(ei37J! z>6XncEaF^(i7vsYqUF}HDz zQ#VnkJE~@2nug+k=Pzq!X{xRJ_v`8gj(d5A5C36GRYHtVOH*<_`Zr*EmIG=($d0^N z2he>Xs=mFUf|`PY0xW)5kPZ&Z0C!g|UxuJUqU<$sv^+fOHIl8b`W-9ZQWpyDimc|pJh?!$UHZdZ+>W)pnkUi)jJ_rJ%s`47j6XipS~b=}9W<8*&K$cvStc>*h*6A}va*t`4y z%<;hkc{H9vTV_P4^V(wSTrwHpXmuov6tR`BySO7c3%efQq1i*BPFKwVw0Y&3pz_}v zua1uO@5fLY2KQogbPpk+(aoC?9Y)5+80CM&NLA5;M`s!y9`Nkr=izzt`t@W+;Q(9! zPCN+%PJv>tM-@SQNq2g3^uehLK(aN6+)|Eh5Ilh7Ay{52NJ>ryz#(NfH8+1I;+!)# ze_)k;L^)6-)R^WHyw~u{ejJw)N!-NroMN{9=f*~2GO|Z1Gy|GDozo=G-oMPI8a{!U zM*$G;iakYL|95vpL6;*RqCl^%o*pv;!*w+J7)ePjH{lcYYtY`FJit~98z7&Qz>)r9F37su_i*=+keuo=d6B~A|z}?aK@6YCJd`hUSvotluy!42C!rJY~Hz>L#CFcO` zumZ~|Re5<*?cMk&N``z~du$HUp4s*?!@jlDdJZ{CP^L^XIpz)b;>lZ@SI}hg{mnax@20 zuP37;E-#g{QwZmwj^I^ZA*DM(%XhC#f~HQ5<}bTLC8Iik6BX-E%cH_FpaX`7P^YD5 z@?Nza>U63jJfj#q29U`*g`|8w_I-I%fg)w+YmCa8knOqg&Tn6P43pAsHG|7l9o{>( zA7}hup3nLFah)R<#4VS>!oxq;j|Lw~@e+Yomr^|*BxG}3fKGR$VeE9-Q8&7i4aBm{ zX{$iT5yO1d`auk56|P*dc!&WzNGs7Gu?Yx{P;*QC>ygTT nSi}Fn2QdHPc_WqiEdr)?_hSp|m(B6GDS_-o1*uF)eXsuyd+D~P literal 0 HcmV?d00001 diff --git a/docs/source/userguide/defining_workflows/index.rst b/docs/source/userguide/defining_workflows/index.rst index 5ca0d27df..0c17a3ae2 100644 --- a/docs/source/userguide/defining_workflows/index.rst +++ b/docs/source/userguide/defining_workflows/index.rst @@ -212,14 +212,23 @@ This can be achieved using the START_CONDITIONS feature based on the dependencie Start conditions are achieved by adding the keyword `STATUS` and optionally `FROM_STEP` keywords into any dependency that you want. -The `STATUS` keyword can be used to select the status of the dependency that you want to check. The possible values are: - -* `running`: The dependency must be running. -* `completed`: The dependency must have completed. -* `failed`: The dependency must have failed. -* `queuing`: The dependency must be queuing. -* `submitted`: The dependency must have been submitted. -* `ready`: The dependency must be ready to be submitted. +The `STATUS` keyword can be used to select the status of the dependency that you want to check. The possible values ( case-insensitive ) are: + +* "WAITING": The task is waiting for its dependencies to be completed. +* "DELAYED": The task is delayed by a delay condition. +* "PREPARED": The task is prepared to be submitted. +* "READY": The task is ready to be submitted. +* "SUBMITTED": The task is submitted. +* "HELD": The task is held. +* "QUEUING": The task is queuing. +* "RUNNING": The task is running. +* "SKIPPED": The task is skipped. +* "FAILED": The task is failed. +* "UNKNOWN": The task is unknown. +* "COMPLETED": The task is completed. # Default +* "SUSPENDED": The task is suspended. + +The status are ordered, so if you select "RUNNING" status, the task will be run if the dependency is running or any status after it. The `FROM_STEP` keyword can be used to select the step of the dependency that you want to check. The possible value is an integer. Additionally, the target dependency, must call to %AS_CHECKPOINT% inside their scripts. This will create a checkpoint that will be used to check the amount of steps processed. @@ -370,28 +379,28 @@ There is also an special character '*' that can be used to specify that the spli ini: FILE: ini.sh - RUNNING: member + RUNNING: once sim: FILE: sim.sh DEPENDENCIES: ini sim-1 - RUNNING: chunk + RUNNING: once asim: FILE: asim.sh DEPENDENCIES: sim - RUNNING: chunk + RUNNING: once SPLITS: 3 post: FILE: post.sh - RUNNING: chunk + RUNNING: once DEPENDENCIES: asim: SPLITS_FROM: - 2,3: - splits_to: 1,2*,3* - SPLITS: 2 + 2,3: # [2:3] is also valid + splits_to: 1,2*,3* # 1,[2:3]* is also valid, you can also specify the step with [2:3:step] + SPLITS: 3 In this example: @@ -400,10 +409,10 @@ Each part will depend on the 1st part of the asim job. The 2nd part of the post job will depend on the 2nd part of the asim job. The 3rd part of the post job will depend on the 3rd part of the asim job. -The resulting workflow can be seen in Figure :numref:`split` +The resulting workflow can be seen in Figure :numref:`split_1_to_1` -.. figure:: fig/split_todo.png - :name: split +.. figure:: fig/splits_1_to_1.png + :name: split_1_to_1 :width: 100% :align: center :alt: TODO diff --git a/test/unit/test_checkpoints.py b/test/unit/test_checkpoints.py index 8772e3ae2..35dca3350 100644 --- a/test/unit/test_checkpoints.py +++ b/test/unit/test_checkpoints.py @@ -3,7 +3,7 @@ from unittest import TestCase import inspect import shutil import tempfile -from mock import Mock +from mock import Mock, MagicMock from random import randrange from autosubmit.job.job import Job @@ -24,35 +24,57 @@ class TestJobList(TestCase): self.temp_directory = tempfile.mkdtemp() self.job_list = JobList(self.experiment_id, FakeBasicConfig, YAMLParserFactory(), JobListPersistenceDb(self.temp_directory, 'db'), self.as_conf) - + dummy_serial_platform = MagicMock() + dummy_serial_platform.name = 'serial' + dummy_platform = MagicMock() + dummy_platform.serial_platform = dummy_serial_platform + dummy_platform.name = 'dummy_platform' # creating jobs for self list self.completed_job = self._createDummyJobWithStatus(Status.COMPLETED) + self.completed_job.platform = dummy_platform self.completed_job2 = self._createDummyJobWithStatus(Status.COMPLETED) + self.completed_job2.platform = dummy_platform self.completed_job3 = self._createDummyJobWithStatus(Status.COMPLETED) + self.completed_job3.platform = dummy_platform self.completed_job4 = self._createDummyJobWithStatus(Status.COMPLETED) - + self.completed_job4.platform = dummy_platform self.submitted_job = self._createDummyJobWithStatus(Status.SUBMITTED) + self.submitted_job.platform = dummy_platform self.submitted_job2 = self._createDummyJobWithStatus(Status.SUBMITTED) + self.submitted_job2.platform = dummy_platform self.submitted_job3 = self._createDummyJobWithStatus(Status.SUBMITTED) + self.submitted_job3.platform = dummy_platform self.running_job = self._createDummyJobWithStatus(Status.RUNNING) + self.running_job.platform = dummy_platform self.running_job2 = self._createDummyJobWithStatus(Status.RUNNING) + self.running_job2.platform = dummy_platform self.queuing_job = self._createDummyJobWithStatus(Status.QUEUING) + self.queuing_job.platform = dummy_platform self.failed_job = self._createDummyJobWithStatus(Status.FAILED) + self.failed_job.platform = dummy_platform self.failed_job2 = self._createDummyJobWithStatus(Status.FAILED) + self.failed_job2.platform = dummy_platform self.failed_job3 = self._createDummyJobWithStatus(Status.FAILED) + self.failed_job3.platform = dummy_platform self.failed_job4 = self._createDummyJobWithStatus(Status.FAILED) - + self.failed_job4.platform = dummy_platform self.ready_job = self._createDummyJobWithStatus(Status.READY) + self.ready_job.platform = dummy_platform self.ready_job2 = self._createDummyJobWithStatus(Status.READY) + self.ready_job2.platform = dummy_platform self.ready_job3 = self._createDummyJobWithStatus(Status.READY) + self.ready_job3.platform = dummy_platform self.waiting_job = self._createDummyJobWithStatus(Status.WAITING) + self.waiting_job.platform = dummy_platform self.waiting_job2 = self._createDummyJobWithStatus(Status.WAITING) + self.waiting_job2.platform = dummy_platform self.unknown_job = self._createDummyJobWithStatus(Status.UNKNOWN) + self.unknown_job.platform = dummy_platform self.job_list._job_list = [self.completed_job, self.completed_job2, self.completed_job3, self.completed_job4, @@ -72,7 +94,7 @@ class TestJobList(TestCase): def test_add_edge_job(self): special_variables = dict() - special_variables["STATUS"] = Status.COMPLETED + special_variables["STATUS"] = Status.VALUE_TO_KEY[Status.COMPLETED] special_variables["FROM_STEP"] = 0 for p in self.waiting_job.parents: self.waiting_job.add_edge_info(p, special_variables) @@ -83,28 +105,28 @@ class TestJobList(TestCase): def test_add_edge_info_joblist(self): special_conditions = dict() - special_conditions["STATUS"] = Status.COMPLETED + special_conditions["STATUS"] = Status.VALUE_TO_KEY[Status.COMPLETED] special_conditions["FROM_STEP"] = 0 self.job_list._add_edge_info(self.waiting_job, special_conditions["STATUS"]) - self.assertEqual(len(self.job_list.jobs_edges.get(Status.COMPLETED,[])),1) + self.assertEqual(len(self.job_list.jobs_edges.get(Status.VALUE_TO_KEY[Status.COMPLETED],[])),1) self.job_list._add_edge_info(self.waiting_job2, special_conditions["STATUS"]) - self.assertEqual(len(self.job_list.jobs_edges.get(Status.COMPLETED,[])),2) + self.assertEqual(len(self.job_list.jobs_edges.get(Status.VALUE_TO_KEY[Status.COMPLETED],[])),2) def test_check_special_status(self): self.waiting_job.edge_info = dict() self.job_list.jobs_edges = dict() # Adds edge info for waiting_job in the list - self.job_list._add_edge_info(self.waiting_job, Status.COMPLETED) - self.job_list._add_edge_info(self.waiting_job, Status.READY) - self.job_list._add_edge_info(self.waiting_job, Status.RUNNING) - self.job_list._add_edge_info(self.waiting_job, Status.SUBMITTED) - self.job_list._add_edge_info(self.waiting_job, Status.QUEUING) - self.job_list._add_edge_info(self.waiting_job, Status.FAILED) + self.job_list._add_edge_info(self.waiting_job, Status.VALUE_TO_KEY[Status.COMPLETED]) + self.job_list._add_edge_info(self.waiting_job, Status.VALUE_TO_KEY[Status.READY]) + self.job_list._add_edge_info(self.waiting_job, Status.VALUE_TO_KEY[Status.RUNNING]) + self.job_list._add_edge_info(self.waiting_job, Status.VALUE_TO_KEY[Status.SUBMITTED]) + self.job_list._add_edge_info(self.waiting_job, Status.VALUE_TO_KEY[Status.QUEUING]) + self.job_list._add_edge_info(self.waiting_job, Status.VALUE_TO_KEY[Status.FAILED]) # Adds edge info for waiting_job special_variables = dict() for p in self.waiting_job.parents: - special_variables["STATUS"] = p.status + special_variables["STATUS"] = Status.VALUE_TO_KEY[p.status] special_variables["FROM_STEP"] = 0 self.waiting_job.add_edge_info(p,special_variables) # call to special status -- GitLab From 8fa04f3e6ca078fbb27a676c6cc7f7e744aa177c Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 27 Jul 2023 15:36:13 +0200 Subject: [PATCH 55/68] todo --- autosubmit/job/job_list.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index b642fcc2b..a6a71a29a 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -397,7 +397,13 @@ class JobList(object): :return: boolean """ filter_value = filter_value.strip("?") - filter_value = filter_value.strip("*") + if "*" in filter_value: + filter_value,split_info = filter_value.split("*") + split_info = int(split_info.split("\\")[-1]) + else: + split_info = None + # strip substring from char "*" + if "NONE".casefold() in str(parent_value).casefold(): return True to_filter = JobList._parse_filters_to_check(filter_value,associative_list,level_to_check) -- GitLab From caef643cce4a4e2f7fc60876f9bb7ac0eb7918ca Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 31 Jul 2023 09:38:00 +0200 Subject: [PATCH 56/68] 1-to-N and n-to-1 implemented --- autosubmit/job/job.py | 4 +- autosubmit/job/job_list.py | 127 +++++++++++++++++++++++---------- test/unit/test_dependencies.py | 5 +- 3 files changed, 97 insertions(+), 39 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 051d6b0d5..abd194546 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1644,8 +1644,10 @@ class Job(object): except: pass for key, value in parameters.items(): + # parameters[key] can have '\\' characters that are interpreted as escape characters + # by re.sub. To avoid this, we use re.escape template_content = re.sub( - '%(? child_splits: + lesser = str(child_splits) + greater = str(parent_splits) + lesser_value = "child" + else: + lesser = str(parent_splits) + greater = str(child_splits) + lesser_value = "parent" + to_look_at_lesser = [associative_list[i:i + 1] for i in range(0, int(lesser), 1)] + for lesser_group in range(len(to_look_at_lesser)): + if lesser_value == "parent": + if str(parent_value) in to_look_at_lesser[lesser_group]: + break + else: + if str(child.split) in to_look_at_lesser[lesser_group]: + break + else: + to_look_at_lesser = associative_list + lesser_group = -1 + if "?" in filter_value: + # replace all ? for "" + filter_value = filter_value.replace("?", "") + if "*" in filter_value: + aux_filter = filter_value + filter_value = "" + for filter_ in aux_filter.split(","): + if "*" in filter_: + filter_,split_info = filter_.split("*") + if "\\" in split_info: + split_info = int(split_info.split("\\")[-1]) + else: + split_info = 1 + # split_info: if a value is 1, it means that the filter is 1-to-1, if it is 2, it means that the filter is 1-to-2, etc. + if (split_info == 1 or level_to_check.casefold() != "splits".casefold()) and str(parent_value).casefold() == str(filter_).casefold() : + if child.split == parent_value: + return True + elif split_info > 1 and level_to_check.casefold() == "splits".casefold(): + # 1-to-X filter + to_look_at_greater = [associative_list[i:i + split_info] for i in + range(0, int(greater), split_info)] + if lesser_value == "parent": + if str(child.split) in to_look_at_greater[lesser_group]: + return True + else: + if str(parent_value) in to_look_at_greater[lesser_group]: + return True + else: + filter_value += filter_ + "," + filter_value = filter_value[:-1] to_filter = JobList._parse_filters_to_check(filter_value,associative_list,level_to_check) if to_filter is None: return False @@ -418,7 +474,7 @@ class JobList(object): return True elif "NONE".casefold() == str(to_filter[0]).casefold(): return False - elif len( [ filter_ for filter_ in to_filter if str(parent_value).casefold() == str(filter_).strip("*").strip("?").casefold() ] )>0: + elif len( [ filter_ for filter_ in to_filter if str(parent_value).casefold() == str(filter_).casefold() ] )>0: return True else: return False @@ -666,6 +722,8 @@ class JobList(object): :param filter_type: "DATES_TO", "MEMBERS_TO", "CHUNKS_TO", "SPLITS_TO" :return: unified_filter """ + if len(unified_filter[filter_type]) > 0 and unified_filter[filter_type][-1] != ",": + unified_filter[filter_type] += "," if filter_type == "DATES_TO": value_list = self._date_list level_to_check = "DATES_FROM" @@ -703,11 +761,12 @@ class JobList(object): continue else: for ele in parsed_element: - if str(ele) not in unified_filter[filter_type]: - if len(unified_filter[filter_type]) > 0 and unified_filter[filter_type][-1] == ",": - unified_filter[filter_type] += str(ele) + extra_data - else: - unified_filter[filter_type] += "," + str(ele) + extra_data + "," + if extra_data: + check_whole_string = str(ele)+extra_data+"," + else: + check_whole_string = str(ele)+"," + if str(check_whole_string) not in unified_filter[filter_type]: + unified_filter[filter_type] += check_whole_string return unified_filter @staticmethod @@ -813,8 +872,17 @@ class JobList(object): associative_list["members"] = member_list associative_list["chunks"] = chunk_list - if parent.splits is not None: - associative_list["splits"] = [str(split) for split in range(1, int(parent.splits) + 1)] + if not child.splits: + child_splits = 0 + else: + child_splits = int(child.splits) + if not parent.splits: + parent_splits = 0 + else: + parent_splits = int(parent.splits) + splits = max(child_splits, parent_splits) + if splits > 0: + associative_list["splits"] = [str(split) for split in range(1, int(splits) + 1)] else: associative_list["splits"] = None dates_to = str(filter_.get("DATES_TO", "natural")).lower() @@ -840,25 +908,10 @@ class JobList(object): if "natural" in splits_to: associative_list["splits"] = [parent.split] if parent.split is not None else parent.splits parsed_parent_date = date2str(parent.date) if parent.date is not None else None - # Check for each * char in the filters - # Get all the dates that match * in the filter in a list separated by , - if "*" in dates_to: - dates_to = [ dat for dat in date_list.split(",") if dat is not None and "*" not in dat or ("*" in dat and date2str(child.date,"%Y%m%d") == dat.split("*")[0]) ] - dates_to = ",".join(dates_to) - if "*" in members_to: - members_to = [ mem for mem in member_list.split(",") if mem is not None and "*" not in mem or ("*" in mem and str(child.member) == mem.split("*")[0]) ] - members_to = ",".join(members_to) - if "*" in chunks_to: - chunks_to = [ chu for chu in chunk_list.split(",") if chu is not None and "*" not in chu or ("*" in chu and str(child.chunk) == chu.split("*")[0]) ] - chunks_to = ",".join(chunks_to) - if "*" in splits_to: - splits_to = [ spl for spl in splits_to.split(",") if child.split is None or spl is None or "*" not in spl or ("*" in spl and str(child.split) == spl.split("*")[0]) ] - splits_to = ",".join(splits_to) - # Apply all filters to look if this parent is an appropriated candidate for the current_job valid_dates = JobList._apply_filter(parsed_parent_date, dates_to, associative_list["dates"], "dates") valid_members = JobList._apply_filter(parent.member, members_to, associative_list["members"], "members") valid_chunks = JobList._apply_filter(parent.chunk, chunks_to, associative_list["chunks"], "chunks") - valid_splits = JobList._apply_filter(parent.split, splits_to, associative_list["splits"], "splits") + valid_splits = JobList._apply_filter(parent.split, splits_to, associative_list["splits"], "splits", child, parent) if valid_dates and valid_members and valid_chunks and valid_splits: return True return False diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index dbf93565e..1432abdbe 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -356,7 +356,7 @@ class TestJobList(unittest.TestCase): def test_valid_parent(self): - child = copy.deepcopy(self.mock_job) + # Call the function to get the result date_list = ["20020201", "20020202", "20020203", "20020204", "20020205", "20020206", "20020207", "20020208", "20020209", "20020210"] member_list = ["fc1", "fc2", "fc3"] @@ -375,6 +375,7 @@ class TestJobList(unittest.TestCase): self.mock_job.member = "fc2" self.mock_job.chunk = 1 self.mock_job.split = 1 + child = copy.deepcopy(self.mock_job) result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) # it returns a tuple, the first element is the result, the second is the optional flag self.assertEqual(result, True) @@ -429,6 +430,7 @@ class TestJobList(unittest.TestCase): def test_valid_parent_1_to_1(self): child = copy.deepcopy(self.mock_job) + child.splits = 6 date_list = ["20020201", "20020202", "20020203", "20020204", "20020205", "20020206", "20020207", "20020208", "20020209", "20020210"] member_list = ["fc1", "fc2", "fc3"] @@ -442,6 +444,7 @@ class TestJobList(unittest.TestCase): "CHUNKS_TO": "1,2,3,4,5,6", "SPLITS_TO": "1*,2*,3*,4*,5*,6" } + self.mock_job.splits = 6 self.mock_job.split = 1 self.mock_job.date = datetime.strptime("20020204", "%Y%m%d") self.mock_job.chunk = 5 -- GitLab From 8e1dace42bba5e6bf0009e6271bfaba0b75ffa04 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 31 Jul 2023 10:20:41 +0200 Subject: [PATCH 57/68] Added tests and missing doc --- .../defining_workflows/fig/splits_1_to_n.png | Bin 0 -> 8323 bytes .../defining_workflows/fig/splits_n_to_1.png | Bin 0 -> 9094 bytes .../userguide/defining_workflows/index.rst | 52 ++++++++- test/unit/test_dependencies.py | 105 ++++++++++++++++++ 4 files changed, 153 insertions(+), 4 deletions(-) create mode 100644 docs/source/userguide/defining_workflows/fig/splits_1_to_n.png create mode 100644 docs/source/userguide/defining_workflows/fig/splits_n_to_1.png diff --git a/docs/source/userguide/defining_workflows/fig/splits_1_to_n.png b/docs/source/userguide/defining_workflows/fig/splits_1_to_n.png new file mode 100644 index 0000000000000000000000000000000000000000..5e700a40087500a6c39f93e00e9354cf45bb5e60 GIT binary patch literal 8323 zcmaKSby!s2+BPB*ic-=oDJ@7S2r__liHLNEFd)()-JqhNGz^^z(m9NDcMd7iLpKaC zyo=xUUFUq~`p*0D$C^EB*4}&VC+_=xo=t>?n*80{G`F#^u9~xVpGlI++I!5@2C5VJXT= zzw(&cnsqn0KQV>3TMo@SxyO|sZu9CRJ|{^Ve$r$g9A%g3nf2mP)eLtWt=WC$K-Nc* zB($WZWaKwU1pFUp6W`LjAy?-=nsdAU*XDW0j61)MgXQM9bCoCJ zGGzETI9OQEn>o|(fK45NyZn1j*X5=_+D6a<^G(+_uAQmpUOuw zW-%_kmXm8MPe}=}6UpTE+G(0y%S%jL!>cV;46yS#`IxYv^o+D#SwU5u;Q4}&9GhhkK^2tE z(t->XYd~C4&FDIxX$KYbjf|aW*XEvStfo522zOJnlzZ{gKuhLQT!`yPZ3%|B()$+c z^Cau};WvT>(PDZjiT8DPlr{IHRa_K(w;M{vE2{78r&%>PKu@=8{9e3F*)cjH+N6_A zd74TL!3nHGp~96hlp{tOioPb7N*%RPxY1N+e;f`M>}fysh55K&{8&SJ8QzsYl;$d1 zUzlBM(H9j+_%Vf2GGrk;iCx9j|0pYxA=%Zq8)Y4tW^I4EE(+;vE)l)h-_qOpRS#ub zE}r6T|15>pF>_y$a0=Y==vbF~g4{_d``$6@&aBWe7*5<=@Po%s_ppt^9hFmVzPC&6JFjLfW6G&|8}O&7PdN_furx z6?C%u>f0TTDTg_<+pJSBNqDW5qWnHIx*rhY%ZVt`8~Ay>lWvsn5Le5?V1BzfskRl> z3^I{y&E^#rOWXJDJa#BE*U{=fJr{)Fr()usMMqy9OS#~t67N4?`7}Ml17#HVjBHET zn@!~PU$WNHHEoKY_8S})VhVoq@qBlL|8=8B%3S{{OR@&cLP=YM?WJ#ruTZQ22kJSrj# zCx$-o_&C*=hsNxQ?RxIdmX6Q!2rJK?jokfA2qZiyuF6r+Qn+&eZ*%dK@yg1~-C7Dt zuhfp^NI&u<)P^xRc>@<$=fDS2kG~4u=ZFLABO)nVn~k(9SHv`gl4~nO$Lod#Px5|w zS`7~d8ySpd)tZ``dNn#07ELXWPmO63b5M4+*S2)lVquomHq2TiQKyZb(5?qty$&l| za-}=_Fy0wSD(IvJ;+$8V6z@fJJ%M!}!sR$IU zMeO6|({R2=x<8DI<;R_E11~>+UxG6Q?@mvRR=VOdOE^S1mc8cFQPnCca)aQzlA1hs zX~--xh zg~pZrkCLLI;*DxCufOWG18uat&$i#n$hZ=|r}^uGM_<3^@Dnhr z?{MU_%if+gCSNHh|K9HRcq_I1)|W_V9X-|VV>V4?^{cl^e;2*?NGJzn4Qs=pF;8nS zqd9w@M?Qrn)bl4x?YBf6ysjHw|0v~qVH!4&7;QDc@4L6Tv`rtx!!xpI?sj@1oi&8d z{La3Fi#vmqWwT*^-au4~W4wN!bHU-Fk4X}-HAS#faDd6Rv*7U}Fcuxnlb!eaM%=Sn zZTNIk*mcgDT_%hS;sle3tXWdWkARa*YII}_*uY3|+OIwXzmW%0=gfwd-XIii(E^hkRx=jAKq9 zigII}y$$g}Vu9-COL?tr?X65%OCz<;tBog<=hJ=low;+?BRMTby!_7Zf`d)!oA+Wl zU7iLhaX`Yo!i>#{A&*^Lgk4<35jjO}5fI*tvZSZ3>Oxo+WMu?Lip2yt*bWC9)nNf$ zW&(-u6b3A0Fo>@}hE%-#g8h?2j}7HTx?DB=-4i1^2B?s`bvyxD@mFa-G3}=A zN&V5r1SJJUZgzHfxfluL2qae}QP6F3@)ND-17fBv9PB0?W1q3fNh)gUhy8zL z@gE5Z-615rI9-hH>+NmCpzT_*j}$2`_Ivo_`Hj^S6o^|wX8M2p{CN`_n}W~4#=$|- zx}PriX7*IEVf_OZme|->;m40#uh#F~yNAJGxL^OElKfB;8%x&AoNGwB#z8_t!b8E$ z%4%Ukx%fNarmd|lH8piOzCuZnPD$CDAJf(Eh&Ih?>gzYw)|Qr*m>!tV3giHlFq z&0)@W5bn;-w7VhiU0va>8{=^A6GB2lVwx{ci5MkQ1TDLpg77Aa4aNV6AHw`Fo_cyT zxE+x|bv4^74-4 zs>W8B?V`rG_4V~BDJhMOX9TktO?mw0rz_rSIXF0&n4~Y=2WePU<))yZU}3S;=yOiY z^5r&mfXP^)c4cLyALcAimcD?Bh9)#P7zYQ(!`quh3 zcvz5=v$L(ua`KmHPaLnzujkEM)7AVE5*KrDZ?GV5@y{dxga>afE5opTmP-&AGr`0g zH*Ul@ zSw%&wcN)*tnkikueJJ^jRMgZMBs>JL@krN(vgCn(@$S*RLHNRh`3sAR>||wSoty-5 zJC2`><~3>>8%w<%&IZfyaBvhbYeC1y$8u2=aF4wQ%*>|GeNp&K5KqtApFg)gQVXfU zBze!2r>CaST^z8{v9VdWw~nqZ^@5B~49^Eizqtm_<>lql($e9w01*)p-@_sK_V#vB zhZ*97pJrIHfA&23`}?s2n7Fu*1TWLHamG0X>wu2czQhjZe(~kFeoU; zW=Bz5TbPH3hlR!b{CF#zN#XN#f>2W8;o>SOzkd*HV`mo@7B*k&Y`rrF4>w_KK84Ti z9~_ALpj}i{K1M`DxbW7NmcDm%{FRlpxVz9|XlO{HMCAzb7#JA*Tvt~&T)89@b`S2h zl@t}#1C~4zs~Ru~S9sUj-d={>RF3CU+~G2)bqZ-SlarHUP7?Ul+|rV$vI4R(+nxO4 z;^ICzIXI;xUGm7l$OwSX97wBdA33+3xt~9OUTz|VTJPg3Sy|?S|W({wS0!KtVjb zFd^%Hh4VU%AaiFYC$IqAH(mqgRYTAZlR{)?v-0t&fj#1m=jY^Dq7T;zab+TLRRf?l z_V%eyoY|v0qCS6)%hwM)q5@7Q|2V%9@l(G#H#ZlcS^-g}gRTB}6`e0Shy=jZ3Ar!&CjgXz*`6-K~@fFENy)ER_rzp+Y` z-(G408Odi*^X1Ez^pX;O<&ClTj+=F^8^mi2&1^L%q>PLMR^HNv6V_Y$3+p1!nXp{1o2%z&l~ zYXP`YUR^Eaef*;rmW~D}35rl)>7&C#V)RKMdX%^MQI?{jBES+sL?ei1%*7di8W7Eb zLby1rvikbd0Eukv>=yt2{r2sfg|CmNL|RmoGH{TEMRxDU8ze-)vJ~#_C#R>Uw4x53 zGQhHf>xWU4{QDja{D_M(IGmmq>oB7)S=8Fn@)ofD^@?9_e%MVEfB*j7)R70JHDB+v ze2tAIhV>#YYZ_;|cCM~Rz??NTHHP(WrnLTzPQbrXm{S%`P6gDq4wCli)2F}*($dl( zv?p=rZt^v`wk|uFo0&bW%FfN@;^u})c)pYrzYY;i&Am-zIX$hil9H3Nb7`f&&C$`( zyP1gX?b&*$$oc+q&pwiQ^v~qRM2S1JcVK{@kFR|^aL;?kZEH#xH)vsTk>%on$fHN^ zotCHGEFtV`dxZ_i|4&UXO<_*{YqGq<}j1XAM-Pn_c_J3qg=j?P3&i%i;i>-01z zf)&NYM8IB^l{dl|eSeP!q7NO)t@>VSXrzy<+l~JyC@F!7z5m;rCriwWMyO1xq49AB?_&!MN!_mb!)WxZctNrc=#zGG}Dcj+X}>4L}5e8cL_cD8m|je*6ACDl#(vuj~`g zeO9krAv&qcj(p9mva+(LRRUCm?6K6u!~*Wy2DMvpNl8h7Pb58d{d<-%6UCnbo59UK zef;=q`8O?459!iDfKlc;+1c1~{`@iF;NYm;N(9$Ympo$?7eD_51@r^N5xB~vhvUT| zMKq%n!0YXXypfri1|XlZzZh{qp^9T;M}WVV%k?WN zDu9%0Up)|)YR}EdS)MF4pL9wu2caI+Wa84#PV-5`Zk(s~neXt(h_j1}tO$Jmt5`)< z6-Y?LkWge~BF1}X@3s>eU?n9di_eoq^;sn+B>{>eyL3L; zHdI#b=|{?D4G9Vg4ocB4oJJ5sFzAgEG#dTcx_He6Jz?yZ8XsS6)^__a`qL*>TU!*6 zbB~3Efq_D{2yt5`CMYj&(i;L&t*q2OHD-_PvJk9(H=h<8d$67d%k-n8rxy|uQcr!F zQ(Qc^mvT3%EoNE>Cgw|OxzHRkx{gotsog|?j*c!((kIek{bS1lbn+L-Vrh@#7mA7n z1qB4pZbH=5)JE@O3Pu+B`1m|MJ=fy;0evIRR+2v>fzIt(+#3{v{oWC;XOmHqUKyb-{fS7!r$07atwsDXg9d8{`ucaA>V_{_o4 z<~B)x|9%DVK42+Y+T?4=!2yg=Z{jrFMqNk?xI5_eEN_JnkL9&LxUA$alJ>(^CpM{{J-i zYHw{#5qIxzY6|GH;$UMVrvy^Ap#~1WJUdtgo|U>h90|fBy2mI98p1$ydqeP#m}dX{ z`4b+FZ@Ym6Q41t$Z(m<*Ow94l{5z1c>+27_syNhAYTS3`Fqh}lgt%cXj|2pE4-X%U zipKXEpz;LZI@ixcp39-0MknE+p{NMNhTYm=#@yW8gS6fVfxCa=8Q3^DtU(}yiic7Y zX2Gv=`4Scw7`VT`pU?4u2Smxq_AG!NfI)_p`ntNV2VcJeqXD_mGce${`b~mir}Hbz zXdlYX))pY-_zztfxXb3`BTyl)kK{2*ewgg;w(W{xYj%HYYwJ{NAd>8})cN(HRxXcG zLjtkj`@gO47uz+JIlZa?Yk)EbfG4OmySlppi36b)Gua_=ET1q}jYpgta9ax?Zb?Z= zU?9ZFX&-DkD>D<6vta^?;4zqkdzR7X=hc;!UDNSK2m`-5m-U#`RQe+@;|jd#;!+WQ zybbiy#>#;v6bh|Na_Ucc5={Iwb?z(&S#GVhg+wCh8P9}Fn{jMQeTe65#2j)HpuidkJ=)_(n(hMJl{U4w>(2Iy5iU0ryk&+#U5 zeSJM6BV+!&jFps?X@za?2Q@JTCnP2s*10Ik$yu%s=e(jB zNO^MLV{K<+lmGLlD%{5;f>%W*l+d$;;dufsS90)1&w?We9eR5Duc|R;Iq@+unK?OQ zsT`C+1dH++lS1uGBPq^iII^?Z{NPv*#a`* z>({TEnwk;}>KDm+=H})=_JBV%wf>Mhl);vM2P=J`ng!8j0QYjWwJls@iV(Au0Q{{e zCM2n0iYfvdLO)c|*4AcVU~pdP&1lRZW>>tU<>TWcE89Apqr7~eT%vOnF|b-xSokso zfJ|9VPEKxaD9wY++}!c^>OVn+T6D}T6WW{b$UMEO3ZNDcq~H%|$dr_n&*!%QY`D3( z_4o9ECvbLmUv(7Q5ncu40p^dlLfH&WQ7zY-U{NVCg4cN+*_xUzs7qbmCnXIOP&7&S zOfOyq1YzSd+i#fyN49wpT^8Wn@v*U6I5?mr(rg+T9c2v$Ds906*uc;b4!Vud)93c~ z^!(&6Z?c5|4xOEyrQ~_z2%34`{K)r|Wk>HQ`2S4QDfp&gj2DJhZ&#!=plK~LnUL4f3S}8>gG7TfXdaoh0wEf53)Zrrk|t4G&_!0 zGTz-CNn?uC7IzIOPL?4(`+zyWXSB5E0W|cxO@^0J?0*i<=xG6hcK7rUFoQe;vD&mKSov-BEum#}5yVX^1-m2+_We6s3Og5< zEl8J65WZwlZ7)&&w(BcM5hT7=3E#ye5}BBkEjs@m1%$^6OAyp#|6 z5D>z=?>Xpd@vyNicQE>8yYTKgV>F&uZD;EJ?1RhjW>>3@dfZ@LX79Y>ATb2!EP%=V%dlR%M zz~ca3!q>VADFC%Yox(x8&sg0+PtV4H)YZk&gfSbd3MlD7K7ddS4Gk3}%bl5-(Jp%Z zzSe*uYOMUp=4`zv4^QvN2pMwJ&$PS0-_6x^tN*Bf&Q~H;E$^=^#ydfZSr(K;U^55$ z*BcupEzHl?K%~dH;gdBV;(}_FVL~sM0LjWg7=Z%K+`&Cu+N;WR!WXOhJcyk`e*tC{ zu#lCPwMcSod>nmx+Hk~1OG}HgqJ<|i5VHppm_iaj=fcB(eS3S(Z{%ik@bww76~7Oh z!Q0q>y-WW0JL7*%{g>Cv=_T+*j5mPXtJ4mtLDkE=Qi+wUd(* zs8-fw-fy~`{4MYT5d?U9tp)SqnZfC7tm`WC_3L`G(2f7zHvj($xBt{~|EC!X^>XXlHmvJ5^BISvX63cj4Iq#6ne8YkFqhM<8hrr5hv@P_Up zA*TU>K>jYPEP+EJS1BD=bq5Pq4`XL@6m=^%S66dq)1WbI6ckz%IZ1JiPqX{;9tI?v z%~(fp`9mVfvLdn;2*d=P^pOI4m?T32#sfhe=gChi)kKlxlguxRexd1XfBou(PZ#bg zB^mjFB)+6Eph^Mmb(b`UuMIa3v}5l$WXklJOJ1K(zDTyrO!ly_WTuM=2eznEu{Pho zeFd9*$p72?=jeZZisakEil7d=mV^YFm%O;u>y)rC8E?J5krBV+l?PsG;@9WyKr?lK@$%kDUYCg;n%gIf+Dt6R(KGojn5;(EjMIJgJDzRLE!;=h zKD^}?q(7K^C+G6S{?b9X<=juL^Ba?=26=jMzLfH&Qxlqq!%io|+q7OHjF_w5Gm0mj zNqTxhoDCi6Z-v9SU5*jsotxwO2eX6H=?y4ysIjjNdr*!&TenODi>FsGe2_;kD>7_& zx1W;N^NX!o`e~l-6`yWdnFWwQMd)I-OpY8HOgO_ObKYetWj0u#M@nnUU6UmzH_UvJ z!BZ8SC@jv^=Of0bp1ZRgM*6ASvXuB;xbMItn<-f^scu`xMH?$-8 zX{N*wEA&mYef4oIg|@wP+OUGcLma(@Ze|uAdwA}&?V+ZgfB@g`(cr!L>a(>FXa6*=&vKwF5xKwOC8)=%&o-kt!v#zVEK9Tjm`&43x=DVPg9a` zFmAu~e&$Svq%7Cfb|*~n$2^x3mtQi&Q{7sinmS}<95uY$Gajj{DSR*Fcex7JVOjJT zZ+SQR^9Z-nTrKCVJt6sH+V1?(?p1LTR=56ybdOc8?pq{VdVgBWwP1W+G<&2+BTAj9 zZ`UjP_c!#2y_xMA9^{QuHM;8)H1whJGO&1hNawHNw!q&_l-Dh%x_BJr*wiPkr}%od zK9iOcmK?e)H=XRa2cpsz8?((`u8eWCkb;Ez`iczmiwqHck7<_~3~{U79p9rOC7gHx z-+P;TwjRy58n|Nj4qdLU3A3b+p-rM_LfQ z@t`Pu`$VPKRP%XH=wCeY*{AJ`Yr0>pOIUZ|bt{QzB3$b`41bO3yVS4o>l7le6Ggt! zLf)IwS607EP%*n*jIsE}cx%HMxp4a^T}ias)zRfnke6ft%cW_q$I0mS8*TXPPDzAl zg_E^)SB*iz(DycTH(Jws!xB)OC@6qep80JH?6L*XT9@eT<9&y2S-lS+Nfgl_R}#vT z%T{GCYx!ctD(Wks8URfYF)8$$-Kq1$ikIHSTXaCMNL|Dc-#!^_apAYPkgv2`^qEra zeFn)n8tpx!tdB{Snvml1y~?SmmmV*hexRa)tex`SI|Ai5;K3bBtt{uV%fBa>XnIF5 zv}r_gr+IV;Ke&i6?iJV;DS&+WoNHSh1<&a8nG zZ!I|Vz8%$?oN5^NyYqNqL2UO2Yo2)rRp;y4CMSGln(@e!Y3=4*m>ihmWI4tExv?OEmL4t}(J}D3dx5rdprmq=)JZr&`s|l9)MceysFXe)D zZg0S-qnVi8hvD_*%5j!3TN?(N2>-uQ&0WS%L*DvowKm&9=^`iJ?pU*6TQrB}McIEa zi$HmI;*(Y{1w)u9qY zFk;aPr>;?WCY=s?j8Z!6Yb0`WJyF$MuV3@>jy;V)x)Z4UnE03Y3E?S}7oCz9E%sBi z#vyTfV?TZ8MoC0_%=hNsx%LD9(1WvSK3+y_cjIB&q z&Hs4+5uCF3^a>62#U!M^oA>CeYB_dN+KeI-bt)QcEH(c1u;ZWeL85Rdqs4*CMybC* z9jn+H!^>gmFj@#}om<#i;l;k*?dOh|rT)BH2w<*8Cm5e#4<&5UsaUm-<+Zi6u)QBc<|VoSxv2usOM)ymZ>NIr~fTyiP+9)&+8V zr6YsCNhL-^0e^kdjJvk>z@W{F&pznW+dW~&s(nW+1V%xMja|SRY5LzM^mT5o=}ap0 zM(vhu+%I@zZgj0)3GkwaS?SzYtSjc{?EES3r4RR^9Buw5u4LpV>d~wj{oV$Rz{Wjb z%~3-#Dy{uftPGLlz?1@AOi@LBs1N^N@2R*wJjCqDN(atDeBuxSd6Os>YY|fjOUZ~z z;4K>1e2DQ3x<70(7wQ#6+Zc&Yqxb|n#788Pa)-3E*erx*!GYUR7GB!f7$<||->O)J zws{@@QWO#RPl%lA|us@}G>1T3%*X4jM!nUxIntoPMb}Dc6BAGg2 z-dp~T>M*YC^pcwiJ1AVx&2}fK``0OQiJbVR1?h!>Li zLQAnxmf0uCee>t_`kF5)`OX`aAKZN~HExB{Ufpd#DzF6J;9}g??j|z%U7eje_~_Z{ zuFn$D%1xN#shZecI^5}U5r65kiIiQmot@o4sUxF`#2foi2>U9V$tP8LQGpnT@saT# zb7!ksP1Ri7p54Wm^OGiu>`>CK7pkPXZ5{FrENXkmCL|K9c{_ZHuQcfA|i5Se)tD@_t~onqX=C}MjUmvp~tdODLN}F%RN$P%<4|q z&pv6evHpSV!)q|>x8`2UI61@K`2V=U} z^P=a_T^U}#q31*#=L#`Ukv_x_&sH05)`_`8etG~Jb{RuiJXCk5XJ#mf7806}aCsd^ zkY|UwBL;&lp|H^Ty5i$`pYut&eizw!L@QJY=(mQVZ!~2o>jLFMYsFQF#S2~VZa=BntJ?A244%&i5nups`=Z)d}n5FnO9-9BV_;LB%7OzeC{2q99@h-Y#8lA z8;iQ%wQDGz^*x=Y-;J}@=aj0iuU}qXPMOuEBH#cE_WGmuHYPX5qLMwbXRVr6Ha(v0|_son=kB8b?oh8g^f2MkM>DOm|e*QME zvrgk5?8+HJO+HtM%gal-IL2(gtRFwdfBpIehhNR{3z;^4=XW~EVTQk_M7Fr zRsD(-QFQn4u&jZzu(04oQcJ~rTU#^p@VNMlMOdLzjn;;mheb%;*x0xSujw5fC2WJ` z_5aXG28na*KbLN_)x#>1p% zVZk;GjIu_jB>lI)GgZoMInwI1+SB#r5sV_X$5=X&+}zaE1?VAx^eBQrLSVc>yU$a> z8xCrbgM~Kl;S`=Ne7rFphp7_HL{3Amqou&6CVsEgRoFg@dI2;(KEI%V{OQx|w@apx zk&&joQTY?wZywNnv>X9T@9&pmcp{%g;zov#j}IdA=g%MQ3Io_B?r%Mt?yscp-o4x2 z-rjHhfc;wk-Mg5!wzk5;&Fr^fV-!l*A)Oy53gV&Ay!?D{P4-)nfXqhZTYdeTxh9te zheZ*e%NJg`#lAP^A;H02-QC#)fv&(S9gfQqs;a71R?Of-4*mMMHg7&UIyAQ;&o_qW zfw?l#Ajs=$Ygufv$H&L60{=j;w{X70GK9TdU0o06Tf!=64VzuR$mdJT$mFs)!X?-~ zd3i0gcpL&xU}9pXx3W=-`U!cRZt>Xs*_)|Gr@-uM|19WnFjrPqwzp@?&0V9sKrxyj z?BeQrdUA4danW1<3|sykcS&jK{@xy=QVIzr0f*!bb& z$Dslk<@uGBG>;y|rKPpHY>te~5h^6G8`r=$M>A!pBv`D!#n8*x!+)3qhlhlOq&uET z(ZOOF6yA6E`EHLh7#bSd5{_a1ySX65#RUdOM@Ju~30L{wAwk}4ZCTxl{o+4TJ4RtG&gsvALIf)+TGZ2-Zd$t<<6rl$7F z=kmyWFu|=vJhZP*Rz+oWYilc++tP$0z#Z@^^+IJ1E-n=Xg<^Yil>%TeTh z!(tVF7D>OjkI!wyQ*MD#{JEvJMsQL#t*)LP3rkCn-9Px*GAyyFJw4J0=b=*VO6#LKGh1Ho(=C{^v^1Z? zLRST4-+Z-cuaB?qBVyv7D5}z5rJfr@Pr0}%MUj`Lx0lC|<~_#;hJ&@WwIdnAsZi+Y zZe@q{jsQ+b=heyP^q)WX2}Te|#}4WimLuh|F~q{bF*WHnG(2orZwnt4y^9bC&ZU#W zCLlOJJ5y3nsHv%$Xsy%l@cZ}W5tEv_Iu<^8u+Hb8*RNkkPzfgVn=vvnS}&Bhl$MlG zQc{NEQ#_`kip*TeCSe(yHu6u8k9WI6`r@}dBcdiEkUzOTv+?%63Bi3r{l-!3cMoZS zO6GeVourr;V=Jq0v4+Q(TYAr!nZrjnEA<5WNl8QcRz~IqYCyVzh=PJV1-19K5(124okK_NlvYxU_#`DKf3&ro%bs3`*I3fa zL>bgsYch5GeJ_|zQfIFE39;7C!@~nAbH9-fqZ7STyw2LjCOs_;OntnMSGuFTckKzn zRzXpI{OGybgFkmvE9_WNbY>`>9~v6^`0-==zxCy-6(O0KM)vmh1_qQnL`A`Yf#SBd zwrXnQz#PE?y)uKQ9|Z*)oguhIZ=K($h1LuIg-Yknw|dslbNYa8QeW?GZOys@DItCI z=xr`+Wp!0>rIQA@R9|0^jSbhyRur03e{yzKS6j=@#dYW?MM4O{I;-~k=LM=2l&42o zcf0omXI#IK(+UP%_~2uWvg&H5$)a}vgSN%~@2cjHu6#g8C@CqKoScku-aSAdoI&iK za$8XHJ7YYA;zD!WJRjkh<+x3%rP-BGSSqUclm}5 z_J)3bei|AiJAfFFAA=%2I!fZURJOA#|74=9JaW9+tJ$al@Jz|*>rsHCs)Mq!a>p&= z8*r_>y!^yAxYN|sREb_mYild0hXVBx&}~5}O%$rEq7pfWPMGg&8yW)ib^yQu2m%QX zA+?Zf;e?%?oo1a7z$^_`w}}EJw7`J0A8xx^;L@QXMGV}@R{>#=z&^>Tsimc*6huS;vdcIEJn_lN{U2ULQSu|MHZ#BZ8>{N* z6vc+X>+L8$|H_)!=C&C6Py>JX@ZoFn{gv)-kVi#L*VkU&-rjtC4Y9GYFHL)1km2Ek ztZZx)#D+YCU}X9ru*Hdhb-#ihfMR$a^Sm)j5YcV_6!UR05#}A_ESk18)vt*3CGKqmzOiBgGQ7P zA0MMBo_sLZ#O-(c2|*FtZL4@@)_87GE^@r{EBmVxhWd2ayRuV~~m%Y1nq};OP`Yw-c)02YLPJ}d z&pVJMSpsxc={_;x_{7A&aJXDE7i+vcL#4TDN^md+ONVC?zl+7&x9B;HGGtjcPsk$^ zzJIT>`^(E3KfCvji0gf8(a<_JA>k98kYVZ@wc^R}a0r5uIH0+@s_N+Y7zTrVJQrUD ziNMM#pFHS-yuGpr6G&1l0Fi2F;PcZ6Mip>_rliaQRuEm&_w#2!?<$Mk%e}olk%f5E zSIMB=1M;`&g_Z2pr}=&oZp z0Ff3J7S>dCCSs}qm0M^336eZjDFa}>wuT0+*w{P3Rdn_A{2XV*vsK@>`xK`=2#=1w z`%v`#*N4+RIQPq!4`IJh(wn(oynql8L`6sQ@$it+P`^wb8cgJ3vqV)+d-H7MJx+*3 zE)0{9IxQo^4|E+s`7j^?U%A>Z=|n{rdwatqvcnb7;`$>ZA~a^V(mu7OD;G{!)t>E6 zZ*Fe3G&j=;-~=s__8Un_b%W5YuCDe^@V0qhfU$WUssN*! zvnLR!fI^`T9slZ!io#o~y!*aCBcu`0I1K~yagdIGrlw-ttS*n1v$M0&u?QsJkl@Hj z;qaq+Um<{&`2aWud8C$(&Lvo&l>Zj6Lzn$=q;xLK3g8N;?{AMeB&dQ2?1qwFMpFyV zRvDu=8E>wy0|st@T?}$Xrf|Z|%?%wLo%_`*)i6hmN$IAhrk5{YvKmF~?Cc~ZC9SNi z^v5v?3JS8F)^z^<4Vahr*$$80Y;CkZVfO2{VFYJKM`G_k67!}~5fNq0ntuXJru*BI z!NEZ$#iSP&Lu4U-T7agDyH=j-e1T3T9}e%HlOQBl^3 zc{w>bB_#uY{%ASTJR`vYHz+}9&_Tetl7fOzO6Nnot(0aH(8ceOGll)0bO4%}-MIYf z@i)NlDiJgS-#yrSJQ}30kvE56dHDo_?ga$}!^jnx_YVSoec0OFE%i})^ym@rli(<( zv`iF>IAEfrv+X!CYbenE@o_a+OVyTH{7-XS6qEop3=|Y*De#xU{pLSM|HUaPWpQ!T zt1KZDFUh3_l9E!s%IM%AXn6BYE`{tWYW)Cn@bL}X zyy}27IQ6*a>_S2Y<8$kSiPKY4`Gti~NJs#d9NRTD8E|HfUK|~fy)?lPCjx2+NDY%$ z$wJf)M@zpz>;X~WU}cS6aK1R02UHK#B3S4Hw7}flRX}AYm)*OwUc#S$Oo)I_&hD!_ z!Og(GJSMB4jf(REwxDcozMjHkbNlZf79QF6l$2)B_ZI!{()-QiQcgmLTU29YiAeq1IY+z#+GudeWi;C6tviuznufhH(M`^F!63#ik^Q1XPyAZU1^ z&TF#3wkJkD$3Th9%*=qh{`@IM9vRAkF`KERX3?LdR6Z|4nyB>SH8lmrH``bObXOVy zPXtg!>j9|fkRV4Vr{c=W=!gjDt8`ijV;sNRjs{>`k11$G{cc-5Pt1&xfMiTbN&>w5 zJE0W}02x5@*}7f86n1xZAVEZ<*GxGYC1GJ$O)r3r6%`eL;bMr7S=BzkeGC?*myp0u z8I_&Ae7^q|q~;(HRX`Gn_}yNDh0h3yal)UQC$PgO>+jwEyzRK8gw^bk{5#x*4*$_* z$1bibA7r_HL*FT_@AY;esF1|hhPSbU>&{IEx*{=i4A5>Ui~82-2BH2iV7>+h&;LD3 z9CY*WV1ncT=JlUur7tu%HKm!5&}3jg5qkx<%#8632}XrKl9&G@t5Wp1rWpTk;mVKV z?jE49zAG_H5rf3KyIH(z_dMAk6uqsopKk_TJct%~y<6Gb+*}Yn`ihT_is~-_pVYaH33pmx8*;>FMdkX1Bq=e<|Q-z#on1Nb``8JQ|G2_lx4o0~g`HYG!tZq(9!_fI)| zdyQXlJ8V-vtFm&j!`~mV2taciBu&bQo4b2`Lql{_6ahZIu-|QCb@hFnm^<9gRGR`O zrPQm2VkXBd2M3Ddxw#g=iBb{mgTG8auw|s=NkIUKi4j0ca_Urx2~`80rNLUa{j?io zsDi>E^4|@gVB0q2b4nJ>YE4%wh<1EpqTe3~7myDRFfm_|MF0pMSkO~c914F(|Dpol zPWJwnz-Yl-t5ie-cKia&sqKn`2^NeKGWzLY{OPAa@3VM3hgd zsAszbq6ySCpf!NsX3SI?b%=k)>_-ZogG8{hv5BVOQ@%A1T>SZY51WC(W-V4#{x1j@ zv(#6KjHh1bdr!rQ)==N14s$Xy(?V8&k{lTxW(dOt{Nd_+UtdKk$LcP+83gE!Hx;hR{PWx;6qwDKy&@sV`YCW=i3p_RO`2Jad z{&57zJurn(Qd9&e0VF0M282`sd5CP;EuDL_o)5>qOr5RqW#vqL&J$&8{f+yF@XeyhleMD90M~oeFF`U?og=f{wxoo^3tchriPoJO;q$=;7DX2 zf#4yL$hGJOP^X}1`;KjJalZk!CiLm(1NdpQz$JiIKr-AQ2D$WYufQP1`{ID~izHBs zq+I=9N!h-?7%K?|GX)!7o97Hr&^0#x17G=0W^wGyN`(A8(mFabKtq+vZ#{eU3sw@GzbdAVpKe^}o3PBCchU0uP0>p)n7tEh#&`kWQQd2u0- z7cX7_)E7d?$3YVl^>uZ@xJtIeT~}U0u0MqUswzKH(ACxT^XJcj0eMQ&k3bz;8ykxw zZ&sD$Ra7eB`-=p zcFfPm6Dp$p>nbSUmHHT=bl<^Xlbx1U49su?Y$bRL_x_S0XzTM+#mgy zr^Gh1s~g^u;)G~wYC@`BIW-x$1{IVP7mG`w;QSg!xEm{%_R6Z0^iI^I? vCnZ!6PW}JklMLYLhyTX&4*#>OhL1bQm Date: Mon, 31 Jul 2023 10:23:36 +0200 Subject: [PATCH 58/68] n-to-1 change test to precise syntax --- test/unit/test_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 8e8766d4b..ab8b4e357 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -526,7 +526,7 @@ class TestJobList(unittest.TestCase): "DATES_TO": "[20020201:20020202],20020203,20020204,20020205", "MEMBERS_TO": "fc2", "CHUNKS_TO": "1,2,3,4,5,6", - "SPLITS_TO": "1*\\2,2*\\2" + "SPLITS_TO": "1*\\2,2*\\2,3*\\2,4*\\2" } child.split = 1 self.mock_job.split = 1 -- GitLab From d6ddae45a03bbd9aa0f797e9eab54610dc763dc9 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 3 Aug 2023 08:39:00 +0200 Subject: [PATCH 59/68] Fixed part of reviews, Fixed as_checkpoint() in python --- autosubmit/job/job.py | 18 ++++++------------ autosubmit/job/job_common.py | 3 ++- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index abd194546..a459d2d91 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -191,7 +191,6 @@ class Job(object): self.x11 = False self._local_logs = ('', '') self._remote_logs = ('', '') - self._checkpoint = None self.script_name = self.name + ".cmd" self.status = status self.prev_status = status @@ -259,18 +258,13 @@ class Job(object): @property @autosubmit_parameter(name='checkpoint') def checkpoint(self): - """Generates a checkpoint step for this job based on job.type""" - return self._checkpoint - - @checkpoint.setter - def checkpoint(self): - """Generates a checkpoint step for this job based on job.type""" + '''Generates a checkpoint step for this job based on job.type.''' if self.type == Type.PYTHON: - self._checkpoint = "checkpoint()" + return "checkpoint()" elif self.type == Type.R: - self._checkpoint = "checkpoint()" - else: # bash - self._checkpoint = "as_checkpoint" + return "checkpoint()" + else: # bash + return "as_checkpoint" def get_checkpoint_files(self): """ @@ -1456,7 +1450,7 @@ class Job(object): parameters['EXPORT'] = self.export parameters['PROJECT_TYPE'] = as_conf.get_project_type() self.wchunkinc = as_conf.get_wchunkinc(self.section) - for key,value in as_conf.jobs_data.get(self.section,{}).items(): + for key,value in as_conf.jobs_data[self.section].items(): parameters["CURRENT_"+key.upper()] = value return parameters diff --git a/autosubmit/job/job_common.py b/autosubmit/job/job_common.py index f6d34ccff..d705b1d1d 100644 --- a/autosubmit/job/job_common.py +++ b/autosubmit/job/job_common.py @@ -200,13 +200,14 @@ class StatisticsSnippetPython: stat_file = open(job_name_ptrn + '_STAT', 'w') stat_file.write('{0:.0f}\\n'.format(time.time())) stat_file.close() - ################### # Autosubmit Checkpoint ################### # Creates a new checkpoint file upton call based on the current numbers of calls to the function AS_CHECKPOINT_CALLS = 0 def as_checkpoint(): + global AS_CHECKPOINT_CALLS + global job_name_ptrn AS_CHECKPOINT_CALLS = AS_CHECKPOINT_CALLS + 1 open(job_name_ptrn + '_CHECKPOINT_' + str(AS_CHECKPOINT_CALLS), 'w').close() ################### -- GitLab From 62dfc04086a66a543f03c6c178a2ace8cd72c38e Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 3 Aug 2023 08:43:32 +0200 Subject: [PATCH 60/68] changed is false for not --- autosubmit/job/job_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index ee7a5c6e3..58402d439 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -135,7 +135,7 @@ class JobList(object): processed_job_list = [] for job in self._job_list: # We are assuming that the jobs are sorted in topological order (which is the default) if ( - job.member is None and found_member is False) or job.member in self._run_members or job.status not in [ + job.member is None and not found_member) or job.member in self._run_members or job.status not in [ Status.WAITING, Status.READY]: processed_job_list.append(job) if job.member is not None and len(str(job.member)) > 0: -- GitLab From d9741609aaf5cfdce297390d299196d4d6a3375a Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 3 Aug 2023 09:36:33 +0200 Subject: [PATCH 61/68] fixed two issues --- autosubmit/job/job_list.py | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 58402d439..2304cc096 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -359,7 +359,6 @@ class JobList(object): except Exception as e: pass if parameters.get(section, None) is None: - Log.printlog("WARNING: SECTION {0} is not defined in jobs.conf".format(section)) continue # raise AutosubmitCritical("Section:{0} doesn't exists.".format(section),7014) dependency_running_type = str(parameters[section].get('RUNNING', 'once')).lower() @@ -387,7 +386,7 @@ class JobList(object): @staticmethod - def _apply_filter(parent_value, filter_value, associative_list, level_to_check="DATES_FROM",child=None,parent=None): + def _apply_filter(parent_value, filter_value, associative_list, level_to_check="DATES_FROM", child=None, parent=None): """ Check if the current_job_value is included in the filter_value :param parent_value: @@ -399,7 +398,6 @@ class JobList(object): """ if "NONE".casefold() in str(parent_value).casefold(): return True - if parent and child and level_to_check.casefold() == "splits".casefold(): if not parent.splits: parent_splits = -1 @@ -412,6 +410,9 @@ class JobList(object): if parent_splits == child_splits: to_look_at_lesser = associative_list lesser_group = -1 + lesser = str(parent_splits) + greater = str(child_splits) + lesser_value = "parent" else: if parent_splits > child_splits: lesser = str(child_splits) @@ -446,19 +447,22 @@ class JobList(object): else: split_info = 1 # split_info: if a value is 1, it means that the filter is 1-to-1, if it is 2, it means that the filter is 1-to-2, etc. - if (split_info == 1 or level_to_check.casefold() != "splits".casefold()) and str(parent_value).casefold() == str(filter_).casefold() : - if child.split == parent_value: - return True - elif split_info > 1 and level_to_check.casefold() == "splits".casefold(): - # 1-to-X filter - to_look_at_greater = [associative_list[i:i + split_info] for i in - range(0, int(greater), split_info)] - if lesser_value == "parent": - if str(child.split) in to_look_at_greater[lesser_group]: - return True - else: - if str(parent_value) in to_look_at_greater[lesser_group]: + if child and parent: + if (split_info == 1 or level_to_check.casefold() != "splits".casefold()) and str(parent_value).casefold() == str(filter_).casefold(): + if child.split == parent_value: return True + elif split_info > 1 and level_to_check.casefold() == "splits".casefold(): + # 1-to-X filter + to_look_at_greater = [associative_list[i:i + split_info] for i in + range(0, int(greater), split_info)] + if lesser_value == "parent": + if str(child.split) in to_look_at_greater[lesser_group]: + return True + else: + if str(parent_value) in to_look_at_greater[lesser_group]: + return True + else: + filter_value += filter_ + "," else: filter_value += filter_ + "," filter_value = filter_value[:-1] @@ -582,6 +586,9 @@ class JobList(object): if end is None: end = value_list[-1] try: + if level_to_check == "CHUNKS_TO": + start = int(start) + end = int(end) return value_list[slice(value_list.index(start), value_list.index(end)+1, int(step))] except ValueError: return value_list[slice(0,len(value_list)-1,int(step))] -- GitLab From 132673a5ed8888741f14a169bb8090f7cb800b3a Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 3 Aug 2023 09:39:46 +0200 Subject: [PATCH 62/68] added `%AS_CHECKPOINT%` --- docs/source/userguide/defining_workflows/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/userguide/defining_workflows/index.rst b/docs/source/userguide/defining_workflows/index.rst index 49c35401d..ac4462b37 100644 --- a/docs/source/userguide/defining_workflows/index.rst +++ b/docs/source/userguide/defining_workflows/index.rst @@ -231,7 +231,7 @@ The `STATUS` keyword can be used to select the status of the dependency that you The status are ordered, so if you select "RUNNING" status, the task will be run if the dependency is running or any status after it. The `FROM_STEP` keyword can be used to select the step of the dependency that you want to check. The possible value is an integer. -Additionally, the target dependency, must call to %AS_CHECKPOINT% inside their scripts. This will create a checkpoint that will be used to check the amount of steps processed. +Additionally, the target dependency, must call to `%AS_CHECKPOINT%` inside their scripts. This will create a checkpoint that will be used to check the amount of steps processed. .. code-block:: yaml -- GitLab From 6272bc53219dee37e68a7d72752f6573b8e14835 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 3 Aug 2023 10:12:37 +0200 Subject: [PATCH 63/68] fixed color --- autosubmit/monitor/monitor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/autosubmit/monitor/monitor.py b/autosubmit/monitor/monitor.py index 53293ef11..bb6b841ca 100644 --- a/autosubmit/monitor/monitor.py +++ b/autosubmit/monitor/monitor.py @@ -235,10 +235,10 @@ class Monitor: def _check_final_status(self, job, child): # order of self._table - # child.edge_info is a tuple, I want to get first element of each tuple with a lambda + # the dictionary is composed by: label = None if len(child.edge_info) > 0: - if job in child.edge_info.get("FAILED",{}): + if job.name in child.edge_info.get("FAILED",{}): color = self._table.get(Status.FAILED,None) label = child.edge_info["FAILED"].get(job.name,0)[1] elif job.name in child.edge_info.get("RUNNING",{}): @@ -263,7 +263,7 @@ class Monitor: elif job.name in child.edge_info.get("SUBMITTED",{}): color = self._table.get(Status.SUBMITTED,None) else: - color = self._table.get(Status.COMPLETED,None) + return None, None if label and label == 0: label = None return color,label -- GitLab From 789e7295fafacb96cf507b4eecab46a0e0c30d45 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 3 Aug 2023 11:41:09 +0200 Subject: [PATCH 64/68] update doc --- .../userguide/defining_workflows/index.rst | 33 +++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/source/userguide/defining_workflows/index.rst b/docs/source/userguide/defining_workflows/index.rst index ac4462b37..39a41d7b2 100644 --- a/docs/source/userguide/defining_workflows/index.rst +++ b/docs/source/userguide/defining_workflows/index.rst @@ -228,9 +228,38 @@ The `STATUS` keyword can be used to select the status of the dependency that you * "COMPLETED": The task is completed. # Default * "SUSPENDED": The task is suspended. -The status are ordered, so if you select "RUNNING" status, the task will be run if the dependency is running or any status after it. +The status are ordered, so if you select "RUNNING" status, the task will be run if the parent is in any of the following statuses: "RUNNING", "QUEUING", "HELD", "SUBMITTED", "READY", "PREPARED", "DELAYED", "WAITING". + +The `FROM_STEP` keyword can be used to select the **internal** step of the dependency that you want to check. The possible value is an integer. + +To select an specific task, you have to combine the `STATUS` and `CHUNKS_TO` , `MEMBERS_TO` and `DATES_TO`, `SPLITS_TO` keywords. + +```yaml +JOBS: + A: + FILE: a + RUNNING: once + SPLITS: 1 + B: + FILE: b + RUNNING: once + SPLITS: 2 + DEPENDENCIES: A + C: + FILE: c + RUNNING: once + SPLITS: 1 + DEPENDENCIES: B + RECOVER_B_2: + FILE: fix_b + RUNNING: once + DEPENDENCIES: + B: + SPLIT_TO: "2" + STATUS: "RUNNING" + FROM_STEP: 1 +``` -The `FROM_STEP` keyword can be used to select the step of the dependency that you want to check. The possible value is an integer. Additionally, the target dependency, must call to `%AS_CHECKPOINT%` inside their scripts. This will create a checkpoint that will be used to check the amount of steps processed. .. code-block:: yaml -- GitLab From 8742852737de663c689dfe68c25ba228f4a92473 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 3 Aug 2023 11:43:12 +0200 Subject: [PATCH 65/68] update doc --- .../userguide/defining_workflows/index.rst | 50 ++++++++----------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/docs/source/userguide/defining_workflows/index.rst b/docs/source/userguide/defining_workflows/index.rst index 39a41d7b2..652bdc01e 100644 --- a/docs/source/userguide/defining_workflows/index.rst +++ b/docs/source/userguide/defining_workflows/index.rst @@ -230,12 +230,31 @@ The `STATUS` keyword can be used to select the status of the dependency that you The status are ordered, so if you select "RUNNING" status, the task will be run if the parent is in any of the following statuses: "RUNNING", "QUEUING", "HELD", "SUBMITTED", "READY", "PREPARED", "DELAYED", "WAITING". -The `FROM_STEP` keyword can be used to select the **internal** step of the dependency that you want to check. The possible value is an integer. +.. code-block:: yaml + + ini: + FILE: ini.sh + RUNNING: member + + sim: + FILE: sim.sh + DEPENDENCIES: ini sim-1 + RUNNING: chunk + + postprocess: + FILE: postprocess.sh + DEPENDENCIES: + SIM: + STATUS: "RUNNING" + RUNNING: chunk + + +The `FROM_STEP` keyword can be used to select the **internal** step of the dependency that you want to check. The possible value is an integer. Additionally, the target dependency, must call to `%AS_CHECKPOINT%` inside their scripts. This will create a checkpoint that will be used to check the amount of steps processed. To select an specific task, you have to combine the `STATUS` and `CHUNKS_TO` , `MEMBERS_TO` and `DATES_TO`, `SPLITS_TO` keywords. -```yaml -JOBS: +.. code-block:: yaml + A: FILE: a RUNNING: once @@ -257,31 +276,6 @@ JOBS: B: SPLIT_TO: "2" STATUS: "RUNNING" - FROM_STEP: 1 -``` - -Additionally, the target dependency, must call to `%AS_CHECKPOINT%` inside their scripts. This will create a checkpoint that will be used to check the amount of steps processed. - -.. code-block:: yaml - - ini: - FILE: ini.sh - RUNNING: member - - sim: - FILE: sim.sh - DEPENDENCIES: ini sim-1 - RUNNING: chunk - - postprocess: - FILE: postprocess.sh - DEPENDENCIES: - SIM: - STATUS: "RUNNING" - FROM_STEP: 0 - RUNNING: chunk - - Job frequency ~~~~~~~~~~~~~ -- GitLab From 632aca09af3bd3ebf2c3766264a29e0387cb30ad Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 4 Aug 2023 12:39:04 +0200 Subject: [PATCH 66/68] docs --- .../userguide/defining_workflows/index.rst | 52 +++++++++++++++++-- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/docs/source/userguide/defining_workflows/index.rst b/docs/source/userguide/defining_workflows/index.rst index 652bdc01e..4f498daea 100644 --- a/docs/source/userguide/defining_workflows/index.rst +++ b/docs/source/userguide/defining_workflows/index.rst @@ -210,9 +210,9 @@ Start conditions Sometimes you want to run a job only when a certain condition is met. For example, you may want to run a job only when a certain task is running. This can be achieved using the START_CONDITIONS feature based on the dependencies rework. -Start conditions are achieved by adding the keyword `STATUS` and optionally `FROM_STEP` keywords into any dependency that you want. +Start conditions are achieved by adding the keyword ``STATUS`` and optionally ``FROM_STEP`` keywords into any dependency that you want. -The `STATUS` keyword can be used to select the status of the dependency that you want to check. The possible values ( case-insensitive ) are: +The ``STATUS`` keyword can be used to select the status of the dependency that you want to check. The possible values ( case-insensitive ) are: * "WAITING": The task is waiting for its dependencies to be completed. * "DELAYED": The task is delayed by a delay condition. @@ -249,9 +249,53 @@ The status are ordered, so if you select "RUNNING" status, the task will be run RUNNING: chunk -The `FROM_STEP` keyword can be used to select the **internal** step of the dependency that you want to check. The possible value is an integer. Additionally, the target dependency, must call to `%AS_CHECKPOINT%` inside their scripts. This will create a checkpoint that will be used to check the amount of steps processed. +The ``FROM_STEP`` keyword can be used to select the **internal** step of the dependency that you want to check. The possible value is an integer. Additionally, the target dependency, must call to `%AS_CHECKPOINT%` inside their scripts. This will create a checkpoint that will be used to check the amount of steps processed. -To select an specific task, you have to combine the `STATUS` and `CHUNKS_TO` , `MEMBERS_TO` and `DATES_TO`, `SPLITS_TO` keywords. +.. code-block:: yaml + + A: + FILE: a.sh + RUNNING: once + SPLITS: 2 + A_2: + FILE: a_2.sh + RUNNING: once + DEPENDENCIES: + A: + SPLIT_TO: "2" + STATUS: "RUNNING" + FROM_STEP: 2 + +There is now a new function that is automatically added in your scripts which is called ``as_checkpoint``. This is the function that is generating the checkpoint file. You can see the function below: + +.. code-block:: bash + + ################### + # AS CHECKPOINT FUNCTION + ################### + # Creates a new checkpoint file upon call based on the current numbers of calls to the function + + AS_CHECKPOINT_CALLS=0 + function as_checkpoint { + AS_CHECKPOINT_CALLS=$((AS_CHECKPOINT_CALLS+1)) + touch ${job_name_ptrn}_CHECKPOINT_${AS_CHECKPOINT_CALLS} + } + +And what you would have to include in your target dependency or dependencies is the call to this function which in this example is a.sh. + +The amount of calls is strongly related to the ``FROM_STEP`` value. + +``$expid/proj/$projname/as.sh`` + +.. code-block:: bash + + ##compute somestuff + as_checkpoint + ## compute some more stuff + as_checkpoint + + +To select an specific task, you have to combine the ``STATUS`` and ``CHUNKS_TO`` , ``MEMBERS_TO`` and ``DATES_TO``, ``SPLITS_TO`` keywords. .. code-block:: yaml -- GitLab From 56f8115d5ca89b8acca0a9fe673b0da1680bbff5 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 7 Aug 2023 10:21:00 +0200 Subject: [PATCH 67/68] changed to f-string --- autosubmit/platforms/platform.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/autosubmit/platforms/platform.py b/autosubmit/platforms/platform.py index 059609cc6..79ea7919a 100644 --- a/autosubmit/platforms/platform.py +++ b/autosubmit/platforms/platform.py @@ -523,12 +523,12 @@ class Platform(object): """ if job.current_checkpoint_step < job.max_checkpoint_step: - remote_checkpoint_path = f"{self.get_files_path()}/CHECKPOINT_" - self.get_file(remote_checkpoint_path+str(job.current_checkpoint_step), False, ignore_log=True) - while self.check_file_exists(remote_checkpoint_path+str(job.current_checkpoint_step)) and job.current_checkpoint_step < job.max_checkpoint_step: - self.remove_checkpoint_file(remote_checkpoint_path+str(job.current_checkpoint_step)) + remote_checkpoint_path = f'{self.get_files_path()}/CHECKPOINT_' + self.get_file(f'{remote_checkpoint_path}{str(job.current_checkpoint_step)}', False, ignore_log=True) + while self.check_file_exists(f'{remote_checkpoint_path}{str(job.current_checkpoint_step)}') and job.current_checkpoint_step < job.max_checkpoint_step: + self.remove_checkpoint_file(f'{remote_checkpoint_path}{str(job.current_checkpoint_step)}') job.current_checkpoint_step += 1 - self.get_file(remote_checkpoint_path+str(job.current_checkpoint_step), False, ignore_log=True) + self.get_file(f'{remote_checkpoint_path}{str(job.current_checkpoint_step)}', False, ignore_log=True) def get_completed_files(self, job_name, retries=0, recovery=False, wrapper_failed=False): """ Get the COMPLETED file of the given job -- GitLab From ced293ac87d181453c30ba0b733964cd66d738eb Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 7 Aug 2023 14:53:04 +0200 Subject: [PATCH 68/68] docs --- docs/source/userguide/defining_workflows/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/userguide/defining_workflows/index.rst b/docs/source/userguide/defining_workflows/index.rst index 4f498daea..257178ee7 100644 --- a/docs/source/userguide/defining_workflows/index.rst +++ b/docs/source/userguide/defining_workflows/index.rst @@ -440,7 +440,7 @@ This behaviour can be achieved using the SPLITS attribute to specify the number It is also possible to specify the splits for each task using the SPLITS_FROM and SPLITS_TO attributes. -There is also an special character '*' that can be used to specify that the split is 1-to-1 dependency. +There is also an special character '*' that can be used to specify that the split is 1-to-1 dependency. In order to use this character, you have to specify both SPLITS_FROM and SPLITS_TO attributes. .. code-block:: yaml -- GitLab