From f1b5d0ef3891c69efd73e7046490a3f4df310bdf Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 17 Sep 2020 12:01:59 +0200 Subject: [PATCH 1/5] Added semi-strict Policy --- autosubmit/job/job_packager.py | 15 +++++++++++++-- docs/source/usage/wrappers.rst | 4 +++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/autosubmit/job/job_packager.py b/autosubmit/job/job_packager.py index ae90a5e6b..ff1f2ac46 100644 --- a/autosubmit/job/job_packager.py +++ b/autosubmit/job/job_packager.py @@ -248,7 +248,7 @@ class JobPackager(object): for job in p.jobs: independent_inner_job = True for parent in job.parents: - if parent in p.jobs and parent.name != job.name: # This job depends on others inner jobs? T/F + if parent in p.jobs and parent.name != job.name: # This job depends on others inner jobs? T independent_inner_job = False break tmp = [parent for parent in job.parents if @@ -259,7 +259,18 @@ class JobPackager(object): Log.debug("Wrapper policy is set to strict, there is a deadlock so autosubmit will sleep a while") for job in p.jobs: job.packed = False - elif deadlock and self.wrapper_policy != "strict": + elif deadlock and self.wrapper_policy == "semi-strict": + Log.debug("Wrapper policy is set to semi-strict, there is a deadlock") + for job in p.jobs: + job.packed = False + if job.fail_count > 0 and job.status == Status.READY: + Log.debug("Wrapper policy is set to semi-strict, there is a failed job that will be sent sequential") + if job.type == Type.PYTHON and not self._platform.allow_python_jobs: + package = JobPackageSimpleWrapped([job]) + else: + package = JobPackageSimple([job]) + packages_to_submit.append(package) + elif deadlock and self.wrapper_policy != "strict" and self.wrapper_policy != "semi-strict": Log.debug("Wrapper policy is set to flexible and there is a deadlock, As will submit the jobs sequentally") for job in p.jobs: job.packed = False diff --git a/docs/source/usage/wrappers.rst b/docs/source/usage/wrappers.rst index de0a5f843..fa6d8823f 100644 --- a/docs/source/usage/wrappers.rst +++ b/docs/source/usage/wrappers.rst @@ -58,8 +58,10 @@ Number of jobs in a package - **MIN_WRAPPED** can be defined in ``autosubmit_cxxx.conf`` in order to limit the minimum number of jobs that a wrapper can contain - If not defined, it considers that **MIN_WRAPPED** is 2. - If **POLICY** is flexible and it is not possible to wrap **MIN_WRAPPED** or more tasks, these tasks will be submitted as individual jobs, as long as the condition is not satisfied. + - If **POLICY** is semi-strict and it is not possible to wrap **MIN_WRAPPED** or more tasks, these tasks will not be submitted as individual jobs unless there are failed jobs. - If **POLICY** is strict and it is not possible to wrap **MIN_WRAPPED** or more tasks, these tasks will not be submitted until there are enough tasks to build a package. - - strict policy can cause deadlocks. + - strict policy can cause **deadlocks** even with failed jobs. + - semi-strict policy can also cause **deadlocks** unless there are failed jobs. Wrapper check time -- GitLab From fc3a885a6517756e17b47fde92aa8923893a666a Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 17 Sep 2020 12:16:29 +0200 Subject: [PATCH 2/5] Added semi-strict Policy, will send failed jobs if any before others jobs. --- autosubmit/job/job_packager.py | 118 ++++++++++++++++++--------------- 1 file changed, 66 insertions(+), 52 deletions(-) diff --git a/autosubmit/job/job_packager.py b/autosubmit/job/job_packager.py index ff1f2ac46..e1c4a2227 100644 --- a/autosubmit/job/job_packager.py +++ b/autosubmit/job/job_packager.py @@ -239,62 +239,76 @@ class JobPackager(object): jobs_to_submit_by_section[section], max_wrapped_jobs, section)) if wrapped: for p in built_packages_tmp: - for job in p.jobs: - job.packed = True - if len(p.jobs) >= min_wrapped_jobs: # if the quantity is not enough, don't make the wrapper - packages_to_submit.append(p) + #Check failed jobs first + if self.wrapper_policy == "semi-strict": + for job in p.jobs: + if job.fail_count == 0: + continue + Log.debug("Wrapper policy is set to semi-flexible and there are failed jobs") + job.packed = False + if job.status == Status.READY: + if job.type == Type.PYTHON and not self._platform.allow_python_jobs: + package = JobPackageSimpleWrapped([job]) + else: + package = JobPackageSimple([job]) + packages_to_submit.append(package) else: - deadlock = True for job in p.jobs: - independent_inner_job = True - for parent in job.parents: - if parent in p.jobs and parent.name != job.name: # This job depends on others inner jobs? T - independent_inner_job = False - break - tmp = [parent for parent in job.parents if - independent_inner_job and parent.status == Status.COMPLETED] - if len(tmp) != len(job.parents): - deadlock = False - if deadlock and self.wrapper_policy == "strict": - Log.debug("Wrapper policy is set to strict, there is a deadlock so autosubmit will sleep a while") - for job in p.jobs: - job.packed = False - elif deadlock and self.wrapper_policy == "semi-strict": - Log.debug("Wrapper policy is set to semi-strict, there is a deadlock") - for job in p.jobs: - job.packed = False - if job.fail_count > 0 and job.status == Status.READY: - Log.debug("Wrapper policy is set to semi-strict, there is a failed job that will be sent sequential") - if job.type == Type.PYTHON and not self._platform.allow_python_jobs: - package = JobPackageSimpleWrapped([job]) - else: - package = JobPackageSimple([job]) - packages_to_submit.append(package) - elif deadlock and self.wrapper_policy != "strict" and self.wrapper_policy != "semi-strict": - Log.debug("Wrapper policy is set to flexible and there is a deadlock, As will submit the jobs sequentally") - for job in p.jobs: - job.packed = False - if job.status == Status.READY: - if job.type == Type.PYTHON and not self._platform.allow_python_jobs: - package = JobPackageSimpleWrapped([job]) - else: - package = JobPackageSimple([job]) - packages_to_submit.append(package) - elif not deadlock: # last case - last_inner_job = False - for job in p.jobs: - all_children_out_wrapper = True - for child in job.children: # Check if job is considered child - if child in p.jobs and child.name != job.name: - all_children_out_wrapper = False - if all_children_out_wrapper: - last_inner_job = True - break - if last_inner_job: # Last case - packages_to_submit.append(p) + job.packed = True + if len(p.jobs) >= min_wrapped_jobs: # if the quantity is enough, make the wrapper + packages_to_submit.append(p) else: + deadlock = True for job in p.jobs: - job.packed = False + independent_inner_job = True + for parent in job.parents: + if parent in p.jobs and parent.name != job.name: # This job depends on others inner jobs? T + independent_inner_job = False + break + tmp = [parent for parent in job.parents if + independent_inner_job and parent.status == Status.COMPLETED] + if len(tmp) != len(job.parents): + deadlock = False + if deadlock and self.wrapper_policy == "strict": + Log.debug("Wrapper policy is set to strict, there is a deadlock so autosubmit will sleep a while") + for job in p.jobs: + job.packed = False + elif deadlock and self.wrapper_policy == "semi-strict": + Log.debug("Wrapper policy is set to semi-strict, there is a deadlock") + for job in p.jobs: + job.packed = False + if job.fail_count > 0 and job.status == Status.READY: + Log.debug("Wrapper policy is set to semi-strict, there is a failed job that will be sent sequential") + if job.type == Type.PYTHON and not self._platform.allow_python_jobs: + package = JobPackageSimpleWrapped([job]) + else: + package = JobPackageSimple([job]) + packages_to_submit.append(package) + elif deadlock and self.wrapper_policy != "strict" and self.wrapper_policy != "semi-strict": + Log.debug("Wrapper policy is set to flexible and there is a deadlock, As will submit the jobs sequentally") + for job in p.jobs: + job.packed = False + if job.status == Status.READY: + if job.type == Type.PYTHON and not self._platform.allow_python_jobs: + package = JobPackageSimpleWrapped([job]) + else: + package = JobPackageSimple([job]) + packages_to_submit.append(package) + elif not deadlock: # last case + last_inner_job = False + for job in p.jobs: + all_children_out_wrapper = True + for child in job.children: # Check if job is considered child + if child in p.jobs and child.name != job.name: + all_children_out_wrapper = False + if all_children_out_wrapper: + last_inner_job = True + break + if last_inner_job: # Last case + packages_to_submit.append(p) + else: + for job in p.jobs: + job.packed = False else: for job in jobs_to_submit_by_section[section]: if job.type == Type.PYTHON and not self._platform.allow_python_jobs: -- GitLab From 5924403c88d6db499f93612fe9763c113acf3e6e Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 17 Sep 2020 12:30:47 +0200 Subject: [PATCH 3/5] added mixed #548 --- autosubmit/job/job_packager.py | 10 +++++----- docs/source/usage/wrappers.rst | 5 ++--- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/autosubmit/job/job_packager.py b/autosubmit/job/job_packager.py index e1c4a2227..351b57e47 100644 --- a/autosubmit/job/job_packager.py +++ b/autosubmit/job/job_packager.py @@ -240,11 +240,11 @@ class JobPackager(object): if wrapped: for p in built_packages_tmp: #Check failed jobs first - if self.wrapper_policy == "semi-strict": + if self.wrapper_policy == "mixed": for job in p.jobs: if job.fail_count == 0: continue - Log.debug("Wrapper policy is set to semi-flexible and there are failed jobs") + Log.debug("Wrapper policy is set to mixed and there are failed jobs") job.packed = False if job.status == Status.READY: if job.type == Type.PYTHON and not self._platform.allow_python_jobs: @@ -273,8 +273,8 @@ class JobPackager(object): Log.debug("Wrapper policy is set to strict, there is a deadlock so autosubmit will sleep a while") for job in p.jobs: job.packed = False - elif deadlock and self.wrapper_policy == "semi-strict": - Log.debug("Wrapper policy is set to semi-strict, there is a deadlock") + elif deadlock and self.wrapper_policy == "mixed": + Log.debug("Wrapper policy is set to mixed, there is a deadlock") for job in p.jobs: job.packed = False if job.fail_count > 0 and job.status == Status.READY: @@ -284,7 +284,7 @@ class JobPackager(object): else: package = JobPackageSimple([job]) packages_to_submit.append(package) - elif deadlock and self.wrapper_policy != "strict" and self.wrapper_policy != "semi-strict": + elif deadlock and self.wrapper_policy != "strict" and self.wrapper_policy != "mixed": Log.debug("Wrapper policy is set to flexible and there is a deadlock, As will submit the jobs sequentally") for job in p.jobs: job.packed = False diff --git a/docs/source/usage/wrappers.rst b/docs/source/usage/wrappers.rst index fa6d8823f..c961f1a19 100644 --- a/docs/source/usage/wrappers.rst +++ b/docs/source/usage/wrappers.rst @@ -58,10 +58,9 @@ Number of jobs in a package - **MIN_WRAPPED** can be defined in ``autosubmit_cxxx.conf`` in order to limit the minimum number of jobs that a wrapper can contain - If not defined, it considers that **MIN_WRAPPED** is 2. - If **POLICY** is flexible and it is not possible to wrap **MIN_WRAPPED** or more tasks, these tasks will be submitted as individual jobs, as long as the condition is not satisfied. - - If **POLICY** is semi-strict and it is not possible to wrap **MIN_WRAPPED** or more tasks, these tasks will not be submitted as individual jobs unless there are failed jobs. + - If **POLICY** is mixed and there are failed jobs inside a wrapper, these jobs will be submitted as individual jobs. - If **POLICY** is strict and it is not possible to wrap **MIN_WRAPPED** or more tasks, these tasks will not be submitted until there are enough tasks to build a package. - - strict policy can cause **deadlocks** even with failed jobs. - - semi-strict policy can also cause **deadlocks** unless there are failed jobs. + - strict and mixed policies can cause **deadlocks**. Wrapper check time -- GitLab From bdc3953f2cd351a609005e9cbdca642ac8548275 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 17 Sep 2020 12:35:52 +0200 Subject: [PATCH 4/5] Added a failed_checker loop #548 --- autosubmit/job/job_packager.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/autosubmit/job/job_packager.py b/autosubmit/job/job_packager.py index 351b57e47..2f1e9b37d 100644 --- a/autosubmit/job/job_packager.py +++ b/autosubmit/job/job_packager.py @@ -238,9 +238,15 @@ class JobPackager(object): built_packages_tmp.append(self._build_hybrid_package( jobs_to_submit_by_section[section], max_wrapped_jobs, section)) if wrapped: + for p in built_packages_tmp: + failed_innerjobs = False #Check failed jobs first - if self.wrapper_policy == "mixed": + for job in p.jobs: + if job.fail_count > 0: + failed_innerjobs = True + break + if failed_innerjobs and self.wrapper_policy == "mixed": for job in p.jobs: if job.fail_count == 0: continue -- GitLab From 3b741d063cb881ea46313a208a91f3adbb2483e5 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 17 Sep 2020 12:44:00 +0200 Subject: [PATCH 5/5] changed job.packed of compatible wrappers #548 --- autosubmit/job/job_packager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/autosubmit/job/job_packager.py b/autosubmit/job/job_packager.py index 2f1e9b37d..88075d990 100644 --- a/autosubmit/job/job_packager.py +++ b/autosubmit/job/job_packager.py @@ -246,7 +246,7 @@ class JobPackager(object): if job.fail_count > 0: failed_innerjobs = True break - if failed_innerjobs and self.wrapper_policy == "mixed": + if failed_innerjobs and str(self.wrapper_policy) == "mixed": for job in p.jobs: if job.fail_count == 0: continue @@ -259,9 +259,9 @@ class JobPackager(object): package = JobPackageSimple([job]) packages_to_submit.append(package) else: - for job in p.jobs: - job.packed = True if len(p.jobs) >= min_wrapped_jobs: # if the quantity is enough, make the wrapper + for job in p.jobs: + job.packed = True packages_to_submit.append(p) else: deadlock = True -- GitLab