From 8390dd901b89a06142799461ee468623afcab26e Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 20 Sep 2023 08:37:20 +0200 Subject: [PATCH 1/3] first try --- autosubmit/autosubmit.py | 4 +++- autosubmit/job/job.py | 4 ++-- autosubmit/job/job_list.py | 1 - 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 0050026a8..ead9cc8c3 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -1385,7 +1385,7 @@ class Autosubmit: job_list.check_scripts(as_conf) - job_list.update_list(as_conf, False) + job_list.update_list(as_conf, True) # Loading parameters again Autosubmit._load_parameters(as_conf, job_list, submitter.platforms) # Related to TWO_STEP_START new variable defined in expdef @@ -2126,6 +2126,7 @@ class Autosubmit: else: Log.debug("\nJobs prepared for {1}: {0}", len( job_list.get_prepared(platform)), platform.name) + job_list.update_list(as_conf, False) packages_to_submit = JobPackager( as_conf, platform, job_list, hold=hold).build_packages() if not inspect: @@ -2133,6 +2134,7 @@ class Autosubmit: valid_packages_to_submit = [] # type: List[JobPackageBase] for package in packages_to_submit: try: + # If called from inspect command or -cw if only_wrappers or inspect: if hasattr(package, "name"): diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index ccda2ff84..2fd177426 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1241,7 +1241,7 @@ class Job(object): template = '' if as_conf.get_remote_dependencies(): if self.type == Type.BASH: - template = 'sleep 360' + "\n" + template = 'sleep 1' + "\n" elif self.type == Type.PYTHON: template = 'time.sleep(5)' + "\n" elif self.type == Type.R: @@ -1250,7 +1250,7 @@ class Job(object): template_file.close() else: if self.type == Type.BASH: - template = 'sleep 360' + template = 'sleep 1' elif self.type == Type.PYTHON or self.type == Type.PYTHON2 or self.type == Type.PYTHON3: template = 'time.sleep(5)' elif self.type == Type.R: diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index f2fbe86d2..2ca7179c5 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -1210,7 +1210,6 @@ class JobList(object): :return: jobs in platforms :rtype: list """ - in_queue = self.get_submitted(platform) + self.get_running(platform) + self.get_queuing( platform) + self.get_unknown(platform) + self.get_held_jobs(platform) if wrapper: -- GitLab From f7d4c27622dec316ab550e8909ca1ff0914e7bee Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 20 Sep 2023 12:32:30 +0200 Subject: [PATCH 2/3] Wrappers can now be preview even if they fail Fixed an issue with mixed,strictp olicy --- autosubmit/autosubmit.py | 9 ++++++--- autosubmit/job/job_list.py | 9 +++++++++ autosubmit/job/job_packager.py | 5 +++-- autosubmit/job/job_packages.py | 22 ++++++++++++---------- 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index ead9cc8c3..98bb0e07e 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -2522,9 +2522,12 @@ class Autosubmit: job.children = job.children - referenced_jobs_to_remove job.parents = job.parents - referenced_jobs_to_remove - - Autosubmit.generate_scripts_andor_wrappers(as_conf, job_list_wrappers, jobs_wr, - packages_persistence, True) + try: + Autosubmit.generate_scripts_andor_wrappers(as_conf, job_list_wrappers, jobs_wr, + packages_persistence, True) + except AutosubmitCritical as e: + Log.printlog("Incomplete preview of wrappers",5000) + Log.printlog(e.message,7000) packages = packages_persistence.load(True) packages += JobPackagePersistence(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "pkl"), diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 2ca7179c5..0ee972fa4 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -1503,6 +1503,15 @@ class JobList(object): def parameters(self, value): self._parameters = value + def check_possible_ready_jobs(self,job_list): + jobs = [ job for job in self.get_active() if job not in job_list ] + jobs += [ job for job in self.get_ready() if job not in job_list ] + for job in self.get_waiting(): + tmp = [parent for parent in job.parents if parent.status == Status.COMPLETED] + if len(tmp) == len(job.parents): + Log.debug("Job {0} is ready to run".format(job.name)) + jobs.append(job) + return list(set(jobs)) def update_list(self, as_conf, store_change=True, fromSetStatus=False, submitter=None, first_time=False): # type: (AutosubmitConfig, bool, bool, object, bool) -> bool """ diff --git a/autosubmit/job/job_packager.py b/autosubmit/job/job_packager.py index ea3e0b9f1..9d664931e 100644 --- a/autosubmit/job/job_packager.py +++ b/autosubmit/job/job_packager.py @@ -449,7 +449,7 @@ class JobPackager(object): message += "\nCheck your configuration: Only jobs_in_wrappers are active, check their dependencies." if not balanced: message += "\nPackages are not well balanced: Check your dependencies(This is not the main cause of the Critical error)" - if len(self._jobs_list.get_in_queue()) == 0: + if len(self._jobs_list.check_possible_ready_jobs(p.jobs)) == 0: raise AutosubmitCritical(message, 7014) elif self.wrapper_policy[self.current_wrapper_section] == "mixed": error = True @@ -497,7 +497,8 @@ class JobPackager(object): message += "\nCheck your configuration: Only jobs_in_wrappers are active, check your jobs_in_wrapper dependencies." if not balanced: message += "\nPackages are not well balanced! (This is not the main cause of the Critical error)" - if len(self._jobs_list.get_in_queue()) == 0: # When there are not more posible jobs, autosubmit will stop the execution + if len(self._jobs_list.check_possible_ready_jobs(p.jobs)) == 0: # When there are not more posible jobs, autosubmit will stop the execution + raise AutosubmitCritical(message, 7014) else: for job in p.jobs: diff --git a/autosubmit/job/job_packages.py b/autosubmit/job/job_packages.py index 8e3c3409a..bd4972ddf 100644 --- a/autosubmit/job/job_packages.py +++ b/autosubmit/job/job_packages.py @@ -149,16 +149,18 @@ class JobPackageBase(object): try: # get one job of each section jobs by section if only_generate: - sections = configuration.get_wrapper_jobs(self.current_wrapper_section) - if "&" in sections: - sections.split("&") - elif " " in sections: - sections.split(" ") - else: - sections = [sections] - for section in sections: - if str(configuration._jobs_parser.get_option(section, "CHECK", 'True')).lower() == str(Job.CHECK_ON_SUBMISSION).lower(): - exit = True + # has self.current_wrapper_section attr + if hasattr(self, 'current_wrapper_section'): + sections = configuration.get_wrapper_jobs(self.current_wrapper_section) + if "&" in sections: + sections.split("&") + elif " " in sections: + sections.split(" ") + else: + sections = [sections] + for section in sections: + if str(configuration._jobs_parser.get_option(section, "CHECK", 'True')).lower() == str(Job.CHECK_ON_SUBMISSION).lower(): + exit = True if not exit: if len(self.jobs) < thread_number: for job in self.jobs: -- GitLab From 2ac78d648e349cb1ae13bede4b2dae48bf9a29e9 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 20 Sep 2023 12:40:33 +0200 Subject: [PATCH 3/3] revert some debug code --- autosubmit/autosubmit.py | 2 +- autosubmit/job/job.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 98bb0e07e..f282733f0 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -1385,7 +1385,7 @@ class Autosubmit: job_list.check_scripts(as_conf) - job_list.update_list(as_conf, True) + job_list.update_list(as_conf, False) # Loading parameters again Autosubmit._load_parameters(as_conf, job_list, submitter.platforms) # Related to TWO_STEP_START new variable defined in expdef diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 2fd177426..505d53c50 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1250,7 +1250,7 @@ class Job(object): template_file.close() else: if self.type == Type.BASH: - template = 'sleep 1' + template = 'sleep 5' elif self.type == Type.PYTHON or self.type == Type.PYTHON2 or self.type == Type.PYTHON3: template = 'time.sleep(5)' elif self.type == Type.R: -- GitLab