diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 8e51ac58bd5e81220db6a78ab43751ad9ce9625c..6b7d3527b4ef25f28bb4616d59a117de93c42b9e 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -663,7 +663,7 @@ class Job(object): pass return - def update_status(self, copy_remote_logs=False): + def update_status(self, copy_remote_logs=False, failed_file=False): """ Updates job status, checking COMPLETED file if needed @@ -694,15 +694,17 @@ class Job(object): elif self.status == Status.COMPLETED: Log.result("Job {0} is COMPLETED", self.name) elif self.status == Status.FAILED: - Log.printlog("Job {0} is FAILED. Checking completed files to confirm the failure...".format( - self.name), 3000) - self._platform.get_completed_files( - self.name, wrapper_failed=self.packed) - self.check_completion() - if self.status == Status.COMPLETED: - Log.result("Job {0} is COMPLETED", self.name) - else: - self.update_children_status() + if not failed_file: + Log.printlog("Job {0} is FAILED. Checking completed files to confirm the failure...".format( + self.name), 3000) + self._platform.get_completed_files( + self.name, wrapper_failed=self.packed) + self.check_completion() + if self.status == Status.COMPLETED: + Log.result("Job {0} is COMPLETED", self.name) + else: + self.update_children_status() + elif self.status == Status.UNKNOWN: Log.printlog("Job {0} is UNKNOWN. Checking completed files to confirm the failure...".format( self.name), 3000) @@ -1506,6 +1508,7 @@ class WrapperJob(Job): self.status = Status.FAILED def _check_finished_job(self, job, failed_file=False): + job.new_status = Status.FAILED if not failed_file: wait = 2 retries = 2 @@ -1515,13 +1518,9 @@ class WrapperJob(Job): if output is None or output == '': sleep(wait) retries = retries - 1 - if failed_file or (output is not None and output != '' and 'COMPLETED' in output): - job.new_status = Status.COMPLETED - job.update_status(self.as_config.get_copy_remote_logs() == 'true') - else: - #Log.info("No completed filed found, setting {0} to FAILED...".format(job.name)) - job.status = Status.FAILED - #job.update_status(self.as_config.get_copy_remote_logs() == 'true') + if output is not None and output != '' and 'COMPLETED' in output: + job.new_status = Status.COMPLETED + job.update_status(self.as_config.get_copy_remote_logs() == 'true',failed_file) self.running_jobs_start.pop(job, None) def update_failed_jobs(self, canceled_wrapper=False): @@ -1530,7 +1529,7 @@ class WrapperJob(Job): for job in running_jobs: if job.platform.check_file_exists('{0}_FAILED'.format(job.name), wrapper_failed=True): if job.platform.get_file('{0}_FAILED'.format(job.name), False, wrapper_failed=True): - self._check_finished_job(job) + self._check_finished_job(job, True) else: self.inner_jobs_running.append(job)