From cf9b8e40f0826f7618270c4539757aa744ab332d Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 29 Feb 2024 09:18:59 +0100 Subject: [PATCH 1/5] Traces hidden Changed pkl open --- autosubmit/autosubmit.py | 2 +- autosubmit/job/job_list.py | 4 +++ autosubmit/job/job_list_persistence.py | 14 +++++--- bin/autosubmit | 47 ++++++++++---------------- 4 files changed, 31 insertions(+), 36 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 8653285dc..1895cd82b 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -4798,7 +4798,7 @@ class Autosubmit: raise AutosubmitError(e.message, e.code, e.trace) except AutosubmitCritical as e: # TODO: == "" or is None? - if e.trace == "": + if e.trace == "" or not e.trace: e.trace = traceback.format_exc() raise AutosubmitCritical(e.message, e.code, e.trace) except BaseException as e: diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index eca3aa9ca..6e924f109 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -207,6 +207,8 @@ class JobList(object): self.graph = self.load() if type(self.graph) is not DiGraph: self.graph = nx.DiGraph() + except AutosubmitCritical: + raise except: self.graph = nx.DiGraph() self._dic_jobs = DicJobs(date_list, member_list, chunk_list, date_format, default_retrials, as_conf) @@ -2322,6 +2324,8 @@ class JobList(object): Log.info("Loading JobList") try: return self._persistence.load(self._persistence_path, self._persistence_file) + except AutosubmitCritical: + raise except: Log.printlog( "Autosubmit will use a backup for recover the job_list", 6010) diff --git a/autosubmit/job/job_list_persistence.py b/autosubmit/job/job_list_persistence.py index bb884ba43..284abfc2c 100644 --- a/autosubmit/job/job_list_persistence.py +++ b/autosubmit/job/job_list_persistence.py @@ -21,7 +21,7 @@ import pickle from sys import setrecursionlimit import shutil from autosubmit.database.db_manager import DbManager -from log.log import Log +from log.log import AutosubmitCritical, Log class JobListPersistence(object): @@ -66,7 +66,14 @@ class JobListPersistencePkl(JobListPersistence): """ path = os.path.join(persistence_path, persistence_file + '.pkl') - if os.path.exists(path): + try: + open(path) + except PermissionError: + raise AutosubmitCritical(f'Permission denied to read {path}', 7012) + except FileNotFoundError: + Log.printlog(f'File {path} does not exist. ',Log.WARNING) + return list() + else: # copy the path to a tmp file randomseed to avoid corruption path_tmp = f'{path}.tmp_{os.urandom(8).hex()}' shutil.copy(path, path_tmp) @@ -82,9 +89,6 @@ class JobListPersistencePkl(JobListPersistence): graph.nodes[u]["job"]._serial_platform = None graph.nodes[u]["job"].submitter = None return graph - else: - Log.printlog('File {0} does not exist'.format(path),Log.WARNING) - return list() def save(self, persistence_path, persistence_file, job_list, graph): """ diff --git a/bin/autosubmit b/bin/autosubmit index 4280dc71e..85dee32ad 100755 --- a/bin/autosubmit +++ b/bin/autosubmit @@ -22,6 +22,7 @@ import os import sys import traceback from io import StringIO # for handling the traceback print +from contextlib import suppress scriptdir = os.path.abspath(os.path.dirname(sys.argv[0])) sys.path.append(scriptdir) @@ -31,6 +32,18 @@ from log.log import Log, AutosubmitCritical , AutosubmitError from autosubmit.autosubmit import Autosubmit +def exit_from_error(e): + with suppress(FileNotFoundError): + os.remove(os.path.join(Log.file_path, "autosubmit.lock")) + try: + if e.trace is not None or e.trace == "": + Log.debug("Trace: {0}", e.trace) + Log.critical("{1} [eCode={0}]", e.code, e.message) + except: + Log.critical("An Unknown error occurred {0}, Please report it to Autosubmit Developers through Git", str(e)) + Log.info("More info at https://autosubmit.readthedocs.io/en/master/troubleshooting/error-codes.html") + os._exit(1) + # noinspection PyProtectedMember def main(): try: @@ -41,37 +54,11 @@ def main(): os._exit(return_value) os._exit(0) except AutosubmitError as e: - if os.path.exists(os.path.join(Log.file_path, "autosubmit.lock")): - os.remove(os.path.join(Log.file_path, "autosubmit.lock")) - if e.trace is not None: - if e.trace != "": # trace might be int. - Log.error("Trace: {0}", e.trace) - Log.critical("{1} [eCode={0}]", e.code, e.message) - Log.info("More info at https://autosubmit.readthedocs.io/en/master/troubleshooting/error-codes.html") - os._exit(1) + exit_from_error(e) except AutosubmitCritical as e: - if os.path.exists(os.path.join(Log.file_path, "autosubmit.lock")): - os.remove(os.path.join(Log.file_path, "autosubmit.lock")) - str(traceback.print_exc()) - if e.trace is not None or e.trace == "": - Log.error("Trace: {0}", e.trace) - Log.critical("{1} [eCode={0}]", e.code, e.message) - - Log.info("More info at https://autosubmit.readthedocs.io/en/master/troubleshooting/error-codes.html") - os._exit(1) - except Exception as e: - if os.path.exists(os.path.join(Log.file_path, "autosubmit.lock")): - os.remove(os.path.join(Log.file_path, "autosubmit.lock")) - Log.error("Trace: {0}", str(e)) - if "temporarily unavailable" in str(e): - Log.critical( - "{0}\nAnother instance of autosubmit is running on this experiment. If this is not the case, delete autosubmit.lock".format(str(e)), 7000) - else: - exception_stream = StringIO() - traceback.print_exc(file=exception_stream) - raise AutosubmitCritical("Unhandled error: If you see this message, please report it in Autosubmit's GitLab project", 7000, str(e)) - os._exit(1) - + exit_from_error(e) + except BaseException as e: + exit_from_error(e) if __name__ == "__main__": main() -- GitLab From 8c10c36953bcab1dc270849ed5a4cee58789a63d Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 29 Feb 2024 09:23:49 +0100 Subject: [PATCH 2/5] Added permissionError --- bin/autosubmit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/autosubmit b/bin/autosubmit index 85dee32ad..4991c3ca2 100755 --- a/bin/autosubmit +++ b/bin/autosubmit @@ -33,7 +33,7 @@ from autosubmit.autosubmit import Autosubmit def exit_from_error(e): - with suppress(FileNotFoundError): + with suppress(FileNotFoundError, PermissionError): os.remove(os.path.join(Log.file_path, "autosubmit.lock")) try: if e.trace is not None or e.trace == "": -- GitLab From 9668000f5119ecee2da3ef0c0a8887d38e474d65 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 29 Feb 2024 09:24:42 +0100 Subject: [PATCH 3/5] added \n --- bin/autosubmit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/autosubmit b/bin/autosubmit index 4991c3ca2..00c02dfc8 100755 --- a/bin/autosubmit +++ b/bin/autosubmit @@ -40,7 +40,7 @@ def exit_from_error(e): Log.debug("Trace: {0}", e.trace) Log.critical("{1} [eCode={0}]", e.code, e.message) except: - Log.critical("An Unknown error occurred {0}, Please report it to Autosubmit Developers through Git", str(e)) + Log.critical("An Unknown error occurred {0}.\n Please report it to Autosubmit Developers through Git", str(e)) Log.info("More info at https://autosubmit.readthedocs.io/en/master/troubleshooting/error-codes.html") os._exit(1) -- GitLab From 18675425f3cf26b1d61165fa415cf63d437955ed Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 29 Feb 2024 09:25:32 +0100 Subject: [PATCH 4/5] added : --- bin/autosubmit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/autosubmit b/bin/autosubmit index 00c02dfc8..a520c7402 100755 --- a/bin/autosubmit +++ b/bin/autosubmit @@ -40,7 +40,7 @@ def exit_from_error(e): Log.debug("Trace: {0}", e.trace) Log.critical("{1} [eCode={0}]", e.code, e.message) except: - Log.critical("An Unknown error occurred {0}.\n Please report it to Autosubmit Developers through Git", str(e)) + Log.critical("An Unknown error occurred: {0}.\n Please report it to Autosubmit Developers through Git", str(e)) Log.info("More info at https://autosubmit.readthedocs.io/en/master/troubleshooting/error-codes.html") os._exit(1) -- GitLab From 9209e3df93bd18df6201a4864cefef141f0adcff Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 29 Feb 2024 14:36:27 +0100 Subject: [PATCH 5/5] feedback fix --- autosubmit/job/job_list_persistence.py | 2 +- bin/autosubmit | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/autosubmit/job/job_list_persistence.py b/autosubmit/job/job_list_persistence.py index 284abfc2c..951771bed 100644 --- a/autosubmit/job/job_list_persistence.py +++ b/autosubmit/job/job_list_persistence.py @@ -67,7 +67,7 @@ class JobListPersistencePkl(JobListPersistence): """ path = os.path.join(persistence_path, persistence_file + '.pkl') try: - open(path) + open(path).close() except PermissionError: raise AutosubmitCritical(f'Permission denied to read {path}', 7012) except FileNotFoundError: diff --git a/bin/autosubmit b/bin/autosubmit index a520c7402..21c056019 100755 --- a/bin/autosubmit +++ b/bin/autosubmit @@ -36,8 +36,8 @@ def exit_from_error(e): with suppress(FileNotFoundError, PermissionError): os.remove(os.path.join(Log.file_path, "autosubmit.lock")) try: - if e.trace is not None or e.trace == "": - Log.debug("Trace: {0}", e.trace) + if not e.trace: + Log.debug("Trace: {0}", str(e.trace)) Log.critical("{1} [eCode={0}]", e.code, e.message) except: Log.critical("An Unknown error occurred: {0}.\n Please report it to Autosubmit Developers through Git", str(e)) -- GitLab