From 827465c198a3d13cc1640f1f01967778c335c47b Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 7 Mar 2024 11:32:14 +0100 Subject: [PATCH 01/12] Fix error autosubmit version Fix trace when exception as e is not an AutosubmitCritical or AutosubmitError --- autosubmit/autosubmit.py | 10 +++------- bin/autosubmit | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index bb243e0d5..e90f3bb66 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -649,13 +649,9 @@ class Autosubmit: if args.command is None: parser.print_help() parser.exit() - - except Exception as e: - if type(e) is SystemExit: # todo check - # Version keyword force an exception in parse arg due and os_exit(0) but the program is successfully finished - if "0" in str(e): - print(Autosubmit.autosubmit_version) - return 0 + except SystemExit as e: + return 0 + except BaseException as e: raise AutosubmitCritical( "Incorrect arguments for this command", 7011) diff --git a/bin/autosubmit b/bin/autosubmit index 21c056019..45354e15c 100755 --- a/bin/autosubmit +++ b/bin/autosubmit @@ -36,7 +36,7 @@ def exit_from_error(e): with suppress(FileNotFoundError, PermissionError): os.remove(os.path.join(Log.file_path, "autosubmit.lock")) try: - if not e.trace: + if hasattr(e,"trace") and e.trace: Log.debug("Trace: {0}", str(e.trace)) Log.critical("{1} [eCode={0}]", e.code, e.message) except: -- GitLab From a6b264355104980513a6115bdd2e6078f548bc9a Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 7 Mar 2024 11:33:30 +0100 Subject: [PATCH 02/12] Fix error autosubmit version Fix trace when exception as e is not an AutosubmitCritical or AutosubmitError update version --- bin/autosubmit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/autosubmit b/bin/autosubmit index 45354e15c..5715dc687 100755 --- a/bin/autosubmit +++ b/bin/autosubmit @@ -39,7 +39,7 @@ def exit_from_error(e): if hasattr(e,"trace") and e.trace: Log.debug("Trace: {0}", str(e.trace)) Log.critical("{1} [eCode={0}]", e.code, e.message) - except: + except BaseException as e: Log.critical("An Unknown error occurred: {0}.\n Please report it to Autosubmit Developers through Git", str(e)) Log.info("More info at https://autosubmit.readthedocs.io/en/master/troubleshooting/error-codes.html") os._exit(1) -- GitLab From 53cbca36c1e4d5bffb9167ac9711cbf9088ed33b Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 7 Mar 2024 11:33:52 +0100 Subject: [PATCH 03/12] Fix error autosubmit version Fix trace when exception as e is not an AutosubmitCritical or AutosubmitError update version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index ee74734aa..4d0dcda01 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -4.1.0 +4.1.2 -- GitLab From 413968dd91c89a0839c8dc4e8dea21c8bef9955d Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 7 Mar 2024 11:34:08 +0100 Subject: [PATCH 04/12] update version ( empty line ) --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 4d0dcda01..cd9b8f559 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -4.1.2 +4.1.2 \ No newline at end of file -- GitLab From b8c6fec70d6505114f44a98c2e4d50e4e4029ba9 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 8 Mar 2024 10:11:54 +0100 Subject: [PATCH 05/12] Address feedback --- autosubmit/autosubmit.py | 16 +++++++++------- bin/autosubmit | 28 +++++++++++++++++++++++----- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index e90f3bb66..5c46c9a76 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -167,8 +167,8 @@ class Autosubmit: BasicConfig.read() parser = MyParser( description='Main executable for autosubmit. ') - parser.add_argument('-v', '--version', action='version', - version=Autosubmit.autosubmit_version) + parser.add_argument('-v', '--version', dest='version', action='store_true') + parser.add_argument('-lf', '--logfile', choices=('NO_LOG', 'INFO', 'WARNING', 'DEBUG'), default='DEBUG', type=str, help="sets file's log level.") @@ -644,11 +644,13 @@ class Autosubmit: help='Read job files generated by the inspect subcommand.') subparser.add_argument('ID', metavar='ID', help='An ID of a Workflow (eg a000) or a Job (eg a000_20220401_fc0_1_1_APPLICATION).') - args = parser.parse_args() - - if args.command is None: + args, unknown = parser.parse_known_args() + if args.version: + print(Autosubmit.autosubmit_version) + return 0 + if unknown or args.command is None: parser.print_help() - parser.exit() + return 0 except SystemExit as e: return 0 except BaseException as e: @@ -4791,7 +4793,7 @@ class Autosubmit: e.trace = traceback.format_exc() raise AutosubmitCritical(e.message, e.code, e.trace) except BaseException as e: - raise AutosubmitCritical(str(e), 7070) + raise finally: if profile: profiler.stop() diff --git a/bin/autosubmit b/bin/autosubmit index 5715dc687..dc87f07a6 100755 --- a/bin/autosubmit +++ b/bin/autosubmit @@ -30,20 +30,38 @@ sys.path.append(os.path.normpath(os.path.join(scriptdir, os.pardir))) # noinspection PyUnresolvedReferences from log.log import Log, AutosubmitCritical , AutosubmitError from autosubmit.autosubmit import Autosubmit +from typing import Union -def exit_from_error(e): +def exit_from_error(e: BaseException): + Log.debug(str(traceback.format_exc())) with suppress(FileNotFoundError, PermissionError): os.remove(os.path.join(Log.file_path, "autosubmit.lock")) - try: - if hasattr(e,"trace") and e.trace: + if isinstance(e, (AutosubmitCritical, AutosubmitError)): + e: Union[AutosubmitError, AutosubmitCritical] = e + if e.trace: Log.debug("Trace: {0}", str(e.trace)) Log.critical("{1} [eCode={0}]", e.code, e.message) - except BaseException as e: - Log.critical("An Unknown error occurred: {0}.\n Please report it to Autosubmit Developers through Git", str(e)) + else: + msg = "An Unknown error occurred: {0}.\n Please report it to Autosubmit Developers through Git" + args = [str(e)] + Log.critical(msg.format(*args)) Log.info("More info at https://autosubmit.readthedocs.io/en/master/troubleshooting/error-codes.html") os._exit(1) + +# def exit_from_error(e: BaseException): +# with suppress(FileNotFoundError, PermissionError): +# os.remove(os.path.join(Log.file_path, "autosubmit.lock")) +# try: +# if hasattr(e,"trace") and e.trace: +# Log.debug("Trace: {0}", str(e.trace)) +# Log.critical("{1} [eCode={0}]", e.code, e.message) +# except BaseException as e: +# Log.critical("An Unknown error occurred: {0}.\n Please report it to Autosubmit Developers through Git", str(e)) +# Log.info("More info at https://autosubmit.readthedocs.io/en/master/troubleshooting/error-codes.html") +# os._exit(1) + # noinspection PyProtectedMember def main(): try: -- GitLab From e3d36c353e5c73502d552d33f9fc977c84dd8ec7 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 8 Mar 2024 12:16:32 +0100 Subject: [PATCH 06/12] Deleted systemexit --- autosubmit/autosubmit.py | 2 -- bin/autosubmit | 19 ++++--------------- 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 5c46c9a76..4fa29f0d1 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -651,8 +651,6 @@ class Autosubmit: if unknown or args.command is None: parser.print_help() return 0 - except SystemExit as e: - return 0 except BaseException as e: raise AutosubmitCritical( "Incorrect arguments for this command", 7011) diff --git a/bin/autosubmit b/bin/autosubmit index dc87f07a6..d87c7adce 100755 --- a/bin/autosubmit +++ b/bin/autosubmit @@ -21,7 +21,6 @@ import os import sys import traceback -from io import StringIO # for handling the traceback print from contextlib import suppress scriptdir = os.path.abspath(os.path.dirname(sys.argv[0])) @@ -34,7 +33,10 @@ from typing import Union def exit_from_error(e: BaseException): - Log.debug(str(traceback.format_exc())) + try: + Log.debug(traceback.format_exc()) + except: + Log.debug("No traceback available") with suppress(FileNotFoundError, PermissionError): os.remove(os.path.join(Log.file_path, "autosubmit.lock")) if isinstance(e, (AutosubmitCritical, AutosubmitError)): @@ -49,19 +51,6 @@ def exit_from_error(e: BaseException): Log.info("More info at https://autosubmit.readthedocs.io/en/master/troubleshooting/error-codes.html") os._exit(1) - -# def exit_from_error(e: BaseException): -# with suppress(FileNotFoundError, PermissionError): -# os.remove(os.path.join(Log.file_path, "autosubmit.lock")) -# try: -# if hasattr(e,"trace") and e.trace: -# Log.debug("Trace: {0}", str(e.trace)) -# Log.critical("{1} [eCode={0}]", e.code, e.message) -# except BaseException as e: -# Log.critical("An Unknown error occurred: {0}.\n Please report it to Autosubmit Developers through Git", str(e)) -# Log.info("More info at https://autosubmit.readthedocs.io/en/master/troubleshooting/error-codes.html") -# os._exit(1) - # noinspection PyProtectedMember def main(): try: -- GitLab From fc97d0370bcda1ea9b643edc3a9deb1e08c9bd06 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 8 Mar 2024 12:22:55 +0100 Subject: [PATCH 07/12] added unit test --- autosubmit/autosubmit.py | 4 ++-- test/unit/test_version.py | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 test/unit/test_version.py diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 4fa29f0d1..38eed36a8 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -647,10 +647,10 @@ class Autosubmit: args, unknown = parser.parse_known_args() if args.version: print(Autosubmit.autosubmit_version) - return 0 + return 1 if unknown or args.command is None: parser.print_help() - return 0 + return 1 except BaseException as e: raise AutosubmitCritical( "Incorrect arguments for this command", 7011) diff --git a/test/unit/test_version.py b/test/unit/test_version.py new file mode 100644 index 000000000..767f38080 --- /dev/null +++ b/test/unit/test_version.py @@ -0,0 +1,20 @@ +import subprocess +from pathlib import Path +from unittest import TestCase + +import sys + +from autosubmit.autosubmit import Autosubmit + + +class TestAutosubmit(TestCase): + + def testAutosubmitVersion(self): + bin_path = Path(__file__, '../../../bin/autosubmit').resolve() + out = subprocess.getoutput(' '.join([sys.executable, str(bin_path), '-v'])) + self.assertEquals(Autosubmit.autosubmit_version, out.strip()) + + def testAutosubmitVersionBroken(self): + bin_path = Path(__file__, '../../../bin/autosubmit').resolve() + exit_code, _ = subprocess.getstatusoutput(' '.join([sys.executable, str(bin_path), '-abcdefg'])) + self.assertEquals(1, exit_code) -- GitLab From fc8ab8386d6c594d2ec39084f1e49995c342a1c4 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 7 Mar 2024 13:41:18 +0100 Subject: [PATCH 08/12] Fixes "attempt to cancel an het job" failure message --- autosubmit/platforms/slurmplatform.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/autosubmit/platforms/slurmplatform.py b/autosubmit/platforms/slurmplatform.py index b51920d7d..e741239db 100644 --- a/autosubmit/platforms/slurmplatform.py +++ b/autosubmit/platforms/slurmplatform.py @@ -176,7 +176,11 @@ class SlurmPlatform(ParamikoPlatform): job_name = package.name if hasattr(package, "name") else package.jobs[0].name jobid = self.get_jobid_by_jobname(job_name) if len(jobid) > 1: # Cancel each job that is not the associated - for id_ in [ jobid for jobid in jobid if jobid != package.jobs[0].id ]: + ids_to_check = [package.jobs[0].id] + if package.jobs[0].het: + for i in range(1,package.jobs[0].het.get("HETSIZE",1)): + ids_to_check.append(str(int(ids_to_check[0]) + i)) + for id_ in [ jobid for jobid in jobid if jobid not in ids_to_check]: self.send_command(self.cancel_job(id_)) # This can be faster if we cancel all jobs at once but there is no cancel_all_jobs call right now so todo in future Log.debug(f'Job {id_} with the assigned name: {job_name} has been cancelled') Log.debug(f'Job {package.jobs[0].id} with the assigned name: {job_name} has been submitted') -- GitLab From fb0de45fe45d8d868a19cffd6f87b1f3cd52b55f Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 7 Mar 2024 15:40:48 +0100 Subject: [PATCH 09/12] Changes "backup error message" --- autosubmit/autosubmit.py | 4 +-- autosubmit/job/job_list.py | 44 ++++++++++++++------------ autosubmit/job/job_list_persistence.py | 9 ++++-- 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 38eed36a8..980ed4745 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -2472,12 +2472,10 @@ class Autosubmit: except AutosubmitCritical as e: raise except BaseException as e: - raise AutosubmitCritical("Error while checking the configuration files or loading the job_list", 7040, - str(e)) + raise finally: if profile: profiler.stop() - try: jobs = [] if not isinstance(job_list, type([])): diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 9f5fbfcc4..cbd7e21af 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -204,7 +204,7 @@ class JobList(object): chunk_list = list(range(chunk_ini, num_chunks + 1)) self._chunk_list = chunk_list try: - self.graph = self.load() + self.graph = self.load(create) if type(self.graph) is not DiGraph: self.graph = nx.DiGraph() except AutosubmitCritical: @@ -213,8 +213,6 @@ class JobList(object): self.graph = nx.DiGraph() self._dic_jobs = DicJobs(date_list, member_list, chunk_list, date_format, default_retrials, as_conf) self._dic_jobs.graph = self.graph - if show_log: - Log.info("Creating jobs...") if len(self.graph.nodes) > 0: if show_log: Log.info("Load finished") @@ -246,6 +244,8 @@ class JobList(object): os.remove(os.path.join(self._persistence_path, self._persistence_file + "_backup.pkl")) new = True # This generates the job object and also finds if dic_jobs has modified from previous iteration in order to expand the workflow + if show_log: + Log.info("Creating jobs...") self._create_jobs(self._dic_jobs, 0, default_job_type) # not needed anymore all data is inside their correspondent sections in dic_jobs # This dic_job is key to the dependencies management as they're ordered by date[member[chunk]] @@ -2312,32 +2312,36 @@ class JobList(object): "Autosubmit will use a backup for recover the job_list", 6010) return list() - def load(self): + def load(self, create=False, backup=""): """ Recreates a stored job list from the persistence :return: loaded job list object :rtype: JobList """ - Log.info("Loading JobList") + if backup == "": + Log.info("Loading JobList") try: - return self._persistence.load(self._persistence_path, self._persistence_file) + return self._persistence.load(self._persistence_path, self._persistence_file + backup) except AutosubmitCritical: raise - except: - Log.printlog( - "Autosubmit will use a backup for recover the job_list", 6010) - return self.backup_load() - - def backup_load(self): - """ - Recreates a stored job list from the persistence - - :return: loaded job list object - :rtype: JobList - """ - Log.info("Loading backup JobList") - return self._persistence.load(self._persistence_path, self._persistence_file + "_backup") + except ValueError as e: + if not create: + raise AutosubmitCritical(f'JobList could not be loaded due pkl being saved with a different version of Autosubmit or Python version. {e}') + else: + Log.warning(f'Job list will be created from scratch due pkl being saved with a different version of Autosubmit or Python version. {e}') + except BaseException as e: + if backup == "": + Log.printlog( + "Autosubmit will use a backup for recover the job_list", 6010) + return self.load(create, "_backup") + else: + if not create: + raise AutosubmitCritical( + f'JobList backup could not be loaded due: {e}') + else: + Log.warning( + f'Joblist backup will be created from scratch due: {e}') def save(self): """ diff --git a/autosubmit/job/job_list_persistence.py b/autosubmit/job/job_list_persistence.py index 948f21c01..1a271a1fc 100644 --- a/autosubmit/job/job_list_persistence.py +++ b/autosubmit/job/job_list_persistence.py @@ -78,9 +78,12 @@ class JobListPersistencePkl(JobListPersistence): # copy the path to a tmp file randomseed to avoid corruption path_tmp = f'{path}.tmp_{os.urandom(8).hex()}' shutil.copy(path, path_tmp) - with open(path_tmp, 'rb') as fd: - graph = pickle.load(fd) - os.remove(path_tmp) + try: + with open(path_tmp, 'rb') as fd: + graph = pickle.load(fd) + except: + os.remove(path_tmp) + raise for u in ( node for node in graph ): # Set after the dependencies are set graph.nodes[u]["job"].children = set() -- GitLab From 76da0d0b7795ba1b2d295fdfe444645b6be71f2a Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 7 Mar 2024 16:31:50 +0100 Subject: [PATCH 10/12] Changed the location of pkl.tmp as pkl is not Writeable for the users --- autosubmit/job/job_list.py | 2 +- autosubmit/job/job_list_persistence.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index cbd7e21af..ec71b0ab5 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -2341,7 +2341,7 @@ class JobList(object): f'JobList backup could not be loaded due: {e}') else: Log.warning( - f'Joblist backup will be created from scratch due: {e}') + f'Joblist will be created from scratch due: {e}') def save(self): """ diff --git a/autosubmit/job/job_list_persistence.py b/autosubmit/job/job_list_persistence.py index 1a271a1fc..e232102c0 100644 --- a/autosubmit/job/job_list_persistence.py +++ b/autosubmit/job/job_list_persistence.py @@ -67,6 +67,8 @@ class JobListPersistencePkl(JobListPersistence): """ path = os.path.join(persistence_path, persistence_file + '.pkl') + path_tmp = os.path.join(persistence_path[:-3]+"tmp", persistence_file + f'.pkl.tmp_{os.urandom(8).hex()}') + try: open(path).close() except PermissionError: @@ -76,11 +78,11 @@ class JobListPersistencePkl(JobListPersistence): return list() else: # copy the path to a tmp file randomseed to avoid corruption - path_tmp = f'{path}.tmp_{os.urandom(8).hex()}' - shutil.copy(path, path_tmp) try: + shutil.copy(path, path_tmp) with open(path_tmp, 'rb') as fd: graph = pickle.load(fd) + os.remove(path_tmp) except: os.remove(path_tmp) raise -- GitLab From 47cefe7be8840be5e43627e657e6c7a5bf616aa3 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 8 Mar 2024 11:33:19 +0100 Subject: [PATCH 11/12] added unit test changed code based on feedback --- autosubmit/job/job_list.py | 30 ++++++------ autosubmit/job/job_list_persistence.py | 11 ++--- test/unit/test_job_list.py | 68 ++++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 20 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index ec71b0ab5..2bf7933f6 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -2312,36 +2312,38 @@ class JobList(object): "Autosubmit will use a backup for recover the job_list", 6010) return list() - def load(self, create=False, backup=""): + def load(self, create=False, backup=False): """ Recreates a stored job list from the persistence :return: loaded job list object :rtype: JobList """ - if backup == "": - Log.info("Loading JobList") try: - return self._persistence.load(self._persistence_path, self._persistence_file + backup) - except AutosubmitCritical: - raise + if not backup: + Log.info("Loading JobList") + return self._persistence.load(self._persistence_path, self._persistence_file) + else: + return self._persistence.load(self._persistence_path, self._persistence_file + "_backup") except ValueError as e: if not create: raise AutosubmitCritical(f'JobList could not be loaded due pkl being saved with a different version of Autosubmit or Python version. {e}') else: Log.warning(f'Job list will be created from scratch due pkl being saved with a different version of Autosubmit or Python version. {e}') + except PermissionError as e: + if not create: + raise AutosubmitCritical(f'JobList could not be loaded due to permission error. {e}') + else: + Log.warning(f'Job list will be created from scratch due to permission error. {e}') except BaseException as e: - if backup == "": - Log.printlog( - "Autosubmit will use a backup for recover the job_list", 6010) - return self.load(create, "_backup") + if not backup: + Log.debug("Autosubmit will use a backup to recover the job_list") + return self.load(create, True) else: if not create: - raise AutosubmitCritical( - f'JobList backup could not be loaded due: {e}') + raise AutosubmitCritical(f"JobList could not be loaded due: {e}\nAutosubmit won't do anything") else: - Log.warning( - f'Joblist will be created from scratch due: {e}') + Log.warning(f'Joblist will be created from scratch due: {e}') def save(self): """ diff --git a/autosubmit/job/job_list_persistence.py b/autosubmit/job/job_list_persistence.py index e232102c0..791de7864 100644 --- a/autosubmit/job/job_list_persistence.py +++ b/autosubmit/job/job_list_persistence.py @@ -72,20 +72,19 @@ class JobListPersistencePkl(JobListPersistence): try: open(path).close() except PermissionError: - raise AutosubmitCritical(f'Permission denied to read {path}', 7012) + Log.warning(f'Permission denied to read {path}') + raise except FileNotFoundError: - Log.printlog(f'File {path} does not exist. ',Log.WARNING) - return list() + Log.warning(f'File {path} does not exist. ') + raise else: # copy the path to a tmp file randomseed to avoid corruption try: shutil.copy(path, path_tmp) with open(path_tmp, 'rb') as fd: graph = pickle.load(fd) + finally: os.remove(path_tmp) - except: - os.remove(path_tmp) - raise for u in ( node for node in graph ): # Set after the dependencies are set graph.nodes[u]["job"].children = set() diff --git a/test/unit/test_job_list.py b/test/unit/test_job_list.py index d02322503..0dc87554c 100644 --- a/test/unit/test_job_list.py +++ b/test/unit/test_job_list.py @@ -1,3 +1,4 @@ +import os from unittest import TestCase from copy import copy import networkx @@ -15,6 +16,7 @@ from autosubmit.job.job_common import Type from autosubmit.job.job_list import JobList from autosubmit.job.job_list_persistence import JobListPersistencePkl from autosubmitconfigparser.config.yamlparser import YAMLParserFactory +from log.log import AutosubmitCritical class TestJobList(TestCase): @@ -66,6 +68,72 @@ class TestJobList(TestCase): def tearDown(self) -> None: shutil.rmtree(self.temp_directory) + def test_load(self): + as_conf = Mock() + as_conf.experiment_data = dict() + parser_mock = Mock() + parser_mock.read = Mock() + factory = YAMLParserFactory() + factory.create_parser = Mock(return_value=parser_mock) + date_list = ['fake-date1', 'fake-date2'] + member_list = ['fake-member1', 'fake-member2'] + num_chunks = 999 + parameters = {'fake-key': 'fake-value', + 'fake-key2': 'fake-value2'} + with tempfile.TemporaryDirectory() as temp_dir: + job_list = self.new_job_list(factory, temp_dir) + FakeBasicConfig.LOCAL_ROOT_DIR = str(temp_dir) + Path(temp_dir, self.experiment_id).mkdir() + for path in [f'{self.experiment_id}/tmp', f'{self.experiment_id}/tmp/ASLOGS', + f'{self.experiment_id}/tmp/ASLOGS_{self.experiment_id}', f'{self.experiment_id}/proj', + f'{self.experiment_id}/conf', f'{self.experiment_id}/pkl']: + Path(temp_dir, path).mkdir() + job_list.changes = Mock(return_value=['random_section', 'random_section']) + as_conf.detailed_deep_diff = Mock(return_value={}) + # as_conf.get_member_list = Mock(return_value=member_list) + # act + job_list.generate( + as_conf=as_conf, + date_list=date_list, + member_list=member_list, + num_chunks=num_chunks, + chunk_ini=1, + parameters=parameters, + date_format='H', + default_retrials=9999, + default_job_type=Type.BASH, + wrapper_jobs={}, + new=True, + create=True, + ) + job_list.save() + # Test load + job_list_to_load = self.new_job_list(factory, temp_dir) + # chmod + job_list_to_load.load(False) + self.assertEqual(job_list_to_load._job_list, job_list._job_list) + job_list_to_load.load(True) + self.assertEqual(job_list_to_load._job_list, job_list._job_list) + os.chmod(f'{temp_dir}/{self.experiment_id}/pkl/job_list_random-id.pkl', 0o000) + with self.assertRaises(AutosubmitCritical): + job_list_to_load.load(False) + job_list_to_load.load(True) + self.assertEqual(job_list_to_load._job_list, job_list._job_list) + os.chmod(f'{temp_dir}/{self.experiment_id}/pkl/job_list_random-id.pkl', 0o777) + shutil.copy(f'{temp_dir}/{self.experiment_id}/pkl/job_list_random-id.pkl',f'{temp_dir}/{self.experiment_id}/pkl/job_list_random-id_backup.pkl') + os.remove(f'{temp_dir}/{self.experiment_id}/pkl/job_list_random-id.pkl') + job_list_to_load.load(False) + self.assertEqual(job_list_to_load._job_list, job_list._job_list) + job_list_to_load.load(True) + self.assertEqual(job_list_to_load._job_list, job_list._job_list) + + + + + + + + def test_get_job_list_returns_the_right_list(self): job_list = self.job_list.get_job_list() self.assertEqual(self.job_list._job_list, job_list) -- GitLab From d2630cb646368a15348ce94de194c7ba7b5ad6d7 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 8 Mar 2024 13:18:20 +0100 Subject: [PATCH 12/12] Patched --- autosubmit/autosubmit.py | 2 +- test/unit/test_version.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 980ed4745..f40c65b88 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -647,7 +647,7 @@ class Autosubmit: args, unknown = parser.parse_known_args() if args.version: print(Autosubmit.autosubmit_version) - return 1 + return 0 if unknown or args.command is None: parser.print_help() return 1 diff --git a/test/unit/test_version.py b/test/unit/test_version.py index 767f38080..856979767 100644 --- a/test/unit/test_version.py +++ b/test/unit/test_version.py @@ -11,10 +11,11 @@ class TestAutosubmit(TestCase): def testAutosubmitVersion(self): bin_path = Path(__file__, '../../../bin/autosubmit').resolve() - out = subprocess.getoutput(' '.join([sys.executable, str(bin_path), '-v'])) - self.assertEquals(Autosubmit.autosubmit_version, out.strip()) + exit_code, out = subprocess.getstatusoutput(' '.join([sys.executable, str(bin_path), '-v'])) + self.assertEqual(0, exit_code) + self.assertEqual(Autosubmit.autosubmit_version, out.strip()) def testAutosubmitVersionBroken(self): bin_path = Path(__file__, '../../../bin/autosubmit').resolve() exit_code, _ = subprocess.getstatusoutput(' '.join([sys.executable, str(bin_path), '-abcdefg'])) - self.assertEquals(1, exit_code) + self.assertEqual(1, exit_code) -- GitLab