diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 958303eaa209317d9b7b68a126263433831ff62d..db8ad0657e3ce4cfc990e357d4840da74086a630 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -4520,11 +4520,11 @@ class Autosubmit: message = "We have detected that there is another Autosubmit instance using the experiment\n. Stop other Autosubmit instances that are using the experiment or delete autosubmit.lock file located on tmp folder" raise AutosubmitCritical(message, 7000) except AutosubmitError as e: - if e.trace == "": + if not e.trace: e.trace = traceback.format_exc() raise AutosubmitError(e.message, e.code, e.trace) except AutosubmitCritical as e: - if e.trace == "": + if not e.trace: e.trace = traceback.format_exc() raise AutosubmitCritical(e.message, e.code, e.trace) except BaseException as e: diff --git a/autosubmit/platforms/paramiko_platform.py b/autosubmit/platforms/paramiko_platform.py index 3bb4b8c49ef4480c4eeadb78b880c75058906a10..58ddf64bcdbb16e9937f36d1216f171ba7c94e4d 100644 --- a/autosubmit/platforms/paramiko_platform.py +++ b/autosubmit/platforms/paramiko_platform.py @@ -637,7 +637,7 @@ class ParamikoPlatform(Platform): if slurm_error: raise AutosubmitError("Remote pooling failed with error:{0}\n Resetting platforms connections...".format(e_msg)) job_list_status = self.get_ssh_output() - if retries >= 0: + if retries >= 0 and job_list_status: Log.debug('Successful check job command') in_queue_jobs = [] list_queue_jobid = "" @@ -737,6 +737,18 @@ class ParamikoPlatform(Platform): return [] + def get_queue_status(self, in_queue_jobs, list_queue_jobid, as_conf): + """Get queue status for a list of jobs. + + The job statuses are normally found via a command sent to the remote platform. + + Each ``job`` in ``in_queue_jobs`` must be updated. Implementations may check + for the reason for queueing cancellation, or if the job is held, and update + the ``job`` status appropriately. + """ + raise NotImplementedError + + def get_checkjob_cmd(self, job_id): """ Returns command to check job status on remote platforms diff --git a/log/log.py b/log/log.py index 216fc23eb5c466ee804450bda5e2bf1b33f5c952..317e1846dc815cbf48deeb3d255a8a3ad543e78a 100644 --- a/log/log.py +++ b/log/log.py @@ -3,13 +3,16 @@ import os import sys from time import sleep from datetime import datetime +from typing import Union class AutosubmitError(Exception): - """Exception raised for Autosubmit critical errors . + """Exception raised for Autosubmit errors. + Attributes: - errorcode -- Classified code - message -- explanation of the error + message (str): explanation of the error + code (int): classified code + trace (str): extra information about the error """ def __init__(self, message="Unhandled Error", code=6000, trace=None): @@ -17,6 +20,19 @@ class AutosubmitError(Exception): self.message = message self.trace = trace + @property + def error_message(self): + """ + Return the error message ready to be logged, with both trace + (when present) and the message separated by a space. Or just + the message if no trace is available. + + :return: ``trace`` and ``message`` separated by a space, or just the + ``message`` if no ``trace`` is available. + :rtype: str + """ + return self.message if not self.trace else self.trace+ ' ' + self.message + def __str__(self): return " " diff --git a/test/unit/test_log.py b/test/unit/test_log.py new file mode 100644 index 0000000000000000000000000000000000000000..f283316d3a0044f7aaf9cec8556c3ff067567802 --- /dev/null +++ b/test/unit/test_log.py @@ -0,0 +1,28 @@ +from unittest import TestCase +from log.log import AutosubmitError, AutosubmitCritical + + +"""Tests for the log module.""" + +class TestLog(TestCase): + + + def test_autosubmit_error(self): + ae = AutosubmitError() + assert 'Unhandled Error' == ae.message + assert 6000 == ae.code + assert None is ae.trace + assert 'Unhandled Error' == ae.error_message + assert ' ' == str(ae) + + def test_autosubmit_error_error_message(self): + ae = AutosubmitError(trace='ERROR!') + assert 'ERROR! Unhandled Error' == ae.error_message + + def test_autosubmit_critical(self): + ac = AutosubmitCritical() + assert 'Unhandled Error' == ac.message + assert 7000 == ac.code + assert None is ac.trace + assert ' ' == str(ac) + diff --git a/test/unit/test_paramiko_platform.py b/test/unit/test_paramiko_platform.py new file mode 100644 index 0000000000000000000000000000000000000000..860db8e64fc572be689dedbb6c5ba500b942b25f --- /dev/null +++ b/test/unit/test_paramiko_platform.py @@ -0,0 +1,118 @@ +from collections import namedtuple +from unittest import TestCase + +from shutil import rmtree +from tempfile import mkdtemp +from mock import MagicMock, patch + +from autosubmit.job.job_common import Status +from autosubmit.platforms.paramiko_platform import ParamikoPlatform +from log.log import AutosubmitError + + +class TestParamikoPlatform(TestCase): + + Config = namedtuple('Config', ['LOCAL_ROOT_DIR', 'LOCAL_TMP_DIR']) + + def setUp(self): + self.local_root_dir = mkdtemp() + self.config = TestParamikoPlatform.Config( + LOCAL_ROOT_DIR=self.local_root_dir, + LOCAL_TMP_DIR='tmp' + ) + self.platform = ParamikoPlatform(expid='a000', name='local', config=self.config) + self.platform.job_status = { + 'COMPLETED': [], + 'RUNNING': [], + 'QUEUING': [], + 'FAILED': [] + } + + def tearDown(self): + rmtree(self.local_root_dir) + + def test_paramiko_platform_constructor(self): + assert self.platform.name == 'local' + assert self.platform.expid == 'a000' + assert self.config is self.platform.config + + # TODO: on the 3.15-branch branch the _header and _wrapper are + # not defined? + # assert self.platform.header is None + # assert self.platform.wrapper is None + + assert len(self.platform.job_status) == 4 + + @patch('autosubmit.platforms.paramiko_platform.Log') + @patch('autosubmit.platforms.paramiko_platform.sleep') + def test_check_Alljobs_send_command1_raises_autosubmit_error(self, mock_sleep, mock_log): + """ + Args: + mock_sleep (MagicMock): mocking because the function sleeps for 5 seconds. + """ + # Because it raises a NotImplementedError, but we want to skip it to test an error... + self.platform.get_checkAlljobs_cmd = MagicMock() + self.platform.get_checkAlljobs_cmd.side_effect = ['ls'] + # Raise the AE error here. + self.platform.send_command = MagicMock() + ae = AutosubmitError(message='Test', code=123, trace='ERR!') + self.platform.send_command.side_effect = ae + as_conf = MagicMock() + as_conf.get_copy_remote_logs.return_value = None + job = MagicMock() + job.id = 'TEST' + job.name = 'TEST' + with self.assertRaises(AutosubmitError) as cm: + # Retries is -1 so that it skips the retry code block completely, + # as we are not interested in testing that part here. + self.platform.check_Alljobs( + job_list=[job], + as_conf=as_conf, + retries=-1) + assert cm.exception.message == 'Remote pooling failed with error:ERR! Test\n Resetting platforms connections...' + assert cm.exception.code == 6000 + # #assert cm.exception.trace is None + # + # assert mock_log.warning.called + # assert mock_log.warning.call_args[0][1] == job.id + # assert mock_log.warning.call_args[0][2] == self.platform.name + # assert mock_log.warning.call_args[0][3] == Status.UNKNOWN + + @patch('autosubmit.platforms.paramiko_platform.sleep') + def test_check_Alljobs_send_command2_raises_autosubmit_error(self, mock_sleep): + """ + Args: + mock_sleep (MagicMock): mocking because the function sleeps for 5 seconds. + """ + # Because it raises a NotImplementedError, but we want to skip it to test an error... + self.platform.get_checkAlljobs_cmd = MagicMock() + self.platform.get_checkAlljobs_cmd.side_effect = ['ls'] + # Raise the AE error here. + self.platform.send_command = MagicMock() + ae = AutosubmitError(message='Test', code=123, trace='ERR!') + # Here the first time ``send_command`` is called it returns None, but + # the second time it will raise the AutosubmitError for our test case. + self.platform.send_command.side_effect = [None, ae] + # Also need to make this function return False... + self.platform._check_jobid_in_queue = MagicMock(return_value = False) + # Then it will query the job status of the job, see further down as we set it + as_conf = MagicMock() + as_conf.get_copy_remote_logs.return_value = None + job = MagicMock() + job.id = 'TEST' + job.name = 'TEST' + job.status = Status.UNKNOWN + + self.platform.get_queue_status = MagicMock(side_effect=None) + + with self.assertRaises(AutosubmitError) as cm: + # Here the retries is 1 + self.platform.check_Alljobs( + job_list=[job], + as_conf=as_conf, + retries=1) + # AS raises an exception with the message using the previous exception's + # ``error_message``, but error code 6000 and no trace. + assert ae.error_message in cm.exception.message + assert cm.exception.code == 6000 + assert cm.exception.trace is None diff --git a/test/unit/test_slurm_platform.py b/test/unit/test_slurm_platform.py new file mode 100644 index 0000000000000000000000000000000000000000..37e2b0a21c3ceceae9d8d260e14a563c45c4f75f --- /dev/null +++ b/test/unit/test_slurm_platform.py @@ -0,0 +1,59 @@ +import os + +from collections import namedtuple +from unittest import TestCase + +from shutil import rmtree +from tempfile import mkdtemp +from mock import MagicMock + +from autosubmit.platforms.slurmplatform import SlurmPlatform +from log.log import AutosubmitCritical, AutosubmitError + +# +# class TestSlurmPlatform(TestCase): +# +# Config = namedtuple('Config', ['LOCAL_ROOT_DIR', 'LOCAL_TMP_DIR', 'LOCAL_ASLOG_DIR']) +# +# def setUp(self): +# self.local_root_dir = mkdtemp() +# self.config = TestSlurmPlatform.Config( +# LOCAL_ROOT_DIR=self.local_root_dir, +# LOCAL_TMP_DIR='tmp', +# LOCAL_ASLOG_DIR='ASLOG_a000' +# ) +# # We need to create the submission archive that AS expects to find in this location: +# p = os.path.join(self.local_root_dir, 'a000/tmp/ASLOG_a000') +# os.makedirs(p) +# submit_platform_script = os.path.join(p, 'submit_local.sh') +# if not os.path.exists(submit_platform_script): +# with open(submit_platform_script, 'a'): +# os.utime(submit_platform_script, None) +# +# self.platform = SlurmPlatform(expid='a000', name='local', config=self.config) +# +# def tearDown(self): +# rmtree(self.local_root_dir) +# +# def test_slurm_platform_submit_script_raises_autosubmit_critical_with_trace(self): +# package = MagicMock() +# package.jobs.return_value = [] +# valid_packages_to_submit = [ +# package +# ] +# +# ae = AutosubmitError(message='invalid partition', code=123, trace='ERR!') +# self.platform.submit_Script = MagicMock(side_effect=ae) +# +# # AS will handle the AutosubmitError above, but then raise an AutosubmitCritical. +# # This new error won't contain all the info from the upstream error. +# with self.assertRaises(AutosubmitCritical) as cm: +# self.platform.process_batch_ready_jobs( +# valid_packages_to_submit=valid_packages_to_submit, +# failed_packages=[] +# ) +# +# # AS will handle the error and then later will raise another error message. +# # But the AutosubmitError object we created will have been correctly used +# # without raising any exceptions (such as AttributeError). +# assert cm.exception.message != ae.message