From 98b9d013d9a7e8108e1aa1a2a95e5780db626df0 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Tue, 21 Feb 2023 18:40:24 +0100 Subject: [PATCH 1/6] [cat-log] Add logic to view workflow file --- autosubmit/autosubmit.py | 82 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index dd18b69ed..64bcdad8e 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -614,6 +614,16 @@ class Autosubmit: # Changelog subparsers.add_parser('changelog', description='show changelog') + + # Cat-log + subparser = subparsers.add_parser( + 'cat-log', description='View workflow and job logs.') + subparser.add_argument('-f', '--file', default=None, action='store', metavar='LOG', + help='Workflow or Job log. Options are o(output), j(job), e(error), s(status). Default o(output).') + subparser.add_argument('-m', '--mode', default=None, action='store', metavar='MODE', + help='Mode. Options are c(cat), t(tail). Default is c(cat).') + subparser.add_argument('expid', metavar='ID', help='A Workflow (eg a000) or Job ID (eg a000_20220401_fc0_1_1_APPLICATION)') + args = parser.parse_args() if args.command is None: @@ -719,6 +729,8 @@ class Autosubmit: return Autosubmit.pkl_fix(args.expid) elif args.command == 'updatedescrip': return Autosubmit.update_description(args.expid, args.description) + elif args.command == 'cat-log': + return Autosubmit.cat_log(args.expid, args.file, args.mode) @staticmethod def _init_logs(args, console_level='INFO', log_level='DEBUG', expid='None'): @@ -5760,3 +5772,73 @@ class Autosubmit: job.platform.get_logs_files(expid, job.remote_logs) return job_list + + @staticmethod + def cat_log(expid, file, mode): + from subprocess import Popen, DEVNULL + from contextlib import suppress + + def view_file(log_file, mode): + if mode == 'c': + cmd = ['cat', str(log_file)] + Popen( + cmd, + stdin=DEVNULL, + stdout=None + ) + return 0 + elif mode == 't': + cmd = [ + 'tail', + '--lines=+1', + '--retry', + '--follow=name', + workflow_log_file + ] + proc = Popen(cmd, stdin=DEVNULL) + with suppress(KeyboardInterrupt): + return proc.wait() == 0 + + MODES = { + 'c': 'cat', + 't': 'tail' + } + FILES = { + 'o': 'output', + 'j': 'job', + 'e': 'error', + 's': 'status' + } + if file is None: + file = 'o' + if file not in FILES.keys(): + raise AutosubmitCritical(f'Invalid cat-log file {file}. Expected one of {[f for f in FILES.keys()]}', 7011) + if mode is None: + mode = 'c' + if mode not in MODES.keys(): + raise AutosubmitCritical(f'Invalid cat-log mode {mode}. Expected one of {[m for m in MODES.keys()]}', 7011) + + exp_path = Path(BasicConfig.LOCAL_ROOT_DIR, expid) + tmp_path = exp_path / BasicConfig.LOCAL_TMP_DIR + aslogs_path = tmp_path / BasicConfig.LOCAL_ASLOG_DIR + + is_workflow = '_' not in expid + + if is_workflow: + if file not in ['o', 'e']: + raise AutosubmitCritical(f'Invalid arguments for cat-log: workflow logs only support o(output) and ' + f'e(error). Requested: {mode}', 7011) + + search_pattern = '*_run_err.log' if file == 'e' else '*_run.log' + workflow_log_files = sorted(aslogs_path.glob(search_pattern)) + if not workflow_log_files: + Log.info('No logs found.') + return True + + workflow_log_file = workflow_log_files[-1] + if not workflow_log_file.is_file(): + raise AutosubmitCritical(f'The workflow log file found is not a file: {workflow_log_file}', 7011) + + return view_file(workflow_log_file, mode) == 0 + + return False -- GitLab From 43063c685b45a355f0f71d7f34fdba43a16896c3 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Wed, 22 Feb 2023 15:03:35 +0100 Subject: [PATCH 2/6] [cat-log] Add logic to view job file --- autosubmit/autosubmit.py | 74 +++++++++++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 16 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 64bcdad8e..98715bcaa 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -97,6 +97,8 @@ import autosubmit.history.utils as HUtils import autosubmit.helpers.autosubmit_helper as AutosubmitHelper import autosubmit.statistics.utils as StatisticsUtils +from contextlib import suppress + """ Main module for autosubmit. Only contains an interface class to all functionality implemented on autosubmit """ @@ -622,7 +624,7 @@ class Autosubmit: help='Workflow or Job log. Options are o(output), j(job), e(error), s(status). Default o(output).') subparser.add_argument('-m', '--mode', default=None, action='store', metavar='MODE', help='Mode. Options are c(cat), t(tail). Default is c(cat).') - subparser.add_argument('expid', metavar='ID', help='A Workflow (eg a000) or Job ID (eg a000_20220401_fc0_1_1_APPLICATION)') + subparser.add_argument('ID', metavar='ID', help='An ID of a Workflow (eg a000) or a Job (eg a000_20220401_fc0_1_1_APPLICATION)') args = parser.parse_args() @@ -730,7 +732,7 @@ class Autosubmit: elif args.command == 'updatedescrip': return Autosubmit.update_description(args.expid, args.description) elif args.command == 'cat-log': - return Autosubmit.cat_log(args.expid, args.file, args.mode) + return Autosubmit.cat_log(args.ID, args.file, args.mode) @staticmethod def _init_logs(args, console_level='INFO', log_level='DEBUG', expid='None'): @@ -752,7 +754,8 @@ class Autosubmit: expid_less = ["expid", "describe", "testcase", "install", "-v", - "readme", "changelog", "configure", "unarchive"] + "readme", "changelog", "configure", "unarchive", + "cat-log"] global_log_command = ["delete", "archive", "upgrade"] if "offer" in args: if args.offer: @@ -5774,16 +5777,29 @@ class Autosubmit: return job_list @staticmethod - def cat_log(expid, file, mode): - from subprocess import Popen, DEVNULL - from contextlib import suppress + def cat_log(ID, file, mode): + """The cat-log command allows users to view Autosubmit logs using the command-line. + + It is possible to use ``autosubmit cat-log`` for Workflow and for Job logs. It decides + whether to show Workflow or Job logs based on the ``ID`` given. Shorter ID's, such as + ``a000` are considered Workflow ID's, so it will display logs for that workflow. For + longer ID's, such as ``a000_20220401_fc0_1_GSV``, the command will display logs for + that specific job. + + Users can choose the log file using the ``FILE`` parameter, to display an error or + output log file, for instance. + + Finally, the ``MODE`` parameter allows users to choose whether to display the complete + file contents (similar to the ``cat`` command) or to start tailing its output (akin to + ``tail -f``). + """ def view_file(log_file, mode): if mode == 'c': cmd = ['cat', str(log_file)] - Popen( + subprocess.Popen( cmd, - stdin=DEVNULL, + stdin=subprocess.DEVNULL, stdout=None ) return 0 @@ -5795,7 +5811,7 @@ class Autosubmit: '--follow=name', workflow_log_file ] - proc = Popen(cmd, stdin=DEVNULL) + proc = subprocess.Popen(cmd, stdin=subprocess.DEVNULL) with suppress(KeyboardInterrupt): return proc.wait() == 0 @@ -5818,19 +5834,27 @@ class Autosubmit: if mode not in MODES.keys(): raise AutosubmitCritical(f'Invalid cat-log mode {mode}. Expected one of {[m for m in MODES.keys()]}', 7011) + is_workflow = '_' not in ID + + expid = ID if is_workflow else ID[:4] + exp_path = Path(BasicConfig.LOCAL_ROOT_DIR, expid) tmp_path = exp_path / BasicConfig.LOCAL_TMP_DIR aslogs_path = tmp_path / BasicConfig.LOCAL_ASLOG_DIR - is_workflow = '_' not in expid - if is_workflow: - if file not in ['o', 'e']: - raise AutosubmitCritical(f'Invalid arguments for cat-log: workflow logs only support o(output) and ' - f'e(error). Requested: {mode}', 7011) + if file not in ['o', 'e', 's']: + raise AutosubmitCritical(f'Invalid arguments for cat-log: workflow logs only support o(output), ' + f'e(error), and s(status). Requested: {mode}', 7011) + + if file in ['e', 'o']: + search_pattern = '*_run_err.log' if file == 'e' else '*_run.log' + workflow_log_files = sorted(aslogs_path.glob(search_pattern)) + else: + search_pattern = f'{expid}_*.txt' + stats_files_path = exp_path / 'status' + workflow_log_files = sorted(stats_files_path.glob(search_pattern)) - search_pattern = '*_run_err.log' if file == 'e' else '*_run.log' - workflow_log_files = sorted(aslogs_path.glob(search_pattern)) if not workflow_log_files: Log.info('No logs found.') return True @@ -5840,5 +5864,23 @@ class Autosubmit: raise AutosubmitCritical(f'The workflow log file found is not a file: {workflow_log_file}', 7011) return view_file(workflow_log_file, mode) == 0 + else: + if file == 'j': + workflow_log_file = tmp_path / f'{ID}.cmd' + elif file == 's': + workflow_log_file = tmp_path / f'{ID}_TOTAL_STATS' + else: + search_pattern = f'{ID}.*.{"err" if file == "e" else "out"}' + exp_logs_path = tmp_path / f'LOG_{ID[:4]}' + workflow_log_files = sorted(exp_logs_path.glob(search_pattern)) + if not workflow_log_files: + Log.info('No logs found.') + return True + workflow_log_file = workflow_log_files[-1] + + if not workflow_log_file.is_file(): + raise AutosubmitCritical(f'The job log file {file} found is not a file: {workflow_log_file}', 7011) + + return view_file(workflow_log_file, mode) == 0 return False -- GitLab From da0a85a4622b7798c0c326d27be9b50064564961 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Thu, 23 Feb 2023 16:09:19 +0100 Subject: [PATCH 3/6] [cat-log] Adding docs for cat-log --- autosubmit/autosubmit.py | 7 +++---- docs/source/qstartguide/index.rst | 32 +++++++++++++++++++++++++++---- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 98715bcaa..4237fc9d5 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -620,11 +620,11 @@ class Autosubmit: # Cat-log subparser = subparsers.add_parser( 'cat-log', description='View workflow and job logs.') - subparser.add_argument('-f', '--file', default=None, action='store', metavar='LOG', - help='Workflow or Job log. Options are o(output), j(job), e(error), s(status). Default o(output).') + subparser.add_argument('-f', '--file', default=None, action='store', metavar='FILE', + help='Workflow or Job log file. Options are o(output), j(job), e(error), s(status). Default o(output).') subparser.add_argument('-m', '--mode', default=None, action='store', metavar='MODE', help='Mode. Options are c(cat), t(tail). Default is c(cat).') - subparser.add_argument('ID', metavar='ID', help='An ID of a Workflow (eg a000) or a Job (eg a000_20220401_fc0_1_1_APPLICATION)') + subparser.add_argument('ID', metavar='ID', help='An ID of a Workflow (eg a000) or a Job (eg a000_20220401_fc0_1_1_APPLICATION).') args = parser.parse_args() @@ -5793,7 +5793,6 @@ class Autosubmit: file contents (similar to the ``cat`` command) or to start tailing its output (akin to ``tail -f``). """ - def view_file(log_file, mode): if mode == 'c': cmd = ['cat', str(log_file)] diff --git a/docs/source/qstartguide/index.rst b/docs/source/qstartguide/index.rst index 2426c2b16..7edb6bec5 100644 --- a/docs/source/qstartguide/index.rst +++ b/docs/source/qstartguide/index.rst @@ -90,8 +90,8 @@ The output of the command will show the expid of the experiment and generate the Then, execute ``autosubmit create -np`` and Autosubmit will generate the workflow graph. -Run and monitoring: -=================== +Run and monitoring +================== To run an experiment, use ```autosubmit run ```. Autosubmit runs experiments performing the following operations: @@ -114,8 +114,32 @@ Concurrently, the ``/tmp`` gets filled with the cmd scripts generated by Autosubmit keeps logs at ``ASLOGS`` and ``LOG_a000`` folders, which are filled up with Autosubmit's command logs and job logs. -Configuration summary: -====================== +Viewing the logs +================ + +The ``autosubmit`` commands such as ``expid``, ``run``, ``monitor``, all may produce +log files on the user's file system. To save the user from having to navigate to the +log file, or to memorize the location of these files, Autosubmit provides the +``autosubmit cat-log`` command. + +.. TODO: add a link to complete docs of ``cat-log`` (we must have similar page(s) for each AS sub-command). + +.. code-block:: bash + + $ autosubmit cat-log a000 + Autosubmit is running with 4.0.0b + 2023-02-27 21:45:47,863 Autosubmit is running with 4.0.0b + 2023-02-27 21:45:47,872 + Checking configuration files... + 2023-02-27 21:45:47,900 expdef_a000.yml OK + 2023-02-27 21:45:47,904 platforms_a000.yml OK + 2023-02-27 21:45:47,905 jobs_a000.yml OK + 2023-02-27 21:45:47,906 autosubmit_a000.yml OK + 2023-02-27 21:45:47,907 Configuration files OK + + +Configuration summary +===================== In the folder ``/conf`` there are different files that define the actual experiment configuration. -- GitLab From 6bd229f684543184531c7aa4d2f961025c603385 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Mon, 13 Mar 2023 15:02:38 +0100 Subject: [PATCH 4/6] [cat-log] add unit tests --- autosubmit/autosubmit.py | 18 +++--- test/unit/test_catlog.py | 126 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 8 deletions(-) create mode 100644 test/unit/test_catlog.py diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 4237fc9d5..e190a6271 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -60,7 +60,7 @@ from autosubmitconfigparser.config.basicconfig import BasicConfig import locale from distutils.util import strtobool from log.log import Log, AutosubmitError, AutosubmitCritical -from typing import Set +from typing import Set, Union from autosubmit.database.db_common import update_experiment_descrip_version import sqlite3 @@ -5777,7 +5777,7 @@ class Autosubmit: return job_list @staticmethod - def cat_log(ID, file, mode): + def cat_log(ID: str, file: Union[None, str], mode: Union[None, str]) -> bool: """The cat-log command allows users to view Autosubmit logs using the command-line. It is possible to use ``autosubmit cat-log`` for Workflow and for Job logs. It decides @@ -5793,7 +5793,7 @@ class Autosubmit: file contents (similar to the ``cat`` command) or to start tailing its output (akin to ``tail -f``). """ - def view_file(log_file, mode): + def view_file(log_file: Path, mode: str): if mode == 'c': cmd = ['cat', str(log_file)] subprocess.Popen( @@ -5851,8 +5851,8 @@ class Autosubmit: workflow_log_files = sorted(aslogs_path.glob(search_pattern)) else: search_pattern = f'{expid}_*.txt' - stats_files_path = exp_path / 'status' - workflow_log_files = sorted(stats_files_path.glob(search_pattern)) + status_files_path = exp_path / 'status' + workflow_log_files = sorted(status_files_path.glob(search_pattern)) if not workflow_log_files: Log.info('No logs found.') @@ -5876,10 +5876,12 @@ class Autosubmit: Log.info('No logs found.') return True workflow_log_file = workflow_log_files[-1] - + + if not workflow_log_file.exists(): + Log.info('No logs found.') + return True + if not workflow_log_file.is_file(): raise AutosubmitCritical(f'The job log file {file} found is not a file: {workflow_log_file}', 7011) return view_file(workflow_log_file, mode) == 0 - - return False diff --git a/test/unit/test_catlog.py b/test/unit/test_catlog.py new file mode 100644 index 000000000..86dc6ae83 --- /dev/null +++ b/test/unit/test_catlog.py @@ -0,0 +1,126 @@ +from unittest import TestCase + +import io +import sys +from contextlib import suppress, redirect_stdout +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +from autosubmit.autosubmit import Autosubmit, AutosubmitCritical +from autosubmitconfigparser.config.basicconfig import BasicConfig + + +class TestJob(TestCase): + + def setUp(self): + self.autosubmit = Autosubmit() + # directories used when searching for logs to cat + self.original_root_dir = BasicConfig.LOCAL_ROOT_DIR + self.root_dir = TemporaryDirectory() + BasicConfig.LOCAL_ROOT_DIR = self.root_dir.name + self.exp_path = Path(self.root_dir.name, 'a000') + self.tmp_dir = self.exp_path / BasicConfig.LOCAL_TMP_DIR + self.aslogs_dir = self.tmp_dir / BasicConfig.LOCAL_ASLOG_DIR + self.status_path = self.exp_path / 'status' + self.aslogs_dir.mkdir(parents=True) + self.status_path.mkdir() + + def tearDown(self) -> None: + BasicConfig.LOCAL_ROOT_DIR = self.original_root_dir + if self.root_dir is not None: + self.root_dir.cleanup() + + def test_invalid_file(self): + def _fn(): + self.autosubmit.cat_log(None, '8', None) # type: ignore + self.assertRaises(AutosubmitCritical, _fn) + + def test_invalid_mode(self): + def _fn(): + self.autosubmit.cat_log(None, 'o', '8') # type: ignore + self.assertRaises(AutosubmitCritical, _fn) + + # -- workflow + + def test_is_workflow_invalid_file(self): + def _fn(): + self.autosubmit.cat_log('a000', 'j', None) + self.assertRaises(AutosubmitCritical, _fn) + + @patch('autosubmit.autosubmit.Log') + def test_is_workflow_not_found(self, Log): + self.autosubmit.cat_log('a000', 'o', 'c') + assert Log.info.called + assert Log.info.call_args[0][0] == 'No logs found.' + + def test_is_workflow_log_is_dir(self): + log_file_actually_dir = Path(self.aslogs_dir, 'log_run.log') + log_file_actually_dir.mkdir() + def _fn(): + self.autosubmit.cat_log('a000', 'o', 'c') + self.assertRaises(AutosubmitCritical, _fn) + + @patch('subprocess.Popen') + def test_is_workflow_out_cat(self, popen): + log_file = Path(self.aslogs_dir, 'log_run.log') + with open(log_file, 'w') as f: + f.write('as test') + f.flush() + self.autosubmit.cat_log('a000', file=None, mode='c') + assert popen.called + args = popen.call_args[0][0] + assert args[0] == 'cat' + assert args[1] == str(log_file) + + @patch('subprocess.Popen') + def test_is_workflow_status_tail(self, popen): + log_file = Path(self.status_path, 'a000_anything.txt') + with open(log_file, 'w') as f: + f.write('as test') + f.flush() + self.autosubmit.cat_log('a000', file='s', mode='t') + assert popen.called + args = popen.call_args[0][0] + assert args[0] == 'tail' + assert str(args[-1]) == str(log_file) + + # --- jobs + + @patch('autosubmit.autosubmit.Log') + def test_is_jobs_not_found(self, Log): + for file in ['j', 's', 'o']: + self.autosubmit.cat_log('a000_INI', file=file, mode='c') + assert Log.info.called + assert Log.info.call_args[0][0] == 'No logs found.' + + def test_is_jobs_log_is_dir(self): + log_file_actually_dir = Path(self.tmp_dir, 'LOG_a000/a000_INI.20000101.out') + log_file_actually_dir.mkdir(parents=True) + def _fn(): + self.autosubmit.cat_log('a000_INI', 'o', 'c') + self.assertRaises(AutosubmitCritical, _fn) + + @patch('subprocess.Popen') + def test_is_jobs_out_tail(self, popen): + log_dir = self.tmp_dir / 'LOG_a000' + log_dir.mkdir() + log_file = log_dir / 'a000_INI.20200101.out' + with open(log_file, 'w') as f: + f.write('as test') + f.flush() + self.autosubmit.cat_log('a000_INI', file=None, mode='t') + assert popen.called + args = popen.call_args[0][0] + assert args[0] == 'tail' + assert str(args[-1]) == str(log_file) + + # --- command-line + + def test_command_line_help(self): + args = ['autosubmit', 'cat-log', '--help'] + with patch.object(sys, 'argv', args) as _, io.StringIO() as buf, redirect_stdout(buf): + with suppress(SystemExit): + assert Autosubmit.parse_args() + assert buf + assert 'View workflow and job logs.' in buf.getvalue() -- GitLab From c37d66ee017afc3dde58b341b62de6539b1f424d Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Wed, 22 Mar 2023 10:55:56 +0100 Subject: [PATCH 5/6] Add an `-i` `--`inspect` option to use log files produced by `autosubmit inspect` (i.e. from /tmp) folder --- autosubmit/autosubmit.py | 38 +++++++++++++++++++++++-------- docs/source/qstartguide/index.rst | 7 ++++++ 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index e190a6271..12b076057 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -624,6 +624,8 @@ class Autosubmit: help='Workflow or Job log file. Options are o(output), j(job), e(error), s(status). Default o(output).') subparser.add_argument('-m', '--mode', default=None, action='store', metavar='MODE', help='Mode. Options are c(cat), t(tail). Default is c(cat).') + subparser.add_argument('-i', '--inspect', default=False, action='store_true', + help='Read job files generated by the inspect subcommand.') subparser.add_argument('ID', metavar='ID', help='An ID of a Workflow (eg a000) or a Job (eg a000_20220401_fc0_1_1_APPLICATION).') args = parser.parse_args() @@ -732,7 +734,7 @@ class Autosubmit: elif args.command == 'updatedescrip': return Autosubmit.update_description(args.expid, args.description) elif args.command == 'cat-log': - return Autosubmit.cat_log(args.ID, args.file, args.mode) + return Autosubmit.cat_log(args.ID, args.file, args.mode, args.inspect) @staticmethod def _init_logs(args, console_level='INFO', log_level='DEBUG', expid='None'): @@ -5777,7 +5779,7 @@ class Autosubmit: return job_list @staticmethod - def cat_log(ID: str, file: Union[None, str], mode: Union[None, str]) -> bool: + def cat_log(exp_or_job_id: str, file: Union[None, str], mode: Union[None, str], inspect:bool=False) -> bool: """The cat-log command allows users to view Autosubmit logs using the command-line. It is possible to use ``autosubmit cat-log`` for Workflow and for Job logs. It decides @@ -5792,6 +5794,12 @@ class Autosubmit: Finally, the ``MODE`` parameter allows users to choose whether to display the complete file contents (similar to the ``cat`` command) or to start tailing its output (akin to ``tail -f``). + + Args: + exp_or_job_id: A workflow or job ID. + file: the type of the file to be printed (not the file path!). + mode: the mode to print the file (e.g. cat, tail). + inspect: when True it will use job files in tmp/ instead of tmp/LOG_a000/. """ def view_file(log_file: Path, mode: str): if mode == 'c': @@ -5833,13 +5841,25 @@ class Autosubmit: if mode not in MODES.keys(): raise AutosubmitCritical(f'Invalid cat-log mode {mode}. Expected one of {[m for m in MODES.keys()]}', 7011) - is_workflow = '_' not in ID + is_workflow = '_' not in exp_or_job_id - expid = ID if is_workflow else ID[:4] + expid = exp_or_job_id if is_workflow else exp_or_job_id[:4] + # Workflow folder. + # e.g. ~/autosubmit/a000 exp_path = Path(BasicConfig.LOCAL_ROOT_DIR, expid) + # Directory with workflow temporary/volatile files. Contains the output of commands such as inspect, + # and also STAT/COMPLETED files for each workflow task. + # e.g. ~/autosubmit/a000/tmp tmp_path = exp_path / BasicConfig.LOCAL_TMP_DIR + # Directory with logs for Autosubmit executed commands (create, run, etc.) and jobs statuses files. + # e.g. ~/autosubmit/a000/tmp/ASLOGS aslogs_path = tmp_path / BasicConfig.LOCAL_ASLOG_DIR + # Directory with the logs of the workflow run, for each workflow task. Includes the generated + # .cmd files, and STAT/COMPLETED files for the run. The files with similar names in the parent + # directory are generated with inspect, while these are with the run subcommand. + # e.g. ~/autosubmit/a000/tmp/LOG_a000 + exp_logs_path = tmp_path / f'LOG_{expid}' if is_workflow: if file not in ['o', 'e', 's']: @@ -5864,14 +5884,14 @@ class Autosubmit: return view_file(workflow_log_file, mode) == 0 else: + job_logs_path = tmp_path if inspect else exp_logs_path if file == 'j': - workflow_log_file = tmp_path / f'{ID}.cmd' + workflow_log_file = job_logs_path / f'{exp_or_job_id}.cmd' elif file == 's': - workflow_log_file = tmp_path / f'{ID}_TOTAL_STATS' + workflow_log_file = job_logs_path / f'{exp_or_job_id}_TOTAL_STATS' else: - search_pattern = f'{ID}.*.{"err" if file == "e" else "out"}' - exp_logs_path = tmp_path / f'LOG_{ID[:4]}' - workflow_log_files = sorted(exp_logs_path.glob(search_pattern)) + search_pattern = f'{exp_or_job_id}.*.{"err" if file == "e" else "out"}' + workflow_log_files = sorted(job_logs_path.glob(search_pattern)) if not workflow_log_files: Log.info('No logs found.') return True diff --git a/docs/source/qstartguide/index.rst b/docs/source/qstartguide/index.rst index 7edb6bec5..43c3d87ee 100644 --- a/docs/source/qstartguide/index.rst +++ b/docs/source/qstartguide/index.rst @@ -137,6 +137,13 @@ log file, or to memorize the location of these files, Autosubmit provides the 2023-02-27 21:45:47,906 autosubmit_a000.yml OK 2023-02-27 21:45:47,907 Configuration files OK +.. note:: + The ``-f`` (``--file``) option is for the file type, not the file path. + See the complete help and syntax with ``autosubmit cat-log --help`` for + a list of supported types, depending on whether you choose a workflow + log or a job log file. Note too that there is a ``-i`` (``--inspect``) + flag in the command to tell Autosubmit you want job files generated by + ``autosubmit inspect``, instead of job files generated by ``autosubmit run``. Configuration summary ===================== -- GitLab From 5a6da290a4c472eda3498f8b13e582cc6f715d0f Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Fri, 24 Mar 2023 09:49:43 +0100 Subject: [PATCH 6/6] Replace SafeConfigParser (deprecated) by ConfigParser --- autosubmit/autosubmit.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 12b076057..bc709d27f 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -30,13 +30,7 @@ from .notifications.mail_notifier import MailNotifier from bscearth.utils.date import date2str from pathlib import Path from autosubmitconfigparser.config.yamlparser import YAMLParserFactory - -try: - # noinspection PyCompatibility - from configparser import SafeConfigParser -except ImportError: - # noinspection PyCompatibility - from bscearth.utils.config_parser import ConfigParser as SafeConfigParser +from configparser import ConfigParser from .monitor.monitor import Monitor from .database.db_common import get_autosubmit_version, check_experiment_exists @@ -3549,7 +3543,7 @@ class Autosubmit: config_file = open(rc_path, 'w') Log.info("Writing configuration file...") try: - parser = SafeConfigParser() + parser = ConfigParser() parser.add_section('database') parser.set('database', 'path', database_path) if database_filename is not None and len(str(database_filename)) > 0: @@ -3671,7 +3665,7 @@ class Autosubmit: d.infobox("Reading configuration file...", width=50, height=5) try: if os.path.isfile(path): - parser = SafeConfigParser() + parser = ConfigParser() parser.optionxform = str parser.load(path) if parser.has_option('database', 'path'): @@ -3790,7 +3784,7 @@ class Autosubmit: config_file = open(path, 'w') d.infobox("Writing configuration file...", width=50, height=5) try: - parser = SafeConfigParser() + parser = ConfigParser() parser.add_section('database') parser.set('database', 'path', database_path) if database_filename: -- GitLab