From 52294be774e0e73ab5f04a4c0f55e54a5c812ace Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Fri, 19 Jan 2024 12:08:50 +0100 Subject: [PATCH] Create experiment, workflow, and tasks documentation from configuration, as well as metadata --- autosubmit/autosubmit.py | 21 +++++++ autosubmit/docgen/__init__.py | 104 ++++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 autosubmit/docgen/__init__.py diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index d94662632..ce5bd538e 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -637,6 +637,9 @@ class Autosubmit: help='Read job files generated by the inspect subcommand.') subparser.add_argument('ID', metavar='ID', help='An ID of a Workflow (eg a000) or a Job (eg a000_20220401_fc0_1_1_APPLICATION).') + subparser = subparsers.add_parser('docgen', description='Documentation generator.') + subparser.add_argument('expid', help='Experiment ID.') + args = parser.parse_args() if args.command is None: @@ -743,6 +746,8 @@ class Autosubmit: return Autosubmit.update_description(args.expid, args.description) elif args.command == 'cat-log': return Autosubmit.cat_log(args.ID, args.file, args.mode, args.inspect) + elif args.command == 'docgen': + return Autosubmit.docgen(args.expid) @staticmethod def _init_logs(args, console_level='INFO', log_level='DEBUG', expid='None'): @@ -6155,3 +6160,19 @@ class Autosubmit: raise AutosubmitCritical(f'The job log file {file} found is not a file: {workflow_log_file}', 7011) return view_file(workflow_log_file, mode) == 0 + + @staticmethod + def docgen(expid: str) -> bool: + """Automatically generate documentation for the experiment. + + Args: + expid: An experiment ID. + """ + as_conf = AutosubmitConfig(expid, BasicConfig, YAMLParserFactory()) + as_conf.check_conf_files(False) + + from .docgen import generate_documentation + + generate_documentation(as_conf) + return True + \ No newline at end of file diff --git a/autosubmit/docgen/__init__.py b/autosubmit/docgen/__init__.py new file mode 100644 index 000000000..a983c6289 --- /dev/null +++ b/autosubmit/docgen/__init__.py @@ -0,0 +1,104 @@ +# Copyright 2015-2023 Earth Sciences Department, BSC-CNS +# +# This file is part of Autosubmit. +# +# Autosubmit is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# Autosubmit is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +"""Documentation generator.""" + +from os import linesep + +from typing import Any, Dict + +from autosubmitconfigparser.config.configcommon import AutosubmitConfig + + +def _print_jobs_docs(expid: str, jobs: Dict[str, Any]) -> None: + """Print the documentation for the jobs. + + :param jobs: A dictionary with the job section and the job values. + :type jobs: Dict[str, Any] + :return: Nothing. + :rtype: None + """ + # TODO: use some sort of template -- later + print(f'# Workflow Jobs{linesep}') + # TODO: add description + print(f'These are the jobs in the workflow “{expid}{linesep}”.') + + for job_section, job_dict in jobs.items(): + print(f'## {job_section}{linesep}') + if {'DOC', 'TITLE'}.intersection(job_dict.keys()): + title = job_dict['TITLE'] + doc = job_dict['DOC'] + + print(f'**{title}**') + print(f'{doc}{linesep}') + + +def generate_documentation(as_conf: AutosubmitConfig): + """Automated documentation generation for Autosubmit experiments. + + Goes over experiment configuration, gathering each job's + ``TITLE`` and ``DOC`` to create the documentation for the + jobs. + + Goes over the ``METADATA`` configuration entry, looking + for sections with ``key=value`` pairs, or links to + configuration keys, e.g.: + + .. code-block:: yaml + + METADATA: + MODEL: + # This is a static value that will produce name = IFS + - name: name + # This is a dynamic value that will produce resolution = abc... + - resolution: %RUN.IFS.RESOLUTION% + # This resolves the key and value automatically and will produce RUN.IFS.INIPATH=/path/path/... + - %RUN.IFS.INIPATH% + # This is the extended format + - name: source + value: https://git@... + documentation: | + The source code is hosted at the private repository... + DATA: + - PROVENANCE: ... + + + Each property of the ``METADATA`` YAML object is rendered as + a separate section. These sections, in YAML, are arrays that + contain metadata references in each line. Extra documentation + about each key can be entered using the extended format. + + :param as_conf: Autosubmit configuration + :type as_conf: AutosubmitConfig + :return: Nothing. + :rtype: None + """ + + # TODO: expand variables (like %EXPID%); forgot how that is done -- check with Dani later + _print_jobs_docs(as_conf.expid, as_conf.jobs_data) + + # _print_jobs_metadata(as_conf) + # N.B. These keys must be defined by experiment experts (e.g. workflow devs, app devs, users, etc.). + # Our job here is just on the automation side. We must try to keep it simple and + # transparent as this reflects on reproducibility and traceability in the experiment + # and any customization or logic-heavy work here could make it harder to confirm + # the tool is not responsible for some value that could be incorrect in the report! + print('# Metadata') + + # TODO: fetch the contents of METADATA, and do as the docstring describes + + +___all__ = [ + 'generate_documentation' +] -- GitLab