diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c06571926c020bf8ade109226354e90b57e6ae3e..fd84d0e3d14f4f91d5fa7e93c972a445ac0a83f1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,7 @@ before_script: stages: - prepare + - docs - test - report - clean @@ -18,6 +19,16 @@ prepare: - conda update conda - conda install -n autosubmit3 coverage=6 +docs: + stage: docs + script: + - conda env update -f environment.yml -n autosubmit3 python=3.7.3 + - source activate autosubmit3 + - pip install -e . + - cd docs + - pip install -r requirements.txt + - make html + test_python3: stage: test script: diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 55ec20671a54293b0a3f6ea3c5e65b18827c945d..ba5b2082849fea5267ba8a0e845d1138c529b03b 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -4260,11 +4260,15 @@ class Autosubmit: Archives an experiment: call clean (if experiment is of version 3 or later), compress folder to tar.gz and moves to year's folder - :param uncompress: - :param noclean: - :return: :param expid: experiment identifier :type expid: str + :param noclean: flag telling it whether to clean the experiment or not. + :type noclean: bool + :param uncompress: flag telling it whether to decompress or not. + :type uncompress: bool + :return: ``True`` if the experiment has been successfully archived. ``False`` otherwise. + :rtype: bool + """ exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid) @@ -4441,7 +4445,7 @@ class Autosubmit: :param expid: experiment identifier :type expid: str :param noplot: if True, method omits final plotting of the jobs list. Only needed on large experiments when - plotting time can be much larger than creation time. + plotting time can be much larger than creation time. :type noplot: bool :return: True if successful, False if not :rtype: bool diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 25cd4b1512abc56380563adebfc179ccf2703880..732336f9d7247f133ea4e5f2c0cc7e7e9c7b7897 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -369,7 +369,7 @@ class Job(object): Add parents for the job. It also adds current job as a child for all the new parents :param parents: job's parents to add - :type parents: *Job + :type parents: Job """ for parent in parents: num_parents = 1 diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index b92c892609cd8da6ea6f31f59f7dc1c6e7090428..15a9f8ed74b14c8c3c230de84541aa424cc9d0bd 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -193,8 +193,8 @@ class JobList(object): :type default_retrials: int :param new: is it a new generation? :type new: bool \n - :param wrapper_type: Type of wrapper defined by the user in autosubmit_.yml [wrapper] section. \n - :param wrapper_jobs: Job types defined in autosubmit_.yml [wrapper sections] to be wrapped. \n + :param wrapper_type: Type of wrapper defined by the user in ``autosubmit_.yml`` [wrapper] section. \n + :param wrapper_jobs: Job types defined in ``autosubmit_.yml`` [wrapper sections] to be wrapped. \n :type wrapper_jobs: String \n """ self._parameters = parameters diff --git a/docs/Makefile b/docs/Makefile index 11efdb9c3fcf65b7f8ed02c8206e6e711891dbf1..11b7542449e67bf2b9da817fa45049ce12772be2 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -2,7 +2,7 @@ # # You can set these variables from the command line. -SPHINXOPTS = +SPHINXOPTS = -W SPHINXBUILD = sphinx-build PAPER = BUILDDIR = build diff --git a/docs/requirements.txt b/docs/requirements.txt index 82133027c9eb6dd3ff20c82704ecf25650e474f5..146feb539ee2f80f4fa6d59094b9f9a18225bc48 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,3 @@ sphinx sphinx_rtd_theme +sphinx-reredirects==0.1.* diff --git a/docs/source/_static/.keep b/docs/source/_static/.keep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/docs/source/agui/experiment/index.rst b/docs/source/agui/experiment/index.rst index 55cd49b83aacdcab31fb6cffc8542811445cb875..a052e670b9c99ada1a3541986482440e53137233 100644 --- a/docs/source/agui/experiment/index.rst +++ b/docs/source/agui/experiment/index.rst @@ -17,4 +17,4 @@ At the top left you see the ``Autosubmit Searcher`` home link that will take you At the bottom you see some relevant metadata, including the ``branch`` of the ``model`` that was used in the experiment, the ``HPC name`` targeted by the experiment, the owner, the ``version`` that this experiment us running on, the ``DB`` version of Autosubmit, and the number of jobs in the experiment. -On the center you see the :ref:`treeRepresentation`, which is loaded automatically when you open this page. \ No newline at end of file +On the center you see the :doc:`../tree/index`, which is loaded automatically when you open this page. \ No newline at end of file diff --git a/docs/source/agui/graph/index.rst b/docs/source/agui/graph/index.rst index 96e95d5ae9f6e2ba8e5af7922d809566c873e642..7c12c1dee8ad0e264fde6152ba843a91687ad18f 100644 --- a/docs/source/agui/graph/index.rst +++ b/docs/source/agui/graph/index.rst @@ -1,5 +1,3 @@ -.. _graphRepresentation: - Graph Representation ==================== diff --git a/docs/source/agui/index.rst b/docs/source/agui/index.rst index e319633825f5d61542f9a808a5b9907f30c71aab..a44d7e8aa8edfe2c841c5861dfdb6b49b5209f23 100644 --- a/docs/source/agui/index.rst +++ b/docs/source/agui/index.rst @@ -1,8 +1,11 @@ +:orphan: + Autosubmit GUI ############## .. toctree:: - :hidden: + :hidden: + experiment/index tree/index graph/index diff --git a/docs/source/agui/tree/index.rst b/docs/source/agui/tree/index.rst index ff9e604acf5882e45b3c655e83239eb3134d8695..ff0e16f5dec79a43b70a375af721cebbb21fb2cd 100644 --- a/docs/source/agui/tree/index.rst +++ b/docs/source/agui/tree/index.rst @@ -1,5 +1,3 @@ -.. _treeRepresentation: - Tree Representation =================== @@ -73,7 +71,7 @@ As you might infer, the logic is fairly straightforward: Start your string with You can extend this functionality considering that date, member, section, chunk names start with the symbol ``_`` and finish with the same symbol. -.. important:: This view is designed to show a structured view of your experiment, if you want a more dependency oriented view that shows better the execution sequence of your jobs, you can refer to :ref:`graphRepresentation`. +.. important:: This view is designed to show a structured view of your experiment, if you want a more dependency oriented view that shows better the execution sequence of your jobs, you can refer to :doc:`../graph/index`. diff --git a/docs/source/conf.py b/docs/source/conf.py index beb4ccc4851d567863f65e8103e83621f9543f71..85e0befedb9bb2652e9b870178d2ed770463940e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -33,9 +33,12 @@ extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.imgmath', 'sphinx.ext.autosectionlabel', - 'sphinx_rtd_theme' + 'sphinx_rtd_theme', + 'sphinx_reredirects' ] +autosectionlabel_prefix_document = True + numfig = True numfig_format = {'figure': '%s', 'table': '%s', 'code-block': '%s'} @@ -73,7 +76,7 @@ release = '4.0.0b' # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: @@ -117,6 +120,10 @@ autodoc_mock_imports = ["portalocker", "argparse", "python-dateutil", "py3dotplu 'mock', "networkx", 'networkx.algorithms.dag', 'bscearth.utils', 'bscearth.utils.config_parser', 'bscearth.utils.date'] +redirects = { + 'devguide/variables' : '../userguide/variables.html' +} + # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for diff --git a/docs/source/devguide/fig3.png b/docs/source/devguide/fig3.png new file mode 100644 index 0000000000000000000000000000000000000000..9a330fd50118a8f278cf8af115aea6448ee7df83 Binary files /dev/null and b/docs/source/devguide/fig3.png differ diff --git a/docs/source/index.rst b/docs/source/index.rst index 0271d721b0de0730ebce33aa1c5731a76335a23a..67658a0116557c6893cb0e6d07c49e3a09632097 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -19,7 +19,6 @@ Welcome to autosubmit's documentation! :hidden: /qstartguide/index - /devguide/index .. toctree:: :caption: Installation @@ -33,15 +32,17 @@ Welcome to autosubmit's documentation! :maxdepth: 2 :hidden: + /userguide/index /userguide/create/index /userguide/configure/index /userguide/defining_workflows/index /userguide/wrappers/index /userguide/run/index - /userguide/modifying_workflows/index + /userguide/modifying_workflow/index /userguide/manage/index /userguide/monitor_and_check/index /userguide/set_and_share_the_configuration/index + /userguide/variables .. toctree:: :caption: Database Documentation diff --git a/docs/source/moduledoc/config.rst b/docs/source/moduledoc/config.rst index 5fb79db246f54dee7164c59c8ded7b640b005de4..d26ca87307aa7b1c6a45fb33d75ec33fbebfa9d0 100644 --- a/docs/source/moduledoc/config.rst +++ b/docs/source/moduledoc/config.rst @@ -1,17 +1,17 @@ autosubmit.config ================= -autosubmit.config.basicConfig ------------------------------ +autosubmitconfigparser.config.basicConfig +----------------------------------------- -.. automodule:: autosubmit.config.basicConfig +.. automodule:: autosubmitconfigparser.config.basicconfig :show-inheritance: :inherited-members: :members: -autosubmit.config.config_common -------------------------------- -.. automodule:: autosubmit.config.config_common +autosubmitconfigparser.config.config_common +------------------------------------------- +.. automodule:: autosubmitconfigparser.config.configcommon :show-inheritance: :inherited-members: :members: diff --git a/docs/source/qstartguide/index.rst b/docs/source/qstartguide/index.rst index 518dba4286edce2338f04b53d69ea61cda5e2184..8cde6b736446fff4ddf7c95ff33dcd6fd5ccb9b5 100644 --- a/docs/source/qstartguide/index.rst +++ b/docs/source/qstartguide/index.rst @@ -91,7 +91,7 @@ The output of the command will show the expid of the experiment and generate the Then, execute ``autosubmit create -np`` and Autosubmit will generate the workflow graph. Run and monitoring: -=============== +=================== To run an experiment, use ```autosubmit run ```. Autosubmit runs experiments performing the following operations: @@ -115,7 +115,7 @@ Concurrently, the ``/tmp`` gets filled with the cmd scripts generated by Autosubmit keeps logs at ``ASLOGS`` and ``LOG_a000`` folders, which are filled up with Autosubmit's command logs and job logs. Configuration summary: -================== +====================== In the folder ``/conf`` there are different files that define the actual experiment configuration. diff --git a/docs/source/troubleshooting/changelog.rst b/docs/source/troubleshooting/changelog.rst index ce2278a98cb5b2d27ebeaa1947727fa67ef4bc54..34adb74db5bcbfbc92493e6ed476e1d723159b67 100644 --- a/docs/source/troubleshooting/changelog.rst +++ b/docs/source/troubleshooting/changelog.rst @@ -8,11 +8,13 @@ Mayor mentions: - Python version has changed to 3.7.3 instead of 2.7. - Configuration language has changed to YAML. + - All parameters are now unified into a single dictionary. - All sections are now uppercase. - All parameters, except for job related ones, have now an hierarchy. - An special key, FOR:, has been added. This key allows to create multiple jobs with almost the same configuration. - The configuration of autosubmit is now more flexible. + - New command added, upgrade. This command will update all the scripts and autosubmit configuration. - Wrapper definition has changed. - Tasks dependencies system has changed. @@ -86,7 +88,7 @@ Configuration DEFAULT: EXPID: a02u HPCARCH: local - CUSTOM_CONFIG: %ROOTDIR%/conf/custom_conf + CUSTOM_CONFIG: "%ROOTDIR%/conf/custom_conf" experiment: DATELIST: '20210811' MEMBERS: CompilationEfficiency HardwareBenchmarks WeakScaling StrongScaling @@ -342,18 +344,25 @@ The DEPENDENCIES key is used to define the dependencies of a job. It can be used - Basic: The dependencies are a list of jobs, separated by " ", that runs before the current task is submitted. - New: The dependencies is a list of YAML sections, separated by "\n", that runs before the current job is submitted. + - For each dependency section, you can designate the following keywords to control the current job-affected tasks: + - DATES_FROM: Selects the job dates that you want to alter. - MEMBERS_FROM: Selects the job members that you want to alter. - CHUNKS_FROM: Selects the job chunks that you want to alter. - - For each dependency section and *_FROM keyword, you can designate the following keywords to control the destination of the dependency: + + - For each dependency section and \*_FROM keyword, you can designate the following keywords to control the destination of the dependency: + - DATES_TO: Links current selected tasks to the dependency tasks of the dates specified. - MEMBERS_TO: Links current selected tasks to the dependency tasks of the members specified. - CHUNKS_TO: Links current selected tasks to the dependency tasks of the chunks specified. + - Important keywords for [DATES|MEMBERS|CHUNKS]_TO: + - "natural": Will keep the default linkage. Will link if it would be normally. Example, SIM_FC00_CHUNK_1 -> DA_FC00_CHUNK_1. - "all": Will link all selected tasks of the dependency with current selected tasks. Example, SIM_FC00_CHUNK_1 -> DA_FC00_CHUNK_1, DA_FC00_CHUNK_2, DA_FC00_CHUNK_3... - "none": Will unlink selected tasks of the dependency with current selected tasks. + For the new format, consider that the priority is hierarchy and goes like this DATES_FROM -(includes)-> MEMBERS_FROM -(includes)-> CHUNKS_FROM. - You can define a DATES_FROM inside the DEPENDENCY. @@ -368,6 +377,7 @@ For the examples, we will consider that our experiment has the following configu DATELIST: 20220101 MEMBERS: FC1 FC2 NUMCHUNKS: 4 + Basic ===== @@ -453,7 +463,7 @@ New format RUNNING: chunk .. figure:: fig/new_dependencies_0.png - :name: new_dependencies + :name: new_dependencies_0 :align: center :alt: new_dependencies @@ -508,7 +518,7 @@ In the following example, we want to launch the next member SIM after the last S RUNNING: once .. figure:: fig/new_dependencies_1.png - :name: new_dependencies + :name: new_dependencies_1 :align: center :alt: new_dependencies diff --git a/docs/source/troubleshooting/error-codes.rst b/docs/source/troubleshooting/error-codes.rst index 21a252b9380b7583dd93ef9a26d0597c0de4c62a..a96bcad67b37d8187c922063e88186a6db68e7ed 100644 --- a/docs/source/troubleshooting/error-codes.rst +++ b/docs/source/troubleshooting/error-codes.rst @@ -1,16 +1,15 @@ -.. _errorcodes: - ######################### Error codes and solutions ######################### Experiment Locked - Critical Error 7000 -=================================================== +======================================= +------------------------------------------------------------------------------------------------------------------------------------------------+ | Code | Details | Solution | +======+===================================================================+=====================================================================+ | 7000 | Experiment is locked due another instance of Autosubmit using it | Halt other experiment instances //Delete /tmp/autosubmit.lock| +------+-------------------------------------------------------------------+---------------------------------------------------------------------+ + Database Issues - Critical Error codes [7001-7005] =================================================== @@ -27,11 +26,13 @@ Database Issues - Critical Error codes [7001-7005] +------+-----------------------------------------------+-----------------------------------------------------------------+ | 7005 | AS database is corrupted or locked | Please, open a new issue ASAP. (If you are on BSC environment) | +------+-----------------------------------------------+-----------------------------------------------------------------+ -| 7006 | Experiment database not found | Please, ask your administrator to run `autosubmit install | +| 7006 | Experiment database not found | Please, ask your administrator to run `autosubmit install` | +------+-----------------------------------------------+-----------------------------------------------------------------+ | 7007 | Experiment database permissions | Invalid permissions, ask your administrator to add R/W | +------+-----------------------------------------------+-----------------------------------------------------------------+ +.. _default_solution_1: + Default Solution ---------------- These issues are usually from server side, please, ask first in Autosubmit git if you don't have a custom installation. @@ -55,8 +56,7 @@ Wrong User Input - Critical Error codes [7010-7030] | 7014 | Wrong configuration | Check your experiment/conf files, also take a look to the ASLOG/command.log detailed output | +------+------------------------------------------------------+------------------------------------------------------------------------------------------------+ -Default Solution ----------------- +**Default Solution** These issues are usually mistakes from the user input, check the available logs and git resolved issues. Alternative, you can ask for help to Autosubmit team. @@ -81,8 +81,7 @@ Platform issues - Critical Error codes. Local [7040-7050] and remote [7050-7060 | 7052 | Scheduler is not installed or correctly configured. | Check if there is a scheduler installed in the remote machine. | +------+-----------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------+ -Default Solution ----------------- +**Default Solution** Check autosubmit log for detailed information, there will be additional error codes. @@ -125,8 +124,7 @@ Uncatalogued codes - Critical Error codes [7060+] | | | Alternative: You can use a non-encrypted key, just make sure that nobody except you has access to the file. | +------+-----------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -Default Solution ----------------- +**Default Solution** Check autosubmit log for detailed information, there will be additional error codes. diff --git a/docs/source/troubleshooting/index.rst b/docs/source/troubleshooting/index.rst index 1dc8dd28df3978b2895ba7d90a912dcd0c776fa1..8be0082119fc69e1e0f8799636af416484dbe954 100644 --- a/docs/source/troubleshooting/index.rst +++ b/docs/source/troubleshooting/index.rst @@ -53,9 +53,9 @@ Other possible errors Error codes =========== -The latest version of **Autosubmit** implements a code system that guides you through the process of fixing some of the common problems you might find. Check :ref:`errorcodes`, where you will find the list of error codes, their descriptions, and solutions. +The latest version of **Autosubmit** implements a code system that guides you through the process of fixing some of the common problems you might find. Check :doc:`error-codes`, where you will find the list of error codes, their descriptions, and solutions. Changelog ========= -review :ref:`changelog`. \ No newline at end of file +review :doc:`changelog`. \ No newline at end of file diff --git a/docs/source/userguide/configure/develop_a_project.rst b/docs/source/userguide/configure/develop_a_project.rst index f2b92e6b301e60d77b50890a75d99280134b6fea..4a18272adf0a2d0e02c31c707ad7b15637aaa875 100644 --- a/docs/source/userguide/configure/develop_a_project.rst +++ b/docs/source/userguide/configure/develop_a_project.rst @@ -1,3 +1,4 @@ +:orphan: .. _develproject: diff --git a/docs/source/userguide/configure/index.rst b/docs/source/userguide/configure/index.rst index 5e0ea58b404f8a16de993bb4aaa11da9af46d256..87d63b976bc5551488efebfb435c2c2ec1850590 100644 --- a/docs/source/userguide/configure/index.rst +++ b/docs/source/userguide/configure/index.rst @@ -1,5 +1,3 @@ -.. _configuration: - Configure Experiments ===================== @@ -94,8 +92,9 @@ This is the minimum job definition and usually is not enough. You usually will n * DEPENDENCIES: defines dependencies from job as a list of parents jobs separated by spaces. For example, if 'new_job' has to wait for "old_job" to finish, you must add the line "DEPENDENCIES: old_job". + * For dependencies to jobs running in previous chunks, members or start-dates, use -(DISTANCE). For example, for a job "SIM" waiting for - the previous "SIM" job to finish, you have to add "DEPENDENCIES: SIM-1". + the previous "SIM" job to finish, you have to add "DEPENDENCIES: SIM-1". * For dependencies that are not mandatory for the normal workflow behaviour, you must add the char '?' at the end of the dependency. @@ -118,7 +117,7 @@ To do this use: * RETRIALS: Number of retrials if job fails -* DELAY_RETRY_TIME: Allows to put a delay between retries. Triggered when a job fails. If not specified, Autosubmit will retry the job as soon as possible. Accepted formats are: plain number (there will be a constant delay between retrials, of as many seconds as specified), plus (+) sign followed by a number (the delay will steadily increase by the addition of these number of seconds), or multiplication (*) sign follows by a number (the delay after n retries will be the number multiplied by 10*n). Having this in mind, the ideal scenario is to use +(number) or plain(number) in case that the HPC has little issues or the experiment will run for a little time. Otherwise, is better to use the *(number) approach. +* DELAY_RETRY_TIME: Allows to put a delay between retries. Triggered when a job fails. If not specified, Autosubmit will retry the job as soon as possible. Accepted formats are: plain number (there will be a constant delay between retrials, of as many seconds as specified), plus (+) sign followed by a number (the delay will steadily increase by the addition of these number of seconds), or multiplication (*) sign follows by a number (the delay after n retries will be the number multiplied by 10*n). Having this in mind, the ideal scenario is to use +(number) or plain(number) in case that the HPC has little issues or the experiment will run for a little time. Otherwise, is better to use the \*(number) approach. .. code-block:: ini @@ -427,7 +426,7 @@ Example: DEPENDENCIES: SIM RUNNING: chunk WALLCLOCK: 00:05 - EXECUTABLE: %PROJDIR%/my_python_env/python3 + EXECUTABLE: "%PROJDIR%/my_python_env/python3" The result is a ``shebang`` line ``#!/esarchive/autosubmit/my_python_env/python3``. diff --git a/docs/source/userguide/create/index.rst b/docs/source/userguide/create/index.rst index 23cbf75e6e1c946937b526f60488646f78d60941..72050ce8429fa485a058b01eadc4b041ee06747a 100644 --- a/docs/source/userguide/create/index.rst +++ b/docs/source/userguide/create/index.rst @@ -25,6 +25,7 @@ Options: sets a description for the experiment to store in the database. -c PATH, --config PATH if specified, copies config files from a folder + Example: :: @@ -36,6 +37,7 @@ In this autosubmitrc or .autosubmitrc file, include the configuration setting cu Example: :: + conf: custom_platforms: /home/Earth/user/custom.yml @@ -55,11 +57,13 @@ It registers a new unique identifier and copies all configuration files in the n *COPY* is the experiment identifier to copy from. *Description* is a brief experiment description. *CONFIG* is a folder that exists. + Example: :: autosubmit expid -y cxxx -H ithaca -d "experiment is about..." autosubmit expid -y cxxx -p "/esarchive/autosubmit/genericFiles/conf" -H marenostrum4 -d "experiment is about..." + .. warning:: You can only copy experiments created with Autosubmit 3.11 or above. If there is an autosubmitrc or .autosubmitrc file in your home directory (cd ~), you can setup a default file from where the contents of platforms_expid.yml should be copied. @@ -68,6 +72,7 @@ In this autosubmitrc or .autosubmitrc file, include the configuration setting cu Example: :: + conf: custom_platforms: /home/Earth/user/custom.yml diff --git a/docs/source/userguide/configure/fig/dashed.png b/docs/source/userguide/defining_workflows/fig/dashed.png similarity index 100% rename from docs/source/userguide/configure/fig/dashed.png rename to docs/source/userguide/defining_workflows/fig/dashed.png diff --git a/docs/source/userguide/configure/fig/date-synchronize.png b/docs/source/userguide/defining_workflows/fig/date-synchronize.png similarity index 100% rename from docs/source/userguide/configure/fig/date-synchronize.png rename to docs/source/userguide/defining_workflows/fig/date-synchronize.png diff --git a/docs/source/userguide/configure/fig/dependencies_previous.png b/docs/source/userguide/defining_workflows/fig/dependencies_previous.png similarity index 100% rename from docs/source/userguide/configure/fig/dependencies_previous.png rename to docs/source/userguide/defining_workflows/fig/dependencies_previous.png diff --git a/docs/source/userguide/configure/fig/dependencies_running.png b/docs/source/userguide/defining_workflows/fig/dependencies_running.png similarity index 100% rename from docs/source/userguide/configure/fig/dependencies_running.png rename to docs/source/userguide/defining_workflows/fig/dependencies_running.png diff --git a/docs/source/userguide/configure/fig/experiment_delay_doc.png b/docs/source/userguide/defining_workflows/fig/experiment_delay_doc.png similarity index 100% rename from docs/source/userguide/configure/fig/experiment_delay_doc.png rename to docs/source/userguide/defining_workflows/fig/experiment_delay_doc.png diff --git a/docs/source/userguide/configure/fig/frequency.png b/docs/source/userguide/defining_workflows/fig/frequency.png similarity index 100% rename from docs/source/userguide/configure/fig/frequency.png rename to docs/source/userguide/defining_workflows/fig/frequency.png diff --git a/docs/source/userguide/configure/fig/member-synchronize.png b/docs/source/userguide/defining_workflows/fig/member-synchronize.png similarity index 100% rename from docs/source/userguide/configure/fig/member-synchronize.png rename to docs/source/userguide/defining_workflows/fig/member-synchronize.png diff --git a/docs/source/userguide/configure/fig/no-synchronize.png b/docs/source/userguide/defining_workflows/fig/no-synchronize.png similarity index 100% rename from docs/source/userguide/configure/fig/no-synchronize.png rename to docs/source/userguide/defining_workflows/fig/no-synchronize.png diff --git a/docs/source/userguide/configure/fig/running.png b/docs/source/userguide/defining_workflows/fig/running.png similarity index 100% rename from docs/source/userguide/configure/fig/running.png rename to docs/source/userguide/defining_workflows/fig/running.png diff --git a/docs/source/userguide/configure/fig/select_chunks.png b/docs/source/userguide/defining_workflows/fig/select_chunks.png similarity index 100% rename from docs/source/userguide/configure/fig/select_chunks.png rename to docs/source/userguide/defining_workflows/fig/select_chunks.png diff --git a/docs/source/userguide/configure/fig/select_members.png b/docs/source/userguide/defining_workflows/fig/select_members.png similarity index 100% rename from docs/source/userguide/configure/fig/select_members.png rename to docs/source/userguide/defining_workflows/fig/select_members.png diff --git a/docs/source/userguide/configure/fig/simple.png b/docs/source/userguide/defining_workflows/fig/simple.png similarity index 100% rename from docs/source/userguide/configure/fig/simple.png rename to docs/source/userguide/defining_workflows/fig/simple.png diff --git a/docs/source/userguide/configure/fig/skip.png b/docs/source/userguide/defining_workflows/fig/skip.png similarity index 100% rename from docs/source/userguide/configure/fig/skip.png rename to docs/source/userguide/defining_workflows/fig/skip.png diff --git a/docs/source/userguide/configure/fig/split.png b/docs/source/userguide/defining_workflows/fig/split.png similarity index 100% rename from docs/source/userguide/configure/fig/split.png rename to docs/source/userguide/defining_workflows/fig/split.png diff --git a/docs/source/userguide/defining_workflows/index.rst b/docs/source/userguide/defining_workflows/index.rst index 8ec731c71d31c9c323b41ceb56de21196bdc54ae..a8c19daa029187882cbecf68b2f8a6629fbe89fb 100644 --- a/docs/source/userguide/defining_workflows/index.rst +++ b/docs/source/userguide/defining_workflows/index.rst @@ -6,7 +6,7 @@ of the workflow the experiment will use. In this section you will learn about th be able to exploit autosubmit's full potential .. warning:: - This section is NOT intended to show how to define your jobs. Please go to :doc:`tutorial` section for a comprehensive + This section is NOT intended to show how to define your jobs. Please go to :doc:`/qstartguide/index` section for a comprehensive list of job options. @@ -31,7 +31,7 @@ be finished before launching the job that has the DEPENDENCIES attribute. DEPENDENCIES: One -The resulting workflow can be seen in Figure :numref:`simple` +The resulting workflow can be seen in Fi165gure :numref:`simple` .. figure:: fig/simple.png :name: simple @@ -177,25 +177,32 @@ Dependencies rework The DEPENDENCIES key is used to define the dependencies of a job. It can be used in the following ways: -- Basic: The dependencies are a list of jobs, separated by " ", that runs before the current task is submitted. -- New: The dependencies is a list of YAML sections, separated by "\n", that runs before the current job is submitted. - - For each dependency section, you can designate the following keywords to control the current job-affected tasks: - - DATES_FROM: Selects the job dates that you want to alter. - - MEMBERS_FROM: Selects the job members that you want to alter. - - CHUNKS_FROM: Selects the job chunks that you want to alter. - - For each dependency section and *_FROM keyword, you can designate the following keywords to control the destination of the dependency: - - DATES_TO: Links current selected tasks to the dependency tasks of the dates specified. - - MEMBERS_TO: Links current selected tasks to the dependency tasks of the members specified. - - CHUNKS_TO: Links current selected tasks to the dependency tasks of the chunks specified. - - Important keywords for [DATES|MEMBERS|CHUNKS]_TO: - - "natural": Will keep the default linkage. Will link if it would be normally. Example, SIM_FC00_CHUNK_1 -> DA_FC00_CHUNK_1. - - "all": Will link all selected tasks of the dependency with current selected tasks. Example, SIM_FC00_CHUNK_1 -> DA_FC00_CHUNK_1, DA_FC00_CHUNK_2, DA_FC00_CHUNK_3... - - "none": Will unlink selected tasks of the dependency with current selected tasks. +* Basic: The dependencies are a list of jobs, separated by " ", that runs before the current task is submitted. +* New: The dependencies is a list of YAML sections, separated by "\n", that runs before the current job is submitted. + + * For each dependency section, you can designate the following keywords to control the current job-affected tasks: + + * DATES_FROM: Selects the job dates that you want to alter. + * MEMBERS_FROM: Selects the job members that you want to alter. + * CHUNKS_FROM: Selects the job chunks that you want to alter. + + * For each dependency section and \*_FROM keyword, you can designate the following keywords to control the destination of the dependency: + + * DATES_TO: Links current selected tasks to the dependency tasks of the dates specified. + * MEMBERS_TO: Links current selected tasks to the dependency tasks of the members specified. + * CHUNKS_TO: Links current selected tasks to the dependency tasks of the chunks specified. + + * Important keywords for [DATES|MEMBERS|CHUNKS]_TO: + + * "natural": Will keep the default linkage. Will link if it would be normally. Example, SIM_FC00_CHUNK_1 -> DA_FC00_CHUNK_1. + * "all": Will link all selected tasks of the dependency with current selected tasks. Example, SIM_FC00_CHUNK_1 -> DA_FC00_CHUNK_1, DA_FC00_CHUNK_2, DA_FC00_CHUNK_3... + * "none": Will unlink selected tasks of the dependency with current selected tasks. + For the new format, consider that the priority is hierarchy and goes like this DATES_FROM -(includes)-> MEMBERS_FROM -(includes)-> CHUNKS_FROM. -- You can define a DATES_FROM inside the DEPENDENCY. -- You can define a MEMBERS_FROM inside the DEPENDENCY and DEPENDENCY.DATES_FROM. -- You can define a CHUNKS_FROM inside the DEPENDENCY, DEPENDENCY.DATES_FROM, DEPENDENCY.MEMBERS_FROM, DEPENDENCY.DATES_FROM.MEMBERS_FROM +* You can define a DATES_FROM inside the DEPENDENCY. +* You can define a MEMBERS_FROM inside the DEPENDENCY and DEPENDENCY.DATES_FROM. +* You can define a CHUNKS_FROM inside the DEPENDENCY, DEPENDENCY.DATES_FROM, DEPENDENCY.MEMBERS_FROM, DEPENDENCY.DATES_FROM.MEMBERS_FROM Job frequency ~~~~~~~~~~~~~ @@ -420,7 +427,7 @@ Example 1: How to select an specfic chunk PROCESSORS: 768 .. figure:: fig/select_chunks.png - :name: simple + :name: select_chunks :width: 100% :align: center :alt: select_chunks_workflow @@ -449,7 +456,7 @@ In this workflow you can see an illustrated example of SKIPPABLE parameter used #QUEUE: debug .. figure:: fig/skip.png - :name: simple + :name: skip :width: 100% :align: center :alt: skip_workflow @@ -483,7 +490,7 @@ Weak dependencies, work like this way: SYNCHRONIZE: member .. figure:: fig/dashed.png - :name: simple + :name: dashed :width: 100% :align: center :alt: dashed_workflow @@ -541,7 +548,7 @@ Jobs_conf: SYNCHRONIZE: member .. figure:: fig/select_members.png - :name: simple + :name: select_members :width: 100% :align: center :alt: select_members diff --git a/docs/source/userguide/index.rst b/docs/source/userguide/index.rst index bd1165b3afe70bb0bccc3d8818f218b9c53ac06c..83f8943dc70e9de641d6f50ea2b2c94e3a62c958 100644 --- a/docs/source/userguide/index.rst +++ b/docs/source/userguide/index.rst @@ -1,6 +1,6 @@ -##### -Usage -##### +########## +User Guide +########## .. toctree:: /userguide/create/index @@ -13,47 +13,50 @@ Usage Command list ============ --expid Create a new experiment --create Create specified experiment workflow --check Check configuration for specified experiment --describe Show details for specified experiments --run Run specified experiment --inspect Generate cmd files --test Test experiment --testcase Test case experiment --monitor Plot specified experiment --stats Plot statistics for specified experiment --setstatus Sets job status for an experiment --recovery Recover specified experiment --clean Clean specified experiment --refresh Refresh project directory for an experiment --delete Delete specified experiment --configure Configure database and path for autosubmit --install Install database for Autosubmit on the configured folder --archive Clean, compress and remove from the experiments' folder a finalized experiment --unarchive Restores an archived experiment --migrate_exp Migrates an experiment from one user to another --report extract experiment parameters --updateversion Updates the Autosubmit version of your experiment with the current version of the module you are using --dbfix Fixes the database malformed error in the historical database of your experiment --pklfix Fixed the blank pkl error of your experiment --updatedescrip Updates the description of your experiment (See: :ref:`updateDescrip`) +* expid Create a new experiment +* create Create specified experiment workflow +* check Check configuration for specified experiment +* describe Show details for specified experiments +* run Run specified experiment +* inspect Generate cmd files +* test Test experiment +* testcase Test case experiment +* monitor Plot specified experiment +* stats Plot statistics for specified experiment +* setstatus Sets job status for an experiment +* recovery Recover specified experiment +* clean Clean specified experiment +* refresh Refresh project directory for an experiment +* delete Delete specified experiment +* configure Configure database and path for autosubmit +* install Install database for Autosubmit on the configured folder +* archive Clean, compress and remove from the experiments' folder a finalized experiment +* unarchive Restores an archived experiment +* migrate_exp Migrates an experiment from one user to another +* report extract experiment parameters +* updateversion Updates the Autosubmit version of your experiment with the current version of the module you are using +* dbfix Fixes the database malformed error in the historical database of your experiment +* pklfix Fixed the blank pkl error of your experiment +* updatedescrip Updates the description of your experiment (See: :ref:`updateDescrip`) Tutorials (How to) ------------------ -* :ref:`new_experiment` +* :doc:`create/index` -* :ref:`configuration` +* :doc:`configure/index` * :ref:`run_modes` * :ref:`workflow_recovery` -* :ref:`workflow_validation` +TODO add ``workflow_validation``. -* :ref:`stats` +.. + * :ref:`workflow_validation` + +* :ref:`autoStatistics` * :ref:`archive` diff --git a/docs/source/userguide/manage/index.rst b/docs/source/userguide/manage/index.rst index c139933cf2b6714d7f67903eebb134d3db93b119..41b2cd6f3508e44fe0fb62994d03ced6b89d2747 100644 --- a/docs/source/userguide/manage/index.rst +++ b/docs/source/userguide/manage/index.rst @@ -46,6 +46,8 @@ A bare copy (which occupies less space on disk) will be automatically made. .. note:: In addition, every time you run this command with -pr option, it will check the commit unique identifier for local working tree existing on the ``proj`` directory. In case that commit identifier exists, clean will register it to the ``expdef_cxxx.yml`` file. +.. _archive: + How to archive an experiment ---------------------------- @@ -220,6 +222,8 @@ Example: autosubmit refresh cxxx +.. _updateDescrip: + How to update the description of your experiment ------------------------------------------------ @@ -260,6 +264,7 @@ You must execute: Options: :: + usage: autosubmit setstatus [-h] [-np] [-s] [-t] [-o {pdf,png,ps,svg}] [-fl] [-fc] [-fs] [-ft] [-group_by {date,member,chunk,split} -expand -expand_status] [-cw] expid expid experiment identifier @@ -325,6 +330,7 @@ Date (day) range example: Result: :: + cxxx_19601101_fc1_1_SIM cxxx_19601102_fc1_1_SIM cxxx_19601103_fc1_1_SIM diff --git a/docs/source/userguide/modifying_workflow/index.rst b/docs/source/userguide/modifying_workflow/index.rst index f45cd0ce3b7fd7dc2cdbdeb68e5dafea6b9a197b..b8598e47e262dd670e27906b00e9a8ffe1cdaa2e 100644 --- a/docs/source/userguide/modifying_workflow/index.rst +++ b/docs/source/userguide/modifying_workflow/index.rst @@ -1,3 +1,5 @@ +.. _workflow_recovery: + How to restart the experiment ============================= diff --git a/docs/source/userguide/monitor_and_check/index.rst b/docs/source/userguide/monitor_and_check/index.rst index 07fe7965ad12cfeb01c9be7f1b72b21b8dbed008..e28c110ca62c365f27c023721804a793e6982cc3 100644 --- a/docs/source/userguide/monitor_and_check/index.rst +++ b/docs/source/userguide/monitor_and_check/index.rst @@ -150,18 +150,6 @@ Without autosubmit.lock, it will generate all unless filtered by -fl,fc,fs or ft How to monitor an experiment ---------------------------- -.. toctree:: - :hidden: - :titlesonly: - - job_stats - custom_stats - describe - generate_cmd_files - visualize_groups - report - - To monitor the status of the experiment, use the command: :: @@ -338,11 +326,14 @@ Consider the following workflow: -group_by=chunk -.. figure:: fig/group_chunk.png - :name: group_chunk - :width: 70% - :align: center - :alt: group chunk +TODO: Add ``group_chunk.png`` figure. + +.. + figure:: fig/group_chunk.png + :name: group_chunk + :width: 70% + :align: center + :alt: group chunk Synchronize jobs @@ -464,6 +455,8 @@ Examples: autosubmit describe cxxx,cyyy autosubmit describe -u dbeltran +.. _autoStatistics: + How to monitor job statistics ----------------------------- @@ -570,6 +563,8 @@ The location where user can put this stats is in the file: .. hint:: If it is not yet created, you can manually create the file: ```expid_GENERAL_STATS``` inside the ```tmp``` folder. +.. _report: + How to extract information about the experiment parameters ---------------------------------------------------------- @@ -593,6 +588,7 @@ Or combined as follows: Options: :: + usage: autosubmit report [-all] [-t] [-fp] [-p] expid expid Experiment identifier @@ -605,12 +601,10 @@ Options: -p, --placeholders disable the replacement by - if the variable doesn't exist -Template format and example: -:: - Autosubmit parameters are encapsulated by %_%, once you know how the parameter is called you can create a template similar to the one as follows: .. code-block:: ini + :caption: Template format and example. - **CHUNKS:** %NUMCHUNKS% - %CHUNKSIZE% %CHUNKSIZEUNIT% - **VERSION:** %VERSION% @@ -634,12 +628,8 @@ Although it depends on the experiment. If the parameter doesn't exists, it will be returned as '-' while if the parameter is declared but empty it will remain empty -List of all parameters example: -:: - -On the other hand, if you use the option -l autosubmit will write a file called parameter_list_.txt containing all parameters in the format as follows: - .. code-block:: ini + :caption: List of all parameters example. HPCQUEUE: bsc_es HPCARCH: marenostrum4 diff --git a/docs/source/userguide/run/index.rst b/docs/source/userguide/run/index.rst index 5cb0ada3b4b626aa5ade5c3b7c0294321bc799c8..b03a1cdac3c558f398a03cb965ed66c2300ddc72 100644 --- a/docs/source/userguide/run/index.rst +++ b/docs/source/userguide/run/index.rst @@ -108,7 +108,7 @@ Once you've already loaded / installed the Autosubmit version do you want: *EXPID* is the experiment identifier. The most common problem when you upgrade an experiment with INI configuration to YAML is that some variables may be not automatically translated. -Ensure that all your $EXPID/conf/*.yml files are correct and also revise the templates in $EXPID/proj/$proj_name. +Ensure that all your $EXPID/conf/\*.yml files are correct and also revise the templates in $EXPID/proj/$proj_name. How to run only selected members @@ -187,6 +187,8 @@ This functionality can be used together with other options supplied by the `run` The `-sa` command has a long version `--start_after`. +.. _run_modes: + How to prepare an experiment to run in two independent job_list. (Priority jobs, Two-step-run) (OLD METHOD) ----------------------------------------------------------------------------------------------------------- @@ -247,7 +249,7 @@ The priority jobs will be ( check TWO_STEP_START from expdef conf): ['a02n_20120101_000_1_SIM', 'a02n_20120101_001_1_SIM', 'a02n_COMPILE_DA', 'a02n_20120101_000_1_REDUCE'] How to prepare an experiment to run in two independent job_list. (New method) ---------------------------------------------------------------------------- +----------------------------------------------------------------------------- From AS4, TWO_STEP_START is not longer needed since the users can now specify exactly which tasks of a job are needed to run the current task in the DEPENDENCIES parameter. @@ -410,4 +412,4 @@ To stop immediately experiment cxxx: kill -9 22835 .. important:: In case you want to restart the experiment, you must follow the - :ref:`restart` procedure, explained below, in order to properly resynchronize all completed jobs. \ No newline at end of file + :ref:`workflow_recovery` procedure, explained below, in order to properly resynchronize all completed jobs. \ No newline at end of file diff --git a/docs/source/userguide/set_and_share_the_configuration/index.rst b/docs/source/userguide/set_and_share_the_configuration/index.rst index db16d5a3a5cf2d17ca5ce05850adafb30cf446cd..b3e5a3d0b1c75b8e03920fd4b8cda74be7e68f89 100644 --- a/docs/source/userguide/set_and_share_the_configuration/index.rst +++ b/docs/source/userguide/set_and_share_the_configuration/index.rst @@ -101,6 +101,8 @@ The expid command can copy another user's existing expid to work correctly. .. warning:: You must share the same Autosubmit experiment database for this to work. +.. _advanced_features: + Advanced Configuration ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/source/devguide/variables.rst b/docs/source/userguide/variables.rst similarity index 100% rename from docs/source/devguide/variables.rst rename to docs/source/userguide/variables.rst diff --git a/docs/source/userguide/wrappers/index.rst b/docs/source/userguide/wrappers/index.rst index ecf21cafb20690ca7a3ffb568e500b314defd964..168e5afa8c7093ec5da4d3359946a9ce156c04a1 100644 --- a/docs/source/userguide/wrappers/index.rst +++ b/docs/source/userguide/wrappers/index.rst @@ -170,7 +170,7 @@ The queue parameter allows the users to define a path to a script that will load WRAPPER_0: TYPE: "horizontal" JOBS_IN_WRAPPER: "SIM" - EXPORT: %CURRENT_ROOTDIR%/envmodules.sh + EXPORT: "%CURRENT_ROOTDIR%/envmodules.sh"