diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d8729a49a43db129d7322213a27e843b552f1af7..f0f0a2895a8c4965919d7d6d8c4045ce8b0a805e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -39,14 +39,15 @@ test_python3: coverage_format: cobertura path: test/coverage.xml -docs: - stage: docs - script: - - conda activate autosubmit3 - - pip install -e . - - cd docs - - pip install -r requirements.txt - - make html +# FIXME: Our GitLab worker has 3.7, but pydata-sphinx-theme requires 3.8+ +# docs: +# stage: docs +# script: +# - conda activate autosubmit3 +# - pip install -e . +# - cd docs +# - pip install -r requirements.txt +# - make html clean: diff --git a/docs/requirements.txt b/docs/requirements.txt index 3044a10fac95d9d77073b0e6bea5c0aa86f2bb4d..5deaf0fcd539a53568bbb3fb0b143af9c5c8556a 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,6 @@ +livereload +pydata-sphinx-theme==0.15.* sphinx==5.* sphinx-autobuild==2021.3.* sphinx_rtd_theme sphinx-reredirects==0.1.* -livereload diff --git a/docs/source/_static/Logo.svg b/docs/source/_static/Logo.svg new file mode 100644 index 0000000000000000000000000000000000000000..a96ed4ffb595c2265bde058731e4e7cbab31b136 --- /dev/null +++ b/docs/source/_static/Logo.svg @@ -0,0 +1,117 @@ + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css new file mode 100644 index 0000000000000000000000000000000000000000..20cca0066ee0cd2d35490ab5dd9ecc2d3c4dc2cd --- /dev/null +++ b/docs/source/_static/custom.css @@ -0,0 +1,4 @@ +/* Hide home title */ +#autosubmit-workflow-manager > h1 { + display: none; +} \ No newline at end of file diff --git a/docs/source/_static/isometric.svg b/docs/source/_static/isometric.svg new file mode 100644 index 0000000000000000000000000000000000000000..357f0fa2edc75c722599197c7e895bc6096f5eb5 --- /dev/null +++ b/docs/source/_static/isometric.svg @@ -0,0 +1,1550 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/source/_templates/footer_notes.html b/docs/source/_templates/footer_notes.html new file mode 100644 index 0000000000000000000000000000000000000000..982384822253f757a99d252326867660f5d04044 --- /dev/null +++ b/docs/source/_templates/footer_notes.html @@ -0,0 +1 @@ +Illustrations by StorySet diff --git a/docs/source/agui/experiment/fig/fig_experiment.jpg b/docs/source/agui/experiment/fig/fig_experiment.jpg deleted file mode 100644 index e913e397d009c97035c2d2d2fad81c29a719b407..0000000000000000000000000000000000000000 Binary files a/docs/source/agui/experiment/fig/fig_experiment.jpg and /dev/null differ diff --git a/docs/source/agui/experiment/index.rst b/docs/source/agui/experiment/index.rst deleted file mode 100644 index a052e670b9c99ada1a3541986482440e53137233..0000000000000000000000000000000000000000 --- a/docs/source/agui/experiment/index.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. _experimentGUI: - -Experiment Information -====================== - -This component offers the main information about your experiment. - -.. figure:: fig/fig_experiment.jpg - :name: experiment_view - :width: 100% - :align: center - :alt: experiment_view - - Experiment Information - -At the top left you see the ``Autosubmit Searcher`` home link that will take you back to the :ref:`autosubmitGUIMP`, next to it you see the ``Home`` link that serves the same purpose, then you see the ``About`` link that takes you to a page with important information about the application (including the link to this documentation). Then you see the experiment name and ``status``, which is updated every 5 minutes. Next, you see the `run history` button, this button opens a panel that shows information about previous runs of the experiment, only works for experiments running the latest version of Autosubmit. Then, you see the ``esarchive status`` badge, it shows information about the current status of the `esarchive` file system. - -At the bottom you see some relevant metadata, including the ``branch`` of the ``model`` that was used in the experiment, the ``HPC name`` targeted by the experiment, the owner, the ``version`` that this experiment us running on, the ``DB`` version of Autosubmit, and the number of jobs in the experiment. - -On the center you see the :doc:`../tree/index`, which is loaded automatically when you open this page. \ No newline at end of file diff --git a/docs/source/agui/fig/fig1_gui.png b/docs/source/agui/fig/fig1_gui.png deleted file mode 100644 index d470a265d616dde269ff312cf64a9a07ae7db684..0000000000000000000000000000000000000000 Binary files a/docs/source/agui/fig/fig1_gui.png and /dev/null differ diff --git a/docs/source/agui/fig/fig2_gui.png b/docs/source/agui/fig/fig2_gui.png deleted file mode 100644 index 1bcdcaa0d77878f7b3833336cf246fe8d78ca992..0000000000000000000000000000000000000000 Binary files a/docs/source/agui/fig/fig2_gui.png and /dev/null differ diff --git a/docs/source/agui/fig/fig3_gui.png b/docs/source/agui/fig/fig3_gui.png deleted file mode 100644 index e05cda5e6ec2d88bb24fdd1b523fe5788c55209d..0000000000000000000000000000000000000000 Binary files a/docs/source/agui/fig/fig3_gui.png and /dev/null differ diff --git a/docs/source/agui/fig/fig4_gui.jpg b/docs/source/agui/fig/fig4_gui.jpg deleted file mode 100644 index 4af2ed202821b71c5c94e40db113a15c5a216d4a..0000000000000000000000000000000000000000 Binary files a/docs/source/agui/fig/fig4_gui.jpg and /dev/null differ diff --git a/docs/source/agui/fig/fig5_gui.jpg b/docs/source/agui/fig/fig5_gui.jpg deleted file mode 100644 index c398de0a0756ac33949710406797842d538e81aa..0000000000000000000000000000000000000000 Binary files a/docs/source/agui/fig/fig5_gui.jpg and /dev/null differ diff --git a/docs/source/agui/fig/fig_ev_1.png b/docs/source/agui/fig/fig_ev_1.png deleted file mode 100644 index 32d7c50b072d9a6ae3089bd733c8b9c67996eebd..0000000000000000000000000000000000000000 Binary files a/docs/source/agui/fig/fig_ev_1.png and /dev/null differ diff --git a/docs/source/agui/fig/fig_tree_1.png b/docs/source/agui/fig/fig_tree_1.png deleted file mode 100644 index d541558a5f5bc57a9b81c86bfae7cb191e95f4bd..0000000000000000000000000000000000000000 Binary files a/docs/source/agui/fig/fig_tree_1.png and /dev/null differ diff --git a/docs/source/agui/fig/fig_tree_2.png b/docs/source/agui/fig/fig_tree_2.png deleted file mode 100644 index 2abcb3befd8e88b8d73c49d15e21a9fb5ab2367d..0000000000000000000000000000000000000000 Binary files a/docs/source/agui/fig/fig_tree_2.png and /dev/null differ diff --git a/docs/source/agui/graph/fig/fig_graph_1.jpg b/docs/source/agui/graph/fig/fig_graph_1.jpg deleted file mode 100644 index 382bc8c8e339f100b6d28d7a2463737e4d098f4c..0000000000000000000000000000000000000000 Binary files a/docs/source/agui/graph/fig/fig_graph_1.jpg and /dev/null differ diff --git a/docs/source/agui/graph/fig/fig_graph_2.jpg b/docs/source/agui/graph/fig/fig_graph_2.jpg deleted file mode 100644 index f3aa2a158765aacd735782c6d9ca0ed18ac406d9..0000000000000000000000000000000000000000 Binary files a/docs/source/agui/graph/fig/fig_graph_2.jpg and /dev/null differ diff --git a/docs/source/agui/graph/fig/fig_graph_3.jpg b/docs/source/agui/graph/fig/fig_graph_3.jpg deleted file mode 100644 index 805c76419f455ae823a200bd395cc6fc58d288ca..0000000000000000000000000000000000000000 Binary files a/docs/source/agui/graph/fig/fig_graph_3.jpg and /dev/null differ diff --git a/docs/source/agui/graph/fig/fig_graph_4.jpg b/docs/source/agui/graph/fig/fig_graph_4.jpg deleted file mode 100644 index 65d079f3e7aff425e6eb6396d0333496a601f8b8..0000000000000000000000000000000000000000 Binary files a/docs/source/agui/graph/fig/fig_graph_4.jpg and /dev/null differ diff --git a/docs/source/agui/graph/index.rst b/docs/source/agui/graph/index.rst deleted file mode 100644 index 7c12c1dee8ad0e264fde6152ba843a91687ad18f..0000000000000000000000000000000000000000 --- a/docs/source/agui/graph/index.rst +++ /dev/null @@ -1,104 +0,0 @@ -Graph Representation -==================== - -The Graph Representation of the experiment offers a dependency oriented view. - -.. figure:: fig/fig_graph_2.jpg - :name: experiment_graph - :width: 100% - :align: center - :alt: Experiment Graph 1 - - Experiment Graph Representation - -This view offers a graph representation of the experiments where a node represents a job and an edge represents a directed dependency relationship between nodes. To open it you must click on the button ``Classic``, which is the basic representation that uses either ``GraphViz`` or an heuristic approach depending on experiment complexity; we explain the other options later. - -Once the graph representation is loaded, it will focus on a relevant node according to some established rules. The color of each node represents the status of the job it represents: you can see a color guide at the bottom of the page in the form of buttons. If you click in any of those buttons, the graph will focus on the last node with that status, except in the case of ``WAITING`` where the graph will focus on the first one. You can navigate the graph in this way, but there are other navigation buttons at the left and right corners of the graph canvas. You can also use your mouse or trackpad to navigate the graph, zoom in or zoom out. Below each node you can see the ``job name`` of the job it represents. - -.. important:: For some experiments you will get a well distributed and generally good looking graph representation, for others you get a more straightforward representation. It depends on the size and dependency complexity of your experiments, not all experiments can be modeled as a good looking graph in reasonable time. - -When you click on a node, you can see on the right panel (**Selection Panel**) the following information: - -- *Start*: Starting date. -- *End*: Ending date. -- *Section*: Also known as job type. -- *Member* -- *Chunk* -- *Platform*: Remote platform. -- *Id*: Id in the remote platform. -- *Processors*: Number of processors required by the job. -- *Wallclock*: Time requested by the job. -- *Queue*: Time spent in queue, in minutes. -- *Run*: Time spent running, in minutes. -- *Status*: Job status. -- *Out*: Button that opens a list of jobs that depend on the one selected. -- *In*: Button that opens a list of jobs on which the selected job depends. -- *out path*: Path to the .out log file. -- *err path*: Path to the .err log file. -- *Submit*: Submit time of the job (If applicable). -- *Start*: Start time of the job (If applicable). -- *Finish*: Finish time of the job (If applicable). - -.. important:: Next to the **out** and **err** paths, you see the a ``Copy out/err`` button that copies the path to your clipboard. Then you see an ``eye symbol`` button, that when clicked will show that last 150 lines of the **out/err** file. - -Selection ---------- - -When you click on a node in the tree view, a ``Change Status`` button will appear in the top bar, if you click, you will be presented with the option to generate a change status command that can be run on autosubmit, or to generate a format that can be used to change the status of the job while the experiment is running. - -You can select many nodes at the same time by maintaining ``CTRL`` pressed and clicking on the nodes, then the generated command will include all these jobs. - -Wrappers Representation ------------------------ - -Wrappers are an important feature of Autosubmit, and as such, it should be possible to visualize them in the graph representation. - -.. figure:: fig/fig_graph_3.jpg - :name: experiment_graph_wr - :width: 100% - :align: center - :alt: Experiment Graph Wrapper - - Wrapper Graph Representation - - -Wrappers are represented by nodes that have dashed border, hexagon or square shape (no difference between them), and that share green background edges. On the right side of the graph you can find the **Wrappers Tab** and it will display a list of the existing wrappers as buttons. If you click on any of these buttons, the nodes that belong to that wrapper will be highlighted. - -Monitoring ----------- - -If the experiment is ``RUNNING`` you will see at the top right corner the button ``Start Job Monitor``. When you click on it, a live **Job Monitor** will be initialized and the status of the jobs and wrappers will be queried every minute, any change will be updated in the graph. Also, if the **Job Monitor** is running, the detected changes will be listed in a panel **Monitor Panel** below the **Selection Panel**. You can stop this process by clicking on the button **Stop Job Monitor**. - -.. important:: While this is a good option to monitor the progress of your experiment, you can also use the :ref:`log`. - -Job Search ----------- - -.. figure:: fig/fig_graph_4.jpg - :name: experiment_graph_search - :width: 100% - :align: center - :alt: Job Search - - Job Search in Graph - -On top of the graph you will see an input text box following by the button ``Search by Job Name``. Insert into that box the string that you want to find and the engine will build an internal list of those jobs whose name coincides with that string. For example ``_LOCAL_`` will show only jobs whose title contain the that string. Buttons ``Previous`` and ``Next`` will appear and next to them the number of jobs that coincide with your search, you can use these buttons to traverse the graph highlighting the nodes included in the resulting internal list. - -Grouped by Date Member ----------------------- - -By clicking on the button ``Grouped by D-M`` you get a graph representation where the nodes are clustered by date and member. For example, if your experiment has only one starting date and one member, then you will have only one cluster in this view. These clusters are represented by rectangular boxes whose color gives a general idea of the status of the jobs inside it. - -.. important:: You can double click on any cluster to "open" it, meaning that the nodes that belong to that cluster will be freed and positioned individually. - -Grouped by Status ------------------ - -By clicking on the button ``Grouped by Status`` you get a graph representation where the nodes are clustered by status into 3 clusters: ``WAITING``, ``COMPLETED``, and ``SUSPENDED``. Same rules mentioned for **Grouped by Date Member** apply. - -Laplacian ---------- - -By clicking on the button ``Laplacian`` you get a graph representation where the ``(x,y)`` coordinates of each node are calculated based on the second and third smallest eigenvector of the Graph Laplacian. All functionality is supported. - - diff --git a/docs/source/agui/index.rst b/docs/source/agui/index.rst deleted file mode 100644 index a44d7e8aa8edfe2c841c5861dfdb6b49b5209f23..0000000000000000000000000000000000000000 --- a/docs/source/agui/index.rst +++ /dev/null @@ -1,93 +0,0 @@ -:orphan: - -Autosubmit GUI -############## - -.. toctree:: - :hidden: - - experiment/index - tree/index - graph/index - log/index - performance/index - statistics/index - -.. _autosubmitGUIMP: - -Autosubmit GUI Main Page -======================== - -Inside the Barcelona Supercomputing Internal Network you can find the latest version of Autosubmit GUI deployed for BSC users. It can be accessed by following the url http://bscesweb04.bsc.es/autosubmitapp/ or https://earth.bsc.es/autosubmitapp/. This is a graphic user interface that allows you to easily monitor your experiments and those of your colleagues. This Web App introduces many useful features for experiment monitoring, and we are continuously improving it. - -.. note:: The Web App can also be accessed through the VPN Client provided by BSC. - -When you enter the site, you will be presented with the following page: - -.. figure:: fig/fig1_gui.png - :name: first_page - :width: 100% - :align: center - :alt: autosubmit guide - - Welcome page - -Here you can search for any ongoing or past experiment by typing some text in the Search input box and pressing **Search**: the search engine will look for coincidences between your input string and any of the description, owner or name of the experiment fields. The results will be shown below ordered by status, experiments ``RUNNING`` will be shown in the first rows. You can also click on the ``Running`` button, and all the experiments that are currently running will be listed. The results will look like: - -.. figure:: fig/fig2_gui.png - :name: first_page_search - :width: 100% - :align: center - :alt: result search - - Search Result - -If you click on ``Show Detailed Data``, summary data for each experiment (result) will be loaded. These are data details from the experiment run, useful to see its status at a glance. Progress bars and status will use different colors to highlight the important information. - -.. figure:: fig/fig3_gui.png - :name: first_page_search_plus - :width: 100% - :align: center - :alt: result search plus - - Search Result plus Detailed Data - -For each experiment, you see the following data: - -.. figure:: fig/fig4_gui.jpg - :name: first_page_search_plus_description - :width: 100% - :align: center - :alt: result search plus description - - Description of Detailed Data - -1. *Experiment Name* -2. *Progress Bar*: Shows completed jobs / total jobs. It turns red when there are **failed** jobs in the experiment, but **Show Detailed Data** should have been requested. -3. *Experiment Status*: *RUNNING* or *NOT RUNNING*. -4. *Owner* -5. *Experiment Description* -6. *Refresh button*: It will say *Summary* when the detailed data has not been requested. If it says *Summary* and you click on it, it will load detailed data for that experiment, otherwise it will refresh the existing detailed data. -7. *More button*: Opens the **Experiment Page**. -8. *Average Queue Time* for all jobs. -9. *Average Run Time* for all jobs. -10. *Number of Running Jobs* -11. *Number of Queuing Jobs* -12. *Number of Submitted Jobs* -13. *Number of Suspended Jobs* -14. *Number of Failed Jobs*: If there are Failed jobs, a list of the names of those jobs will be displayed. - -.. figure:: fig/fig5_gui.jpg - :name: first_page_search_plus_description_sim - :width: 100% - :align: center - :alt: result search plus description + sim - - Average Times Feature - -In experiments that include ``SIM`` jobs, you will also see the average queuing and running time for these jobs. In the latest version the time format has been updated to ``HH:mm:ss``. The text for the ``SIM`` average follows the format ``avg. queue HH:mm:ss (M) | run HH:mm:ss (N)`` where ``M`` is the number of jobs considered for the ``avg. queue`` calculation and ``N`` is the number of jobs considered for ``run`` calculation. - -After clicking on the **MORE** button, you will be presented with the **Experiment Page**, which is the main view that Autosubmit provides. These are its main components: - - -.. important:: To improve response times, Autosubmit GUI will try to store the dependency structure of your experiment in a database filed called ``structure_expid.db`` where ``expid`` is the name of your experiment. This file will be located in ``/esarchive/autosubmit/expid/pkl/``. \ No newline at end of file diff --git a/docs/source/agui/log/fig/fig_log_1.jpg b/docs/source/agui/log/fig/fig_log_1.jpg deleted file mode 100644 index 9d774be9dcf9baac9bb8e3294dcfebd9a98cb7bf..0000000000000000000000000000000000000000 Binary files a/docs/source/agui/log/fig/fig_log_1.jpg and /dev/null differ diff --git a/docs/source/agui/log/fig/fig_log_2.jpg b/docs/source/agui/log/fig/fig_log_2.jpg deleted file mode 100644 index 5b5cad3a7f5b71273c377bd682f060273ca72a43..0000000000000000000000000000000000000000 Binary files a/docs/source/agui/log/fig/fig_log_2.jpg and /dev/null differ diff --git a/docs/source/agui/log/index.rst b/docs/source/agui/log/index.rst deleted file mode 100644 index 9b3cc9a886afff145741a6a432a923d0f332ed4e..0000000000000000000000000000000000000000 --- a/docs/source/agui/log/index.rst +++ /dev/null @@ -1,34 +0,0 @@ -.. _log: - -Autosubmit Log -============== - -When you click on the ``Log`` tab, you will see the button ``Show Log``: - -.. figure:: fig/fig_log_1.jpg - :name: experiment_log - :width: 100% - :align: center - :alt: Experiment Log 1 - - Experiment Log - -.. important:: The main Autosubmit log is usually stored in the folder ``/tmp/`` of your experiment, and this is the first path the system will scan. - -When you click on the ``Show Log`` button, the last 150 lines of the log will be displayed: - -.. figure:: fig/fig_log_2.jpg - :name: experiment_log_open - :width: 100% - :align: center - :alt: Experiment Log 2 - - Experiment Log Open - - -At the top of the log you will see the name of the log file that is being displayed along with the timestamp of the last time the log was requested, and to the right you see this timestamp in ``datetime`` format. - -If the experiment is currently running, the log will be updated periodically to keep you up with recent updates in the experiment execution. It is possible to scroll this view. - -If you click on ``Hide Log`` the log will be cleared and the periodic updates will stop. - diff --git a/docs/source/agui/performance/fig/fig_performance_1.jpg b/docs/source/agui/performance/fig/fig_performance_1.jpg deleted file mode 100644 index db2e7188ae2f165b9b41667e1e5b83d4598510e2..0000000000000000000000000000000000000000 Binary files a/docs/source/agui/performance/fig/fig_performance_1.jpg and /dev/null differ diff --git a/docs/source/agui/performance/index.rst b/docs/source/agui/performance/index.rst deleted file mode 100644 index 906e40733a33e43207edb907f4fb701ad64d534a..0000000000000000000000000000000000000000 --- a/docs/source/agui/performance/index.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. _performance: - -Performance Metrics -=================== - -The Performance Metrics tabs offers a set of metrics that try to measure the efficiency of the simulation performed, and other aspects of it. - -.. figure:: fig/fig_performance_1.jpg - :name: experiment_performance - :width: 100% - :align: center - :alt: Performance Metrics 1 - - Performance Metrics Tab - -On the left you have the values of the main performance metrics that apply to the experiment. Then, on the right, you see the list of jobs considered for this calculation with their data, also, ``SYPD`` and ``ASYPD`` are calculated individually for these jobs. This list is scrollable. - -You can also access a ``Show warnings`` button that opens a list of important information that might affect the calculation of the metrics. You can click again on this button to close the list. - -Further information about the metrics is included in the tab. \ No newline at end of file diff --git a/docs/source/agui/statistics/fig/fig_stat_1.jpg b/docs/source/agui/statistics/fig/fig_stat_1.jpg deleted file mode 100644 index ac225308ebf08b5a315d56a878f8830ad3a2cb6f..0000000000000000000000000000000000000000 Binary files a/docs/source/agui/statistics/fig/fig_stat_1.jpg and /dev/null differ diff --git a/docs/source/agui/statistics/fig/fig_stat_2.jpg b/docs/source/agui/statistics/fig/fig_stat_2.jpg deleted file mode 100644 index f5e5407d74c098e8daaafeb906c8231fde2e4ce3..0000000000000000000000000000000000000000 Binary files a/docs/source/agui/statistics/fig/fig_stat_2.jpg and /dev/null differ diff --git a/docs/source/agui/statistics/index.rst b/docs/source/agui/statistics/index.rst deleted file mode 100644 index 7321f125d4838a21a4172e227c3d932fa3a269ad..0000000000000000000000000000000000000000 --- a/docs/source/agui/statistics/index.rst +++ /dev/null @@ -1,31 +0,0 @@ -.. _statistics: - -Autosubmit Statistics -===================== - - -When you click on the ``Statistics`` tab, you will see two input boxes: ``Section`` and ``Hours``, followed by the button ``Get Statistics``: - -.. figure:: fig/fig_stat_1.jpg - :name: experiment_stats - :width: 100% - :align: center - :alt: Experiment Stat 1 - - Experiment Statistics - -There is also a brief explanation of the input fields and expected result. Basically, ``Section`` allows you to narrow your search to certain job types, and ``Hours`` allows you to set a time limit to look into the past in hours. - -In this example we have queried 3 hours into the past: - -.. figure:: fig/fig_stat_2.jpg - :name: experiment_stats_open - :width: 100% - :align: center - :alt: Experiment Stat 2 - - Experiment Statistics (Last 3 hours) - -Click on the button ``Clear Statistics`` to clear the results and submit another query. - -.. important:: For more details about Autosubmit statistics, refer to: :ref:`autoStatistics`. \ No newline at end of file diff --git a/docs/source/agui/tree/fig/fig_tree_2.jpg b/docs/source/agui/tree/fig/fig_tree_2.jpg deleted file mode 100644 index 3876687f0862a0f1069c00e1b492de6f83146e7e..0000000000000000000000000000000000000000 Binary files a/docs/source/agui/tree/fig/fig_tree_2.jpg and /dev/null differ diff --git a/docs/source/agui/tree/index.rst b/docs/source/agui/tree/index.rst deleted file mode 100644 index ff0e16f5dec79a43b70a375af721cebbb21fb2cd..0000000000000000000000000000000000000000 --- a/docs/source/agui/tree/index.rst +++ /dev/null @@ -1,77 +0,0 @@ -Tree Representation -=================== - -The Tree Representation offers a structured view of the experiment. - -.. figure:: fig/fig_tree_2.jpg - :name: experiment_tree - :width: 100% - :align: center - :alt: Experiment Tree 1 - - Experiment Tree Representation - -The view is organized in groups by ``date``, and ``date-member``. Each group has a folder icon, and next to the icon you can find the progress of the group as ``completed / total`` jobs (when all the jobs in a group have been completed, a check symbol will appear); then, an indicator of how many jobs inside that group are **RUNNING**, **QUEUING**, or have **FAILED**. Furthermore, if wrappers exist in the experiment, independent groups will be added for each wrapper that will contain the list of jobs included in the corresponding wrapper. This implies that a job can be repeated: once inside its ``date-member`` group and once in its wrapper group. - -Inside each group you will find the list of jobs that belong to that group. The jobs are shown following this format: *job name* + # *job status* + ( + *queuing time* + ) + *running time*. Jobs that belong to a wrapper have also a badge with the code of the wrapper. - -When you click on a Job, you can see on the right panel (**Selection Panel**) the following information: - -- *Start*: Starting date. -- *End*: Ending date. -- *Section*: Also known as job type. -- *Member* -- *Chunk* -- *Platform*: Remote platform. -- *Id*: Id in the remote platform. -- *Processors*: Number of processors required by the job. -- *Wallclock*: Time requested by the job. -- *Queue*: Time spent in queue, in minutes. -- *Run*: Time spent running, in minutes. -- *Status*: Job status. -- *Out*: Button that opens a list of jobs that depend on the one selected. -- *In*: Button that opens a list of jobs on which the selected job depends. -- *out path*: Path to the .out log file. -- *err path*: Path to the .err log file. -- *Submit*: Submit time of the job (If applicable). -- *Start*: Start time of the job (If applicable). -- *Finish*: Finish time of the job (If applicable). - -.. important:: Next to the **out** and **err** paths, you see the a ``Copy out/err`` button that copies the path to your clipboard. Then you see an ``eye symbol`` button, that when clicked will show that last 150 lines of the **out/err** file. - -Selection ---------- - -When you click on a job in the tree view, a ``Change Status`` button will appear in the top bar, if you click, you will be presented with the option to generate a change status command that can be run on autosubmit, or to generate a format that can be used to change the status of the job while the experiment is running. - -You can select many jobs at the same time by maintaining ``CTRL`` pressed and clicking on the jobs, then the generated command will include all these jobs. - -Monitoring ----------- - -If the experiment status is **RUNNING**, you will see a button called **Refresh** at the top right corner. This button will update the information of the jobs in the tree if necessary. Next to this button, you will see the button **Start Job Monitor**. When you click on it, a live **Job Monitor** will be initialized and the status of the jobs and wrappers will be queried every minute, any change will be updated in the **Tree View**. Also, if the **Job Monitor** is running, the detected changes will be listed in a panel **Monitor Panel** below the **Selection Panel**. You can stop this process by clicking on the button **Stop Job Monitor**. - -The button **Clear Tree View** will clear the Tree Representation. It is also a valid way to refresh the Tree View. - -Filter ------- - -At the top left you can find the **Filter text** input box. Insert any string and the list will show only those jobs whose description coincides with that string. For example ``#COMPLETED`` will show only completed jobs, ``Wrapped`` will show only those jobs that belong to a wrapper, ``_fc0_`` will show only those jobs that belong to the fc0 member. Press **Clear** to reset the filter. On the right side of this bar, you will see the total number of jobs, and the chunk unit used in the experiment. - -Advanced Filter ---------------- - -It is possible to use the key char ``*`` to separate keywords in the name of the job, in order. For example: - -- ``1850*fc0*_1_``: List all the jobs that have the string ``1850`` and then at least 1 occurrence of the string ``fc0`` and then at least 1 occurrence of the string ``_1_``. This will effectively list all the jobs for the DATE that starts with ``1850`` for the member ``fc0`` and the chunk ``_1_``. -- ``000*_5``: List all the jobs that have the string ``000`` followed by at least one occurrence of the string ``_5``. This will effectively list all the jobs that have member ``000`` and chunk number that starts with the digit ``5``. -- ``000*_5*PREPROCVAR``: It will also add the filter for jobs of type ``PREPROCVAR``. - -As you might infer, the logic is fairly straightforward: Start your string with the word or part of the word you are looking for, then add ``*`` and the word or part of the word that follows, and so on. The algorithm will split your string by ``*`` and then search for each part in order, once it finds the part in the title of the job, it takes a substring of the job title to not repeat the next search in the same string, it continues looking for the next part in the new reduced string, and so on. - -You can extend this functionality considering that date, member, section, chunk names start with the symbol ``_`` and finish with the same symbol. - -.. important:: This view is designed to show a structured view of your experiment, if you want a more dependency oriented view that shows better the execution sequence of your jobs, you can refer to :doc:`../graph/index`. - - - diff --git a/docs/source/conf.py b/docs/source/conf.py index 317a9fe30872b4cee9385d8de8ef59e6b0ed2a90..e9c6c69d9caf0375df94609da64a7b332fabfa4f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -65,8 +65,8 @@ master_doc = 'index' # General information about the project. project = 'autosubmit' # noinspection PyShadowingBuiltins -copyright = u'2023, Barcelona Supercomputing Center, BSC' -author = u'Earth Science Department, Barcelona Supercomputing Center, BSC' +copyright = u'2024, Barcelona Supercomputing Center, BSC' +author = u'Earth Sciences Department, Barcelona Supercomputing Center, BSC' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -134,12 +134,41 @@ redirects = { # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'sphinx_rtd_theme' +html_theme = 'pydata_sphinx_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -# html_theme_options = {} +html_theme_options = { + 'header_links_before_dropdown': 6, + 'show_nav_level': 2, + 'use_edit_page_button': True, + 'icon_links': [ + { + 'name': 'GitLab', + 'url': 'https://earth.bsc.es/gitlab/es/autosubmit/', + 'icon': 'fa-brands fa-square-gitlab', + 'type': 'fontawesome' + }, + # TODO: https://github.com/pydata/pydata-sphinx-theme/blob/662758e6afb498be269fd123ba6e446a8099534a/docs/user_guide/header-links.rst#L252 + # { + # 'name': 'PyPI', + # 'url': 'https://pypi.org/project/autosubmit/', + # 'icon': 'fa-custom fa-pypi', + # 'type': 'fontawesome' + # } + ], + 'footer_start': ['copyright', 'sphinx-version'], + 'footer_end': ['theme-version', 'footer_notes'] +} + +html_context = { + 'gitlab_url': 'https://earth.bsc.es/gitlab', + 'gitlab_user': 'es', + 'gitlab_repo': 'autosubmit', + 'gitlab_version': 'master', + 'doc_path': 'docs/source/' +} # Add any paths that contain custom themes here, relative to this directory. # html_theme_path = [] @@ -153,7 +182,7 @@ html_theme = 'sphinx_rtd_theme' # The name of an image file (relative to this directory) to place at the top # of the sidebar. -# html_logo = None +html_logo = '_static/Logo.svg' # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 @@ -168,7 +197,8 @@ html_static_path = ['_static'] # These paths are either relative to html_static_path # or fully qualified paths (eg. https://...) html_css_files = [ - 'css/autosubmit.css' + 'css/autosubmit.css', + 'custom.css' ] # Add any extra paths that contain custom files (such as robots.txt or diff --git a/docs/source/database/index.rst b/docs/source/database/index.rst index fa2c22146aab97427fb7932c0e651d5dc8cf8ba9..1562428e5c2600589c93a280c7feac51312a96d6 100644 --- a/docs/source/database/index.rst +++ b/docs/source/database/index.rst @@ -1,6 +1,6 @@ -#################### -Autosubmit databases -#################### +######### +Databases +######### Introduction ------------ diff --git a/docs/source/devguide/index.rst b/docs/source/devguide/index.rst index a086bb867e356d35e90bb10d0d6edf5babb3500f..bae2d39fb3b5103c5d01d39125585d5635a93e0d 100644 --- a/docs/source/devguide/index.rst +++ b/docs/source/devguide/index.rst @@ -1,3 +1,5 @@ +:orphan: + ############################## Developing an EC-Earth Project ############################## diff --git a/docs/source/index.rst b/docs/source/index.rst index c269937033a08b9c9b290d6267e1c8ec24b9843c..bdc2e5f060c96f94dfce634d10a5322d322ce6d4 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,20 +1,16 @@ +:html_theme.sidebar_secondary.remove: + .. autosubmit documentation master file, created by sphinx-quickstart on Wed Mar 18 16:55:44 2015. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -###################################### -Welcome to autosubmit's documentation! -###################################### +############################ +Autosubmit Workflow Manager +############################ .. toctree:: - :maxdepth: 1 - :hidden: - - /introduction/index - -.. toctree:: - :caption: Quick Start Guide + :caption: Getting Started :maxdepth: 1 :hidden: @@ -29,22 +25,10 @@ Welcome to autosubmit's documentation! .. toctree:: :caption: User Guide - :maxdepth: 2 + :maxdepth: 1 :hidden: /userguide/index - /userguide/create/index - /userguide/configure/index - /userguide/defining_workflows/index - /userguide/wrappers/index - /userguide/run/index - /userguide/modifying_workflow/index - /userguide/manage/index - /userguide/monitor_and_check/index - /userguide/set_and_share_the_configuration/index - /userguide/variables - /userguide/expids - /userguide/provenance .. toctree:: :caption: Database Documentation @@ -53,21 +37,12 @@ Welcome to autosubmit's documentation! /database/index -.. toctree:: - :caption: Developer Guide - :maxdepth: 1 - :hidden: - - /devguide/index - .. toctree:: :caption: Troubleshooting :maxdepth: 1 :hidden: /troubleshooting/index - /troubleshooting/error-codes - /troubleshooting/changelog .. toctree:: :caption: Module Documentation @@ -77,19 +52,107 @@ Welcome to autosubmit's documentation! /moduledoc/index -Autosubmit is a Python software to manage complicated workflows on HPC platforms. - -Automatization - Autosubmit manages job submission and dependencies without user intervention -Data Provenance. - Autosubmit assigns unique ID's to experiments, uses open standards, and - applies other techniques to enable :doc:`data provenance ` - in the experiments and workflows. -Failure Tolerance - Autosubmit manages automatic retrials and has the ability to rerun specific parts of - the experiment in case of failure -Resource Management - Autosubmit supports a per-platform configuration, allowing users to run their experiments - without adapting job scripts. -Multiple Platform - Autosubmit can run jobs of an experiment in different platforms \ No newline at end of file +.. raw:: html + +
+
+
+

Autosubmit

+

+ Autosubmit is an open source Python experiment and workflow + manager used to manage complex workflows on Cloud and HPC + platforms. +

+
$ pip install autosubmit
+
+ Get started + Installation +
+
+
+ Illustration of a person and workflows running on a platform. +
+
+
+ + +Autosubmit is a lightweight workflow manager designed to meet climate research +necessities. Unlike other workflow solutions in the domain, it integrates the +capabilities of an experiment manager, workflow orchestrator and monitor in a +self-contained application. + +It is a Python package available at PyPI. The source code in Git contains a +Dockerfile used in cloud environments with Kubernetes, and there are examples +of how to install Autosubmit with Conda. + + +.. raw:: html + +
+
+
+

+ + Automation +

+

Management of job submission and dependencies without user intervention.

+
+
+

+ + Data Provenance +

+

Experiments with unique PIDs, use of open standards for data provenance + in the experiments and workflows.

+
+
+

+ + Fault Tolerance +

+

Automatic retrials and ability to re-run specific parts of + the experiment in case of failure.

+
+
+

+ + Resource Management +

+

Individual platform configuration, allowing users + to run their experiments without having to modify job scripts.

+
+
+

+ + Multiplatform +

+

Widely used to run experiments on different platforms simultaneously, using batch schedulers such as Slurm, PBS, LSF. It is deployed and used on various HPC and cloud systems.

+
+
+

+ + Open Source +

+

Autosubmit code is hosted at BSC Earth Sciences' GitLab, licensed + under the GPLv3 License, and under active development.

+
+
+
+ +Contact Us +========== + +.. list-table:: + :widths: 20 80 + :header-rows: 0 + :stub-columns: 1 + + * - GitLab + - https://earth.bsc.es/gitlab/es/autosubmit/ + * - Email + - support-autosubmit@bsc.es + diff --git a/docs/source/introduction/fig1.png b/docs/source/introduction/fig1.png deleted file mode 100644 index 92bb18e11b294d6c5efe75d0a1dbf267718a3919..0000000000000000000000000000000000000000 Binary files a/docs/source/introduction/fig1.png and /dev/null differ diff --git a/docs/source/introduction/fig2.png b/docs/source/introduction/fig2.png deleted file mode 100644 index e5af6bf33ab9fabcc463eccd20c791c50bf4bb5d..0000000000000000000000000000000000000000 Binary files a/docs/source/introduction/fig2.png and /dev/null differ diff --git a/docs/source/introduction/fig3.png b/docs/source/introduction/fig3.png deleted file mode 100644 index 9a330fd50118a8f278cf8af115aea6448ee7df83..0000000000000000000000000000000000000000 Binary files a/docs/source/introduction/fig3.png and /dev/null differ diff --git a/docs/source/introduction/index.rst b/docs/source/introduction/index.rst deleted file mode 100644 index 63c46ff5bd8c62ff6c8f09fce6ecd4509b6d7757..0000000000000000000000000000000000000000 --- a/docs/source/introduction/index.rst +++ /dev/null @@ -1,131 +0,0 @@ -############ -Introduction -############ - -What is Autosubmit ? -==================== - -Autosubmit is a lightweight workflow manager designed to meet climate research necessities. Unlike other workflow solutions in the domain, it integrates the capabilities of an experiment manager, workflow orchestrator and monitor in a self-contained application. The experiment manager allows for defining and configuring experiments, supported by a hierarchical database that ensures reproducibility and traceability. The orchestrator is designed to run complex workflows in research and operational mode by managing their dependencies and interfacing with local and remote hosts. These multi-scale workflows can involve from a few to thousands of steps and from one to multiple platforms. - -Autosubmit facilitates easy and fast integration and relocation on new platforms. On the one hand, users can rapidly execute general scripts and progressively parametrize them by reading Autosubmit variables. On the other hand, it is a self-contained desktop application capable of submitting jobs to remote platforms without any external deployment. - -Due to its robustness, it can handle different eventualities, such as networking or I/O errors. Finally, the monitoring capabilities extend beyond the desktop application through a REST API that allows communication with workflow monitoring tools such as the Autosubmit web GUI. - -Autosubmit is a Python package provided in PyPI. Conda recipes can also be found on the website. A containerized version for testing purposes is also available but not public yet. - -It has contributed to various European research projects and runs different operational systems. During the following years, it will support some of the Earth Digital Twins as the Digital Twin Ocean. - -Concretely, it is currently used at Barcelona Supercomputing Centre (BSC) to run models (EC-Earth, MONARCH, NEMO, CALIOPE, HERMES…), operational toolchains (S2S4E), data-download workflows (ECMWF MARS), and many other. Autosubmit has run these workflows in different supercomputers in BSC, ECMWF, IC3, CESGA, EPCC, PDC, and OLCF. - -Get involved or contact us: - -+----------------------------+-------------------------------------------+ -| GitLab: | https://earth.bsc.es/gitlab/es/autosubmit | -+----------------------------+-------------------------------------------+ -| Mail: | support-autosubmit@bsc.es | -+----------------------------+-------------------------------------------+ - -Why is Autosubmit needed ? -========================== - -Autosubmit is the only existing tool that satisfies the following requirements from the weather and climate community: - -- **Automatization** Job submission to machines and dependencies between - jobs are managed by Autosubmit. No user intervention is needed. -- **Data provenance** Assigns unique identifiers for each experiment - and stores information about model version, experiment configuration - and computing facilities used in the whole process. Read more in - the user guide section about :doc:`/userguide/provenance`. -- **Failure tolerance** Automatic retrials and ability to rerun chunks - in case of corrupted or missing data. -- **Resource management** Autosubmit manages supercomputer particularities, - allowing users to run their experiments in the available machine without - having to adapt the code. Autosubmit also allows to submit tasks from - the same experiment to different platforms. - -.. _RO-Crate: https://w3id.org/ro/crate - -How does Autosubmit work ? -========================== - -You can find help about how to use autosubmit and a list of available commands, just executing: -:: - - autosubmit -h - -Execute autosubmit -h for detailed help for each command: -:: - - autosubmit expid -h - -Experiment creation -------------------- - -To create a new experiment, run the command: -:: - - autosubmit expid -H "HPCname" -d "Description" - -*HPCname* is the name of the main HPC platform for the experiment: it will be the default platform for the tasks. -*Description* is a brief experiment description. - -This command assigns to the experiment a unique four alphanumerical characters identifier, where the first has reserved letters *a* and -*t*. It then creates a new folder in experiments repository with structure shown in Figure :numref:`exp_folder`. - - -.. figure:: fig1.png - :name: exp_folder - :width: 33% - :align: center - :alt: experiment folder - - Example of an experiment directory tree. - -Experiment configuration ------------------------- - -To configure the experiment, edit ``expdef_xxxx.yml``, ``jobs_xxxx.yml`` and ``platforms_xxxx.yml`` in the ``conf`` folder of the experiment (see contents in Figure :numref:`exp_config`). - -.. figure:: fig2.png - :name: exp_config - :width: 50% - :align: center - :alt: configuration files - - Configuration files content - -After that, you are expected to run the command: -:: - - autosubmit create xxxx - -This command creates the experiment project in the ``proj`` folder. The experiment project contains the scripts specified in ``jobs_xxxx.yml`` and a copy of model source code and data specified in ``expdef_xxxx.yml``. - -Experiment run --------------- - -To run the experiment, just execute the command: - - .. code-block:: bash - - # Add your key to ssh agent ( if encrypted ) - ssh-add ~/.ssh/id_rsa - autosubmit run a000 - -Autosubmit will start submitting jobs to the relevant platforms (both HPC and supporting computers) by using the scripts specified in ``jobs_xxxx.yml``. Autosubmit will substitute variables present on scripts where handlers appear in *%variable_name%* format. Autosubmit provides variables for *current chunk*, *start date*, *member*, *computer configuration* and more, and also will replace variables form ``proj_xxxx.yml``. - -To monitor the status of the experiment, issue the command: - -:: - - autosubmit monitor xxxx - -This will plot the workflow of the experiment and the current status. - -.. figure:: fig3.png - :width: 70% - :align: center - :alt: experiment plot - - Example of monitoring plot for EC-Earth run with Autosubmit for 1 start date, 1 member and 3 chunks. - diff --git a/docs/source/moduledoc/index.rst b/docs/source/moduledoc/index.rst index 024ec6774186c619435423f3d54b41341fe04a5f..3a269c23d9c68874bacb62b243f7d480e023235d 100644 --- a/docs/source/moduledoc/index.rst +++ b/docs/source/moduledoc/index.rst @@ -1,6 +1,6 @@ -******* -Modules -******* +*** +API +*** .. toctree:: :titlesonly: diff --git a/docs/source/qstartguide/index.rst b/docs/source/qstartguide/index.rst index c99973882114adfeb36996dfb397bb494e342c83..9ec8ad034f100eb03374541992727b7e72b39eba 100644 --- a/docs/source/qstartguide/index.rst +++ b/docs/source/qstartguide/index.rst @@ -1,6 +1,6 @@ -==================== -Tutorial start guide -==================== +=============== +Getting Started +=============== This tutorial is a starter’s guide to run a dummy experiment with Autosubmit. diff --git a/docs/source/troubleshooting/error-codes.rst b/docs/source/troubleshooting/error-codes.rst index 0e6f12fe8356e52f6f99df2e1f92d881499874ab..5d15cc3e77ee8a1214e20ae99c289c2641255786 100644 --- a/docs/source/troubleshooting/error-codes.rst +++ b/docs/source/troubleshooting/error-codes.rst @@ -5,7 +5,7 @@ Error codes and solutions .. note:: Increasing the logging level gives you more detailed information about your error, e.g. ``autosubmit -lc DEBUG -lf DEBUG ``, - where ``>`` could be ``create``, ``run``, etc. + where ```` could be ``create``, ``run``, etc. Every error in Autosubmit contains a numeric error code, to help users and developers to identify the category of the error. These errors are organized as follows: diff --git a/docs/source/troubleshooting/index.rst b/docs/source/troubleshooting/index.rst index 8be0082119fc69e1e0f8799636af416484dbe954..60ed9826c0d4b4f67f54d67a00fa7f659833ae45 100644 --- a/docs/source/troubleshooting/index.rst +++ b/docs/source/troubleshooting/index.rst @@ -1,6 +1,13 @@ -### -FAQ -### +.. toctree:: + :caption: Troubleshooting + :maxdepth: 1 + + /troubleshooting/error-codes + /troubleshooting/changelog + +############### +Troubleshooting +############### How to change the job status stopping autosubmit ================================================ diff --git a/docs/source/userguide/index.rst b/docs/source/userguide/index.rst index 83f8943dc70e9de641d6f50ea2b2c94e3a62c958..f1ec91c9f17b4f7af30e88f337ab27010a50adc7 100644 --- a/docs/source/userguide/index.rst +++ b/docs/source/userguide/index.rst @@ -5,10 +5,16 @@ User Guide .. toctree:: /userguide/create/index /userguide/configure/index + /userguide/defining_workflows/index + /userguide/wrappers/index /userguide/run/index + /userguide/modifying_workflow/index /userguide/manage/index /userguide/monitor_and_check/index /userguide/set_and_share_the_configuration/index + /userguide/variables + /userguide/expids + /userguide/provenance Command list ============