diff --git a/.gitignore b/.gitignore
deleted file mode 100644
index 6769e21d99a63338394e47bc4c7d0aba1e88d5a5..0000000000000000000000000000000000000000
--- a/.gitignore
+++ /dev/null
@@ -1,160 +0,0 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-# Usually these files are written by a python script from a template
-# before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-# For a library or package, you might want to ignore these files since the code is
-# intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-# However, in case of collaboration, if having platform-specific dependencies or dependencies
-# having no cross-platform support, pipenv may install dependencies that don't work, or not
-# install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-# This is especially recommended for binary packages to ensure reproducibility, and is more
-# commonly ignored for libraries.
-# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-# in version control.
-# https://pdm.fming.dev/#use-with-ide
-.pdm.toml
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# PyCharm
-# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-# and can be added to the global gitignore or merged into this file. For a more nuclear
-# option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
\ No newline at end of file
diff --git a/autosubmit_api.egg-info/PKG-INFO b/autosubmit_api.egg-info/PKG-INFO
new file mode 100644
index 0000000000000000000000000000000000000000..31c238680622f81390d66c097e13a3e991728e12
--- /dev/null
+++ b/autosubmit_api.egg-info/PKG-INFO
@@ -0,0 +1,14 @@
+Metadata-Version: 2.1
+Name: autosubmit-api
+Version: 1.0.27
+Summary: An extension to the Autosubmit package that serves its information as an API
+Home-page: https://earth.bsc.es/gitlab/wuruchi/autosubmit_api
+Author: Wilmer Uruchi
+Author-email: wilmer.uruchi@bsc.es
+License: GNU GPL
+Keywords: autosubmit,API
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3.7
diff --git a/autosubmit_api.egg-info/SOURCES.txt b/autosubmit_api.egg-info/SOURCES.txt
new file mode 100644
index 0000000000000000000000000000000000000000..f0ccb7254c04d36adeeb15db22c11d7fdce57470
--- /dev/null
+++ b/autosubmit_api.egg-info/SOURCES.txt
@@ -0,0 +1,161 @@
+setup.py
+autosubmit_api/__init__.py
+autosubmit_api/app.py
+autosubmit_api/get-pip.py
+autosubmit_api.egg-info/PKG-INFO
+autosubmit_api.egg-info/SOURCES.txt
+autosubmit_api.egg-info/dependency_links.txt
+autosubmit_api.egg-info/requires.txt
+autosubmit_api.egg-info/top_level.txt
+autosubmit_api/autosubmit_legacy/__init__.py
+autosubmit_api/autosubmit_legacy/autosubmit.py
+autosubmit_api/autosubmit_legacy/job/__init__.py
+autosubmit_api/autosubmit_legacy/job/job.py
+autosubmit_api/autosubmit_legacy/job/job_common.py
+autosubmit_api/autosubmit_legacy/job/job_dict.py
+autosubmit_api/autosubmit_legacy/job/job_exceptions.py
+autosubmit_api/autosubmit_legacy/job/job_grouping.py
+autosubmit_api/autosubmit_legacy/job/job_list.py
+autosubmit_api/autosubmit_legacy/job/job_list_persistence.py
+autosubmit_api/autosubmit_legacy/job/job_package_persistence.py
+autosubmit_api/autosubmit_legacy/job/job_packager.py
+autosubmit_api/autosubmit_legacy/job/job_packages.py
+autosubmit_api/autosubmit_legacy/job/job_utils.py
+autosubmit_api/autosubmit_legacy/platforms/__init__.py
+autosubmit_api/autosubmit_legacy/platforms/ecmwf_adaptor.py
+autosubmit_api/autosubmit_legacy/platforms/ecplatform.py
+autosubmit_api/autosubmit_legacy/platforms/locplatform.py
+autosubmit_api/autosubmit_legacy/platforms/lsfplatform.py
+autosubmit_api/autosubmit_legacy/platforms/mn_adaptor.py
+autosubmit_api/autosubmit_legacy/platforms/paramiko_platform.py
+autosubmit_api/autosubmit_legacy/platforms/paramiko_submitter.py
+autosubmit_api/autosubmit_legacy/platforms/pbsplatform.py
+autosubmit_api/autosubmit_legacy/platforms/platform.py
+autosubmit_api/autosubmit_legacy/platforms/psplatform.py
+autosubmit_api/autosubmit_legacy/platforms/saga_platform.py
+autosubmit_api/autosubmit_legacy/platforms/saga_submitter.py
+autosubmit_api/autosubmit_legacy/platforms/sgeplatform.py
+autosubmit_api/autosubmit_legacy/platforms/slurmplatform.py
+autosubmit_api/autosubmit_legacy/platforms/submitter.py
+autosubmit_api/autosubmit_legacy/platforms/headers/__init__.py
+autosubmit_api/autosubmit_legacy/platforms/headers/ec_cca_header.py
+autosubmit_api/autosubmit_legacy/platforms/headers/ec_header.py
+autosubmit_api/autosubmit_legacy/platforms/headers/local_header.py
+autosubmit_api/autosubmit_legacy/platforms/headers/lsf_header.py
+autosubmit_api/autosubmit_legacy/platforms/headers/pbs10_header.py
+autosubmit_api/autosubmit_legacy/platforms/headers/pbs11_header.py
+autosubmit_api/autosubmit_legacy/platforms/headers/pbs12_header.py
+autosubmit_api/autosubmit_legacy/platforms/headers/ps_header.py
+autosubmit_api/autosubmit_legacy/platforms/headers/sge_header.py
+autosubmit_api/autosubmit_legacy/platforms/headers/slurm_header.py
+autosubmit_api/autosubmit_legacy/platforms/wrappers/__init__.py
+autosubmit_api/autosubmit_legacy/platforms/wrappers/wrapper_builder.py
+autosubmit_api/autosubmit_legacy/platforms/wrappers/wrapper_factory.py
+autosubmit_api/builders/__init__.py
+autosubmit_api/builders/basic_builder.py
+autosubmit_api/builders/configuration_facade_builder.py
+autosubmit_api/builders/experiment_history_builder.py
+autosubmit_api/builders/joblist_helper_builder.py
+autosubmit_api/builders/joblist_loader_builder.py
+autosubmit_api/builders/pkl_organizer_builder.py
+autosubmit_api/common/__init__.py
+autosubmit_api/common/utils.py
+autosubmit_api/common/utils_for_testing.py
+autosubmit_api/components/__init__.py
+autosubmit_api/components/experiment/__init__.py
+autosubmit_api/components/experiment/configuration_facade.py
+autosubmit_api/components/experiment/pkl_organizer.py
+autosubmit_api/components/experiment/test.py
+autosubmit_api/components/jobs/__init__.py
+autosubmit_api/components/jobs/job_factory.py
+autosubmit_api/components/jobs/job_support.py
+autosubmit_api/components/jobs/joblist_helper.py
+autosubmit_api/components/jobs/joblist_loader.py
+autosubmit_api/components/jobs/test.py
+autosubmit_api/components/jobs/utils.py
+autosubmit_api/components/representations/__init__.py
+autosubmit_api/components/representations/graph/__init__.py
+autosubmit_api/components/representations/graph/edge.py
+autosubmit_api/components/representations/graph/graph.py
+autosubmit_api/components/representations/graph/test.py
+autosubmit_api/components/representations/tree/__init__.py
+autosubmit_api/components/representations/tree/test.py
+autosubmit_api/components/representations/tree/tree.py
+autosubmit_api/config/__init__.py
+autosubmit_api/config/basicConfig.py
+autosubmit_api/config/config_common.py
+autosubmit_api/database/__init__.py
+autosubmit_api/database/autosubmit.py
+autosubmit_api/database/db_common.py
+autosubmit_api/database/db_jobdata.py
+autosubmit_api/database/db_manager.py
+autosubmit_api/database/db_structure.py
+autosubmit_api/experiment/__init__.py
+autosubmit_api/experiment/as_times_db_manager.py
+autosubmit_api/experiment/common_db_requests.py
+autosubmit_api/experiment/common_requests.py
+autosubmit_api/experiment/experiment_common.py
+autosubmit_api/experiment/experiment_db_manager.py
+autosubmit_api/experiment/test.py
+autosubmit_api/experiment/utils.py
+autosubmit_api/git/__init__.py
+autosubmit_api/git/autosubmit_git.py
+autosubmit_api/history/__init__.py
+autosubmit_api/history/experiment_history.py
+autosubmit_api/history/experiment_status.py
+autosubmit_api/history/experiment_status_manager.py
+autosubmit_api/history/internal_logging.py
+autosubmit_api/history/strategies.py
+autosubmit_api/history/test.py
+autosubmit_api/history/test_job_history.py
+autosubmit_api/history/test_strategies.py
+autosubmit_api/history/test_utils.py
+autosubmit_api/history/utils.py
+autosubmit_api/history/data_classes/__init__.py
+autosubmit_api/history/data_classes/experiment_run.py
+autosubmit_api/history/data_classes/job_data.py
+autosubmit_api/history/database_managers/__init__.py
+autosubmit_api/history/database_managers/database_manager.py
+autosubmit_api/history/database_managers/database_models.py
+autosubmit_api/history/database_managers/experiment_history_db_manager.py
+autosubmit_api/history/database_managers/experiment_status_db_manager.py
+autosubmit_api/history/database_managers/test.py
+autosubmit_api/history/platform_monitor/__init__.py
+autosubmit_api/history/platform_monitor/platform_monitor.py
+autosubmit_api/history/platform_monitor/platform_utils.py
+autosubmit_api/history/platform_monitor/slurm_monitor.py
+autosubmit_api/history/platform_monitor/slurm_monitor_item.py
+autosubmit_api/history/platform_monitor/test.py
+autosubmit_api/monitor/__init__.py
+autosubmit_api/monitor/diagram.py
+autosubmit_api/monitor/monitor.py
+autosubmit_api/monitor/utils.py
+autosubmit_api/notifications/__init__.py
+autosubmit_api/notifications/mail_notifier.py
+autosubmit_api/notifications/notifier.py
+autosubmit_api/performance/__init__.py
+autosubmit_api/performance/performance_metrics.py
+autosubmit_api/performance/utils.py
+autosubmit_api/statistics/__init__.py
+autosubmit_api/statistics/job_stat.py
+autosubmit_api/statistics/statistics.py
+autosubmit_api/statistics/stats_summary.py
+autosubmit_api/statistics/test.py
+autosubmit_api/statistics/utils.py
+autosubmit_api/workers/__init__.py
+autosubmit_api/workers/populate_details_db.py
+autosubmit_api/workers/populate_graph.py
+autosubmit_api/workers/populate_queue_run_times.py
+autosubmit_api/workers/populate_running_experiments.py
+autosubmit_api/workers/test.py
+autosubmit_api/workers/test_esarchive.py
+autosubmit_api/workers/verify_complete.py
+autosubmit_api/workers/business/__init__.py
+autosubmit_api/workers/business/populate_times.py
+autosubmit_api/workers/business/process_graph_drawings.py
+autosubmit_api/workers/deprecated/__init__.py
+autosubmit_api/workers/deprecated/fix_historic.py
+autosubmit_api/workers/deprecated/fix_historic_energy.py
+autosubmit_api/workers/populate_details/__init__.py
+autosubmit_api/workers/populate_details/populate.py
+autosubmit_api/workers/populate_details/test.py
\ No newline at end of file
diff --git a/autosubmit_api.egg-info/dependency_links.txt b/autosubmit_api.egg-info/dependency_links.txt
new file mode 100644
index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc
--- /dev/null
+++ b/autosubmit_api.egg-info/dependency_links.txt
@@ -0,0 +1 @@
+
diff --git a/autosubmit_api.egg-info/requires.txt b/autosubmit_api.egg-info/requires.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d8ff9e05a6fd86e529fee22dcb82571fd4c49046
--- /dev/null
+++ b/autosubmit_api.egg-info/requires.txt
@@ -0,0 +1,11 @@
+jwt==1.3.1
+requests==2.28.1
+flask_cors==3.0.10
+bscearth.utils==0.5.2
+pysqlite3==0.4.7
+numpy==1.21.6
+pydotplus==2.0.2
+portalocker==2.6.0
+networkx==2.6.3
+scipy==1.7.3
+paramiko==2.12.0
diff --git a/autosubmit_api.egg-info/top_level.txt b/autosubmit_api.egg-info/top_level.txt
new file mode 100644
index 0000000000000000000000000000000000000000..7ecd916442b11be3fd54d6ad68e3a2ef05838217
--- /dev/null
+++ b/autosubmit_api.egg-info/top_level.txt
@@ -0,0 +1 @@
+autosubmit_api
diff --git a/autosubmit_api/__init__.pyc b/autosubmit_api/__init__.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..224467877f8104b9b5e89aa172bda46b60e446b9
Binary files /dev/null and b/autosubmit_api/__init__.pyc differ
diff --git a/autosubmit_api/app.pyc b/autosubmit_api/app.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f9d49de8c66b7fcd5eac888dc667509e09d6ed79
Binary files /dev/null and b/autosubmit_api/app.pyc differ
diff --git a/autosubmit_api/autosubmit_api.egg-info/PKG-INFO b/autosubmit_api/autosubmit_api.egg-info/PKG-INFO
new file mode 100644
index 0000000000000000000000000000000000000000..754166785604aece4732b5dbe08990418dfd8e2c
--- /dev/null
+++ b/autosubmit_api/autosubmit_api.egg-info/PKG-INFO
@@ -0,0 +1,16 @@
+Metadata-Version: 1.1
+Name: autosubmit-api
+Version: 1.0.0
+Summary: An extension to the Autosubmit package that serves its information as an API
+Home-page: https://earth.bsc.es/gitlab/wuruchi/autosubmit_api
+Author: Wilmer Uruchi
+Author-email: wilmer.uruchi@bsc.es
+License: GNU GPL
+Description: UNKNOWN
+Keywords: autosubmit,API
+Platform: UNKNOWN
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3.7
diff --git a/autosubmit_api/autosubmit_api.egg-info/SOURCES.txt b/autosubmit_api/autosubmit_api.egg-info/SOURCES.txt
new file mode 100644
index 0000000000000000000000000000000000000000..861769ff0cfc65c3d1facd06b41e6c194716d681
--- /dev/null
+++ b/autosubmit_api/autosubmit_api.egg-info/SOURCES.txt
@@ -0,0 +1,156 @@
+.egg-info/PKG-INFO
+.egg-info/SOURCES.txt
+.egg-info/dependency_links.txt
+.egg-info/requires.txt
+.egg-info/top_level.txt
+autosubmit_legacy/__init__.py
+autosubmit_legacy/autosubmit.py
+autosubmit_legacy/job/__init__.py
+autosubmit_legacy/job/job.py
+autosubmit_legacy/job/job_common.py
+autosubmit_legacy/job/job_dict.py
+autosubmit_legacy/job/job_exceptions.py
+autosubmit_legacy/job/job_grouping.py
+autosubmit_legacy/job/job_list.py
+autosubmit_legacy/job/job_list_persistence.py
+autosubmit_legacy/job/job_package_persistence.py
+autosubmit_legacy/job/job_packager.py
+autosubmit_legacy/job/job_packages.py
+autosubmit_legacy/job/job_utils.py
+autosubmit_legacy/platforms/__init__.py
+autosubmit_legacy/platforms/ecmwf_adaptor.py
+autosubmit_legacy/platforms/ecplatform.py
+autosubmit_legacy/platforms/locplatform.py
+autosubmit_legacy/platforms/lsfplatform.py
+autosubmit_legacy/platforms/mn_adaptor.py
+autosubmit_legacy/platforms/paramiko_platform.py
+autosubmit_legacy/platforms/paramiko_submitter.py
+autosubmit_legacy/platforms/pbsplatform.py
+autosubmit_legacy/platforms/platform.py
+autosubmit_legacy/platforms/psplatform.py
+autosubmit_legacy/platforms/saga_platform.py
+autosubmit_legacy/platforms/saga_submitter.py
+autosubmit_legacy/platforms/sgeplatform.py
+autosubmit_legacy/platforms/slurmplatform.py
+autosubmit_legacy/platforms/submitter.py
+autosubmit_legacy/platforms/headers/__init__.py
+autosubmit_legacy/platforms/headers/ec_cca_header.py
+autosubmit_legacy/platforms/headers/ec_header.py
+autosubmit_legacy/platforms/headers/local_header.py
+autosubmit_legacy/platforms/headers/lsf_header.py
+autosubmit_legacy/platforms/headers/pbs10_header.py
+autosubmit_legacy/platforms/headers/pbs11_header.py
+autosubmit_legacy/platforms/headers/pbs12_header.py
+autosubmit_legacy/platforms/headers/ps_header.py
+autosubmit_legacy/platforms/headers/sge_header.py
+autosubmit_legacy/platforms/headers/slurm_header.py
+autosubmit_legacy/platforms/wrappers/__init__.py
+autosubmit_legacy/platforms/wrappers/wrapper_builder.py
+autosubmit_legacy/platforms/wrappers/wrapper_factory.py
+builders/__init__.py
+builders/basic_builder.py
+builders/configuration_facade_builder.py
+builders/experiment_history_builder.py
+builders/joblist_helper_builder.py
+builders/joblist_loader_builder.py
+builders/pkl_organizer_builder.py
+common/__init__.py
+common/utils.py
+common/utils_for_testing.py
+components/__init__.py
+components/experiment/__init__.py
+components/experiment/configuration_facade.py
+components/experiment/pkl_organizer.py
+components/experiment/test.py
+components/jobs/__init__.py
+components/jobs/job_factory.py
+components/jobs/job_support.py
+components/jobs/joblist_helper.py
+components/jobs/joblist_loader.py
+components/jobs/test.py
+components/jobs/utils.py
+components/representations/__init__.py
+components/representations/graph/__init__.py
+components/representations/graph/edge.py
+components/representations/graph/graph.py
+components/representations/graph/test.py
+components/representations/tree/__init__.py
+components/representations/tree/test.py
+components/representations/tree/tree.py
+config/__init__.py
+config/basicConfig.py
+config/config_common.py
+database/__init__.py
+database/db_common.py
+database/db_jobdata.py
+database/db_manager.py
+database/db_structure.py
+experiment/__init__.py
+experiment/as_times_db_manager.py
+experiment/common_db_requests.py
+experiment/common_requests.py
+experiment/experiment_common.py
+experiment/experiment_db_manager.py
+experiment/test.py
+experiment/utils.py
+git/__init__.py
+git/autosubmit_git.py
+history/__init__.py
+history/experiment_history.py
+history/experiment_status.py
+history/experiment_status_manager.py
+history/internal_logging.py
+history/strategies.py
+history/test.py
+history/test_job_history.py
+history/test_strategies.py
+history/test_utils.py
+history/utils.py
+history/data_classes/__init__.py
+history/data_classes/experiment_run.py
+history/data_classes/job_data.py
+history/database_managers/__init__.py
+history/database_managers/database_manager.py
+history/database_managers/database_models.py
+history/database_managers/experiment_history_db_manager.py
+history/database_managers/experiment_status_db_manager.py
+history/database_managers/test.py
+history/platform_monitor/__init__.py
+history/platform_monitor/platform_monitor.py
+history/platform_monitor/platform_utils.py
+history/platform_monitor/slurm_monitor.py
+history/platform_monitor/slurm_monitor_item.py
+history/platform_monitor/test.py
+monitor/__init__.py
+monitor/diagram.py
+monitor/monitor.py
+monitor/utils.py
+notifications/__init__.py
+notifications/mail_notifier.py
+notifications/notifier.py
+performance/__init__.py
+performance/performance_metrics.py
+performance/utils.py
+statistics/__init__.py
+statistics/job_stat.py
+statistics/statistics.py
+statistics/stats_summary.py
+statistics/test.py
+statistics/utils.py
+workers/__init__.py
+workers/populate_details_db.py
+workers/populate_graph.py
+workers/populate_queue_run_times.py
+workers/populate_running_experiments.py
+workers/test.py
+workers/test_esarchive.py
+workers/verify_complete.py
+workers/business/__init__.py
+workers/business/populate_times.py
+workers/business/process_graph_drawings.py
+workers/deprecated/__init__.py
+workers/deprecated/fix_historic.py
+workers/deprecated/fix_historic_energy.py
+workers/populate_details/__init__.py
+workers/populate_details/populate.py
+workers/populate_details/test.py
\ No newline at end of file
diff --git a/autosubmit_api/autosubmit_api.egg-info/dependency_links.txt b/autosubmit_api/autosubmit_api.egg-info/dependency_links.txt
new file mode 100644
index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc
--- /dev/null
+++ b/autosubmit_api/autosubmit_api.egg-info/dependency_links.txt
@@ -0,0 +1 @@
+
diff --git a/autosubmit_api/autosubmit_api.egg-info/requires.txt b/autosubmit_api/autosubmit_api.egg-info/requires.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d48668615a82bcef4d7bdaf81c0326244541e470
--- /dev/null
+++ b/autosubmit_api/autosubmit_api.egg-info/requires.txt
@@ -0,0 +1,37 @@
+bscearth.utils==0.5.2
+argparse<2,>=1.2
+python-dateutil>2
+pydotplus>=2
+pyparsing>=2.0.1
+numpy
+matplotlib
+paramiko==2.6.0
+mock>=1.3.0
+portalocker>=0.5.7
+networkx
+bscearth.utils
+Flask==1.0.4
+Flask-Cors==3.0.8
+Flask-Jsonpify==1.5.0
+Flask-RESTful==0.3.7
+SQLAlchemy==1.3.11
+PyJWT==1.7.1
+Flask==1.1.1
+Flask-Cors==3.0.8
+Flask-Jsonpify==1.5.0
+Flask-RESTful==0.3.7
+gunicorn==19.9.0
+mock==3.0.5
+networkx==2.2
+numpy==1.16.4
+paramiko==1.15.0
+portalocker==0.5.7
+pydotplus==2.0.2
+pydot==1.4.1
+regex==2019.6.8
+requests==2.22.0
+graphviz==0.13
+enum34==1.1.6
+typing==3.7.4.3
+radical.saga==0.70.0
+scipy==1.2.2
diff --git a/autosubmit_api/autosubmit_api.egg-info/top_level.txt b/autosubmit_api/autosubmit_api.egg-info/top_level.txt
new file mode 100644
index 0000000000000000000000000000000000000000..25b178aab7083c3934c3d3f0b90bdc5ac4e5988d
--- /dev/null
+++ b/autosubmit_api/autosubmit_api.egg-info/top_level.txt
@@ -0,0 +1,14 @@
+autosubmit_legacy
+builders
+common
+components
+config
+database
+experiment
+git
+history
+monitor
+notifications
+performance
+statistics
+workers
diff --git a/autosubmit_api/autosubmit_legacy/__init__.pyc b/autosubmit_api/autosubmit_legacy/__init__.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..49bdbde9f260188e7702232471cd86878c1b2c07
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/__init__.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/autosubmit.pyc b/autosubmit_api/autosubmit_legacy/autosubmit.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f8cfe1132f1d4ded653b4c2c646d0ff8d14ca596
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/autosubmit.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/job/__init__.pyc b/autosubmit_api/autosubmit_legacy/job/__init__.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..325c7874fe54b1de27c286e0f053df817ffd2faa
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/__init__.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/job/job.pyc b/autosubmit_api/autosubmit_legacy/job/job.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ae608934554368795529ad102160520af40b4a98
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/job.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/job/job_common.pyc b/autosubmit_api/autosubmit_legacy/job/job_common.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..60a7501d0a1c9a3fd5e7da7cf30f8f9edbbd0445
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/job_common.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/job/job_dict.pyc b/autosubmit_api/autosubmit_legacy/job/job_dict.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c058fefac0e19e828d9991d3c56b870793bdff15
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/job_dict.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/job/job_exceptions.pyc b/autosubmit_api/autosubmit_legacy/job/job_exceptions.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ba4cb9e314daa9d575fd84140e2594532327f081
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/job_exceptions.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/job/job_grouping.pyc b/autosubmit_api/autosubmit_legacy/job/job_grouping.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..124384bd4d54e37d5aaaab0fe330bbbe3f7e344d
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/job_grouping.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/job/job_list.pyc b/autosubmit_api/autosubmit_legacy/job/job_list.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b7c1a0681b5f1b58b8205e53c781a8a8ef650a21
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/job_list.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/job/job_list_persistence.pyc b/autosubmit_api/autosubmit_legacy/job/job_list_persistence.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d3a3d5d5a4492947b21a39d919f9ed26060c4086
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/job_list_persistence.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/job/job_package_persistence.pyc b/autosubmit_api/autosubmit_legacy/job/job_package_persistence.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..83c8873d6fa7175cfb71b6858695cf81ffa85f3e
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/job_package_persistence.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/job/job_packager.pyc b/autosubmit_api/autosubmit_legacy/job/job_packager.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..69ae92fd04e3311bb9303bb0f9aab4bc3f31814f
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/job_packager.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/job/job_packages.pyc b/autosubmit_api/autosubmit_legacy/job/job_packages.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c5894a181ffc59481e96af25e42dca05271297ee
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/job_packages.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/job/job_utils.pyc b/autosubmit_api/autosubmit_legacy/job/job_utils.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c2e68e53adcc9e578bd95350ebdc15b28c4c524a
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/job_utils.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/job/tree.pyc b/autosubmit_api/autosubmit_legacy/job/tree.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7653d608b704d17781ec27ea28c25f5853887735
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/tree.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/__init__.pyc b/autosubmit_api/autosubmit_legacy/platforms/__init__.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e19ab6bbc3f392d6c279be2e83c4bd12108a9cc0
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/__init__.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/ecmwf_adaptor.pyc b/autosubmit_api/autosubmit_legacy/platforms/ecmwf_adaptor.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..69e9dec946c6d64914c2b15d5ee82ea0e82f7149
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/ecmwf_adaptor.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/ecplatform.pyc b/autosubmit_api/autosubmit_legacy/platforms/ecplatform.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c93c83f385cf8d33e774e1ce5d2b7dc2db9950ff
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/ecplatform.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/headers/__init__.pyc b/autosubmit_api/autosubmit_legacy/platforms/headers/__init__.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9fd954a2a3d9c2b945c67b52bb63da939b73639c
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/headers/__init__.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/headers/ec_cca_header.pyc b/autosubmit_api/autosubmit_legacy/platforms/headers/ec_cca_header.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..cd001ae52cf62d862be142ece3def116df8d539b
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/headers/ec_cca_header.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/headers/ec_header.pyc b/autosubmit_api/autosubmit_legacy/platforms/headers/ec_header.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e324bc0a7d9836e19035d629da4f1abf8c2263bb
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/headers/ec_header.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/headers/local_header.pyc b/autosubmit_api/autosubmit_legacy/platforms/headers/local_header.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..daeddc94b247145bf10f977dc2bba99eac31fce8
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/headers/local_header.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/headers/lsf_header.pyc b/autosubmit_api/autosubmit_legacy/platforms/headers/lsf_header.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ffb6ec747b282fd3b3275967bfd34b1695cd4593
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/headers/lsf_header.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/headers/pbs10_header.pyc b/autosubmit_api/autosubmit_legacy/platforms/headers/pbs10_header.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..62a925d9f0a85cc729a99a20f1872770619154b1
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/headers/pbs10_header.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/headers/pbs11_header.pyc b/autosubmit_api/autosubmit_legacy/platforms/headers/pbs11_header.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3081967dc161d1d62789c9a4a53adb8344cb7f21
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/headers/pbs11_header.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/headers/pbs12_header.pyc b/autosubmit_api/autosubmit_legacy/platforms/headers/pbs12_header.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..eb960f9c8e978a82b78db95b6e1d1adfe4b7fc52
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/headers/pbs12_header.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/headers/ps_header.pyc b/autosubmit_api/autosubmit_legacy/platforms/headers/ps_header.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e76bc5ffcb3777a25da78e75ad082a46b25636b3
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/headers/ps_header.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/headers/sge_header.pyc b/autosubmit_api/autosubmit_legacy/platforms/headers/sge_header.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b74a22bb3265c58d69aeb77beb79c838d4d0c0ba
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/headers/sge_header.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/headers/slurm_header.pyc b/autosubmit_api/autosubmit_legacy/platforms/headers/slurm_header.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..592852ba36080ec12f5dc1cf98ac91cad35e10c2
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/headers/slurm_header.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/locplatform.pyc b/autosubmit_api/autosubmit_legacy/platforms/locplatform.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d01dc7cab7a9a0076550e7f328a12d2c522d84a2
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/locplatform.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/lsfplatform.pyc b/autosubmit_api/autosubmit_legacy/platforms/lsfplatform.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d4c08f54ac4ad112b069a136b1b611e506830ef1
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/lsfplatform.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/mn_adaptor.pyc b/autosubmit_api/autosubmit_legacy/platforms/mn_adaptor.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8f77b751b4f9a9368163501cb152dd7322e0fd79
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/mn_adaptor.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/paramiko_platform.pyc b/autosubmit_api/autosubmit_legacy/platforms/paramiko_platform.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..095b8b8402fe7411f8eff751bddf3ff89d1c4eb5
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/paramiko_platform.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/paramiko_submitter.pyc b/autosubmit_api/autosubmit_legacy/platforms/paramiko_submitter.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b56f668024f120a9455cb8fabfbf10987a2d2dad
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/paramiko_submitter.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/pbsplatform.pyc b/autosubmit_api/autosubmit_legacy/platforms/pbsplatform.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..99f206314ad773100ad50c86a958e0d05f322695
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/pbsplatform.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/platform.pyc b/autosubmit_api/autosubmit_legacy/platforms/platform.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..12dcdfa4a5267dfa05f4feb6079b0c508f61990f
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/platform.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/psplatform.pyc b/autosubmit_api/autosubmit_legacy/platforms/psplatform.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..13b26c82a77ccd22e91eeec18e3a3fca2dffaa87
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/psplatform.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/saga_platform.pyc b/autosubmit_api/autosubmit_legacy/platforms/saga_platform.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..61ae638d8f0de2a0db2f5ed7d61b0a83344ff030
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/saga_platform.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/saga_submitter.pyc b/autosubmit_api/autosubmit_legacy/platforms/saga_submitter.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a06901982eea2f87835817cc89160e15088da0e2
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/saga_submitter.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/sgeplatform.pyc b/autosubmit_api/autosubmit_legacy/platforms/sgeplatform.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..35fb5ba153e47709d7707a025de6c701f1da7642
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/sgeplatform.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/slurmplatform.pyc b/autosubmit_api/autosubmit_legacy/platforms/slurmplatform.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e06a8d820c3b99a8b902cb79a6637df788831fe5
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/slurmplatform.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/submitter.pyc b/autosubmit_api/autosubmit_legacy/platforms/submitter.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a6457b9b3f1aa54dbc8c75117d7b2833f68d70e5
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/submitter.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/wrappers/__init__.pyc b/autosubmit_api/autosubmit_legacy/platforms/wrappers/__init__.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7cdcbb3f33a32e1423681f466f99897db9326a32
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/wrappers/__init__.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/wrappers/wrapper_builder.pyc b/autosubmit_api/autosubmit_legacy/platforms/wrappers/wrapper_builder.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8d6581211c0b4bcacd394070b3f873d748f0a448
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/wrappers/wrapper_builder.pyc differ
diff --git a/autosubmit_api/autosubmit_legacy/platforms/wrappers/wrapper_factory.pyc b/autosubmit_api/autosubmit_legacy/platforms/wrappers/wrapper_factory.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a8853878c5776519f36a9c4d4afd6bec46419f76
Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/wrappers/wrapper_factory.pyc differ
diff --git a/autosubmit_api/build/lib/autosubmit_legacy/__init__.py b/autosubmit_api/build/lib/autosubmit_legacy/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/autosubmit_api/build/lib/autosubmit_legacy/autosubmit.py b/autosubmit_api/build/lib/autosubmit_legacy/autosubmit.py
new file mode 100644
index 0000000000000000000000000000000000000000..4a7c4dba46018119fbda5636654d4babf5aac686
--- /dev/null
+++ b/autosubmit_api/build/lib/autosubmit_legacy/autosubmit.py
@@ -0,0 +1,3498 @@
+#!/usr/bin/env python
+
+# Copyright 2017 Earth Sciences Department, BSC-CNS
+
+# This file is part of Autosubmit.
+
+# Autosubmit is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# Autosubmit is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with Autosubmit. If not, see .
+# pipeline_test
+
+
+import traceback
+from pyparsing import nestedExpr
+from collections import defaultdict
+from distutils.util import strtobool
+from pkg_resources import require, resource_listdir, resource_exists, resource_string
+import portalocker
+import datetime
+import signal
+import random
+import re
+import shutil
+import sys
+import pwd
+import os
+import copy
+import time
+import tarfile
+import json
+import subprocess
+import argparse
+
+sys.path.insert(0, os.path.abspath('.'))
+from ..config.basicConfig import BasicConfig
+from ..config.config_common import AutosubmitConfig
+from bscearth.utils.config_parser import ConfigParserFactory
+from .job.job_common import Status
+from ..git.autosubmit_git import AutosubmitGit
+from .job.job_list import JobList
+from .job.job_packages import JobPackageThread
+from .job.job_package_persistence import JobPackagePersistence
+from .job.job_list_persistence import JobListPersistenceDb
+from .job.job_list_persistence import JobListPersistencePkl
+from .job.job_grouping import JobGrouping
+from bscearth.utils.log import Log
+from ..database.db_common import create_db
+from ..experiment.experiment_common import new_experiment
+from ..experiment.experiment_common import copy_experiment
+from ..database.db_common import delete_experiment
+from ..database.db_common import get_autosubmit_version
+from ..monitor.monitor import Monitor
+from bscearth.utils.date import date2str
+from ..notifications.mail_notifier import MailNotifier
+from ..notifications.notifier import Notifier
+from .platforms.saga_submitter import SagaSubmitter
+from .platforms.paramiko_submitter import ParamikoSubmitter
+from .job.job_exceptions import WrongTemplateException
+from .job.job_packager import JobPackager
+from .platforms.paramiko_platform import ParamikoTimeout
+"""
+Main module for autosubmit. Only contains an interface class to all functionality implemented on autosubmit
+"""
+
+try:
+ # noinspection PyCompatibility
+ from configparser import SafeConfigParser
+except ImportError:
+ # noinspection PyCompatibility
+ from configparser import SafeConfigParser
+
+# It is Python dialog available? (optional dependency)
+try:
+ import dialog
+except Exception:
+ dialog = None
+
+
+# noinspection PyPackageRequirements
+# noinspection PyPackageRequirements
+# noinspection PyPackageRequirements
+
+# noinspection PyUnusedLocal
+
+
+def signal_handler(signal_received, frame):
+ """
+ Used to handle interrupt signals, allowing autosubmit to clean before exit
+
+ :param signal_received:
+ :param frame:
+ """
+ Log.info('Autosubmit will interrupt at the next safe occasion')
+ Autosubmit.exit = True
+
+
+class Autosubmit:
+ """
+ Interface class for autosubmit.
+ """
+ # sys.setrecursionlimit(500000)
+ # # Get the version number from the relevant file. If not, from autosubmit package
+ # scriptdir = os.path.abspath(os.path.dirname(__file__))
+
+ # if not os.path.exists(os.path.join(scriptdir, 'VERSION')):
+ # scriptdir = os.path.join(scriptdir, os.path.pardir)
+
+ # version_path = os.path.join(scriptdir, 'VERSION')
+ # readme_path = os.path.join(scriptdir, 'README')
+ # changes_path = os.path.join(scriptdir, 'CHANGELOG')
+ # if os.path.isfile(version_path):
+ # with open(version_path) as f:
+ # autosubmit_version = f.read().strip()
+ # else:
+ # autosubmit_version = require("autosubmitAPIwu")[0].version
+
+ exit = False
+
+ @staticmethod
+ def parse_args():
+ """
+ Parse arguments given to an executable and start execution of command given
+ """
+ try:
+ BasicConfig.read()
+
+ parser = argparse.ArgumentParser(
+ description='Main executable for autosubmit. ')
+ parser.add_argument('-v', '--version', action='version', version=Autosubmit.autosubmit_version,
+ help="returns autosubmit's version number and exit")
+ parser.add_argument('-lf', '--logfile', choices=('EVERYTHING', 'DEBUG', 'INFO', 'RESULT', 'USER_WARNING',
+ 'WARNING', 'ERROR', 'CRITICAL', 'NO_LOG'),
+ default='DEBUG', type=str,
+ help="sets file's log level.")
+ parser.add_argument('-lc', '--logconsole', choices=('EVERYTHING', 'DEBUG', 'INFO', 'RESULT', 'USER_WARNING',
+ 'WARNING', 'ERROR', 'CRITICAL', 'NO_LOG'),
+ default='INFO', type=str,
+ help="sets console's log level")
+
+ subparsers = parser.add_subparsers(dest='command')
+
+ # Run
+ subparser = subparsers.add_parser(
+ 'run', description="runs specified experiment")
+ subparser.add_argument('expid', help='experiment identifier')
+ subparser.add_argument('-nt', '--notransitive', action='store_true',
+ default=False, help='Disable transitive reduction')
+
+ # Expid
+ subparser = subparsers.add_parser(
+ 'expid', description="Creates a new experiment")
+ group = subparser.add_mutually_exclusive_group()
+ group.add_argument(
+ '-y', '--copy', help='makes a copy of the specified experiment')
+ group.add_argument('-dm', '--dummy', action='store_true',
+ help='creates a new experiment with default values, usually for testing')
+ group.add_argument('-op', '--operational', action='store_true',
+ help='creates a new experiment with operational experiment id')
+ subparser.add_argument('-H', '--HPC', required=True,
+ help='specifies the HPC to use for the experiment')
+ subparser.add_argument('-d', '--description', type=str, required=True,
+ help='sets a description for the experiment to store in the database.')
+
+ # Delete
+ subparser = subparsers.add_parser(
+ 'delete', description="delete specified experiment")
+ subparser.add_argument('expid', help='experiment identifier')
+ subparser.add_argument(
+ '-f', '--force', action='store_true', help='deletes experiment without confirmation')
+
+ # Monitor
+ subparser = subparsers.add_parser(
+ 'monitor', description="plots specified experiment")
+ subparser.add_argument('expid', help='experiment identifier')
+ subparser.add_argument('-o', '--output', choices=('pdf', 'png', 'ps', 'svg'), default='pdf',
+ help='chooses type of output for generated plot')
+ subparser.add_argument('-group_by', choices=('date', 'member', 'chunk', 'split', 'automatic'), default=None,
+ help='Groups the jobs automatically or by date, member, chunk or split')
+ subparser.add_argument('-expand', type=str,
+ help='Supply the list of dates/members/chunks to filter the list of jobs. Default = "Any". '
+ 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"')
+ subparser.add_argument(
+ '-expand_status', type=str, help='Select the statuses to be expanded')
+ subparser.add_argument('--hide_groups', action='store_true',
+ default=False, help='Hides the groups from the plot')
+ subparser.add_argument('-cw', '--check_wrapper', action='store_true',
+ default=False, help='Generate possible wrapper in the current workflow')
+
+ group2 = subparser.add_mutually_exclusive_group(required=False)
+
+ group.add_argument('-fs', '--filter_status', type=str,
+ choices=('Any', 'READY', 'COMPLETED',
+ 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN'),
+ help='Select the original status to filter the list of jobs')
+ group = subparser.add_mutually_exclusive_group(required=False)
+ group.add_argument('-fl', '--list', type=str,
+ help='Supply the list of job names to be filtered. Default = "Any". '
+ 'LIST = "b037_20101101_fc3_21_sim b037_20111101_fc4_26_sim"')
+ group.add_argument('-fc', '--filter_chunks', type=str,
+ help='Supply the list of chunks to filter the list of jobs. Default = "Any". '
+ 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"')
+ group.add_argument('-fs', '--filter_status', type=str,
+ choices=('Any', 'READY', 'COMPLETED',
+ 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN'),
+ help='Select the original status to filter the list of jobs')
+ group.add_argument('-ft', '--filter_type', type=str,
+ help='Select the job type to filter the list of jobs')
+ subparser.add_argument('--hide', action='store_true', default=False,
+ help='hides plot window')
+ group2.add_argument('--txt', action='store_true', default=False,
+ help='Generates only txt status file')
+
+ group2.add_argument('-txtlog', '--txt_logfiles', action='store_true', default=False,
+ help='Generates only txt status file(AS < 3.12b behaviour)')
+
+ subparser.add_argument('-nt', '--notransitive', action='store_true',
+ default=False, help='Disable transitive reduction')
+
+ # Stats
+ subparser = subparsers.add_parser(
+ 'stats', description="plots statistics for specified experiment")
+ subparser.add_argument('expid', help='experiment identifier')
+ subparser.add_argument('-ft', '--filter_type', type=str, help='Select the job type to filter '
+ 'the list of jobs')
+ subparser.add_argument('-fp', '--filter_period', type=int, help='Select the period to filter jobs '
+ 'from current time to the past '
+ 'in number of hours back')
+ subparser.add_argument('-o', '--output', choices=('pdf', 'png', 'ps', 'svg'), default='pdf',
+ help='type of output for generated plot')
+ subparser.add_argument('--hide', action='store_true', default=False,
+ help='hides plot window')
+ subparser.add_argument('-nt', '--notransitive', action='store_true',
+ default=False, help='Disable transitive reduction')
+
+ # Clean
+ subparser = subparsers.add_parser(
+ 'clean', description="clean specified experiment")
+ subparser.add_argument('expid', help='experiment identifier')
+ subparser.add_argument(
+ '-pr', '--project', action="store_true", help='clean project')
+ subparser.add_argument('-p', '--plot', action="store_true",
+ help='clean plot, only 2 last will remain')
+ subparser.add_argument('-s', '--stats', action="store_true",
+ help='clean stats, only last will remain')
+
+ # Recovery
+ subparser = subparsers.add_parser(
+ 'recovery', description="recover specified experiment")
+ subparser.add_argument(
+ 'expid', type=str, help='experiment identifier')
+ subparser.add_argument(
+ '-np', '--noplot', action='store_true', default=False, help='omit plot')
+ subparser.add_argument('--all', action="store_true", default=False,
+ help='Get completed files to synchronize pkl')
+ subparser.add_argument(
+ '-s', '--save', action="store_true", default=False, help='Save changes to disk')
+ subparser.add_argument('--hide', action='store_true', default=False,
+ help='hides plot window')
+ subparser.add_argument('-group_by', choices=('date', 'member', 'chunk', 'split', 'automatic'), default=None,
+ help='Groups the jobs automatically or by date, member, chunk or split')
+ subparser.add_argument('-expand', type=str,
+ help='Supply the list of dates/members/chunks to filter the list of jobs. Default = "Any". '
+ 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"')
+ subparser.add_argument(
+ '-expand_status', type=str, help='Select the statuses to be expanded')
+ subparser.add_argument('-nt', '--notransitive', action='store_true',
+ default=False, help='Disable transitive reduction')
+ subparser.add_argument('-nl', '--no_recover_logs', action='store_true', default=False,
+ help='Disable logs recovery')
+ # Migrate
+ subparser = subparsers.add_parser(
+ 'migrate', description="Migrate experiments from current user to another")
+ subparser.add_argument('expid', help='experiment identifier')
+ group = subparser.add_mutually_exclusive_group(required=True)
+ group.add_argument('-o', '--offer', action="store_true",
+ default=False, help='Offer experiment')
+ group.add_argument('-p', '--pickup', action="store_true",
+ default=False, help='Pick-up released experiment')
+
+ # Inspect
+ subparser = subparsers.add_parser(
+ 'inspect', description="Generate all .cmd files")
+ subparser.add_argument('expid', help='experiment identifier')
+ subparser.add_argument('-nt', '--notransitive', action='store_true',
+ default=False, help='Disable transitive reduction')
+ subparser.add_argument(
+ '-f', '--force', action="store_true", help='Overwrite all cmd')
+ subparser.add_argument('-cw', '--check_wrapper', action='store_true',
+ default=False, help='Generate possible wrapper in the current workflow')
+
+ group.add_argument('-fs', '--filter_status', type=str,
+ choices=('Any', 'READY', 'COMPLETED',
+ 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN'),
+ help='Select the original status to filter the list of jobs')
+ group = subparser.add_mutually_exclusive_group(required=False)
+ group.add_argument('-fl', '--list', type=str,
+ help='Supply the list of job names to be filtered. Default = "Any". '
+ 'LIST = "b037_20101101_fc3_21_sim b037_20111101_fc4_26_sim"')
+ group.add_argument('-fc', '--filter_chunks', type=str,
+ help='Supply the list of chunks to filter the list of jobs. Default = "Any". '
+ 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"')
+ group.add_argument('-fs', '--filter_status', type=str,
+ choices=('Any', 'READY', 'COMPLETED',
+ 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN'),
+ help='Select the original status to filter the list of jobs')
+ group.add_argument('-ft', '--filter_type', type=str,
+ help='Select the job type to filter the list of jobs')
+
+ # Check
+ subparser = subparsers.add_parser(
+ 'check', description="check configuration for specified experiment")
+ subparser.add_argument('expid', help='experiment identifier')
+ subparser.add_argument('-nt', '--notransitive', action='store_true',
+ default=False, help='Disable transitive reduction')
+ # Describe
+ subparser = subparsers.add_parser(
+ 'describe', description="Show details for specified experiment")
+ subparser.add_argument('expid', help='experiment identifier')
+
+ # Create
+ subparser = subparsers.add_parser(
+ 'create', description="create specified experiment joblist")
+ subparser.add_argument('expid', help='experiment identifier')
+ subparser.add_argument(
+ '-np', '--noplot', action='store_true', default=False, help='omit plot')
+ subparser.add_argument('--hide', action='store_true', default=False,
+ help='hides plot window')
+ subparser.add_argument('-o', '--output', choices=('pdf', 'png', 'ps', 'svg'), default='pdf',
+ help='chooses type of output for generated plot')
+ subparser.add_argument('-group_by', choices=('date', 'member', 'chunk', 'split', 'automatic'), default=None,
+ help='Groups the jobs automatically or by date, member, chunk or split')
+ subparser.add_argument('-expand', type=str,
+ help='Supply the list of dates/members/chunks to filter the list of jobs. Default = "Any". '
+ 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"')
+ subparser.add_argument(
+ '-expand_status', type=str, help='Select the statuses to be expanded')
+ subparser.add_argument('-nt', '--notransitive', action='store_true',
+ default=False, help='Disable transitive reduction')
+ subparser.add_argument('-cw', '--check_wrapper', action='store_true',
+ default=False, help='Generate possible wrapper in the current workflow')
+
+ # Configure
+ subparser = subparsers.add_parser('configure', description="configure database and path for autosubmit. It "
+ "can be done at machine, user or local level."
+ "If no arguments specified configure will "
+ "display dialog boxes (if installed)")
+ subparser.add_argument(
+ '--advanced', action="store_true", help="Open advanced configuration of autosubmit")
+ subparser.add_argument('-db', '--databasepath', default=None, help='path to database. If not supplied, '
+ 'it will prompt for it')
+ subparser.add_argument(
+ '-dbf', '--databasefilename', default=None, help='database filename')
+ subparser.add_argument('-lr', '--localrootpath', default=None, help='path to store experiments. If not '
+ 'supplied, it will prompt for it')
+ subparser.add_argument('-pc', '--platformsconfpath', default=None, help='path to platforms.conf file to '
+ 'use by default. Optional')
+ subparser.add_argument('-jc', '--jobsconfpath', default=None, help='path to jobs.conf file to use by '
+ 'default. Optional')
+ subparser.add_argument(
+ '-sm', '--smtphostname', default=None, help='STMP server hostname. Optional')
+ subparser.add_argument(
+ '-mf', '--mailfrom', default=None, help='Notifications sender address. Optional')
+ group = subparser.add_mutually_exclusive_group()
+ group.add_argument('--all', action="store_true",
+ help='configure for all users')
+ group.add_argument('--local', action="store_true", help='configure only for using Autosubmit from this '
+ 'path')
+
+ # Install
+ subparsers.add_parser(
+ 'install', description='install database for autosubmit on the configured folder')
+
+ # Set status
+ subparser = subparsers.add_parser(
+ 'setstatus', description="sets job status for an experiment")
+ subparser.add_argument('expid', help='experiment identifier')
+ subparser.add_argument(
+ '-np', '--noplot', action='store_true', default=False, help='omit plot')
+ subparser.add_argument(
+ '-s', '--save', action="store_true", default=False, help='Save changes to disk')
+
+ subparser.add_argument('-t', '--status_final',
+ choices=('READY', 'COMPLETED', 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN',
+ 'QUEUING', 'RUNNING'),
+ required=True,
+ help='Supply the target status')
+ group = subparser.add_mutually_exclusive_group(required=True)
+ group.add_argument('-fl', '--list', type=str,
+ help='Supply the list of job names to be changed. Default = "Any". '
+ 'LIST = "b037_20101101_fc3_21_sim b037_20111101_fc4_26_sim"')
+ group.add_argument('-fc', '--filter_chunks', type=str,
+ help='Supply the list of chunks to change the status. Default = "Any". '
+ 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"')
+ group.add_argument('-fs', '--filter_status', type=str,
+ help='Select the status (one or more) to filter the list of jobs.'
+ "Valid values = ['Any', 'READY', 'COMPLETED', 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN']")
+ group.add_argument('-ft', '--filter_type', type=str,
+ help='Select the job type to filter the list of jobs')
+
+ subparser.add_argument('--hide', action='store_true', default=False,
+ help='hides plot window')
+ subparser.add_argument('-group_by', choices=('date', 'member', 'chunk', 'split', 'automatic'), default=None,
+ help='Groups the jobs automatically or by date, member, chunk or split')
+ subparser.add_argument('-expand', type=str,
+ help='Supply the list of dates/members/chunks to filter the list of jobs. Default = "Any". '
+ 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"')
+ subparser.add_argument(
+ '-expand_status', type=str, help='Select the statuses to be expanded')
+ subparser.add_argument('-nt', '--notransitive', action='store_true',
+ default=False, help='Disable transitive reduction')
+ subparser.add_argument('-cw', '--check_wrapper', action='store_true',
+ default=False, help='Generate possible wrapper in the current workflow')
+
+ # Test Case
+ subparser = subparsers.add_parser(
+ 'testcase', description='create test case experiment')
+ subparser.add_argument(
+ '-y', '--copy', help='makes a copy of the specified experiment')
+ subparser.add_argument(
+ '-d', '--description', required=True, help='description of the test case')
+ subparser.add_argument('-c', '--chunks', help='chunks to run')
+ subparser.add_argument('-m', '--member', help='member to run')
+ subparser.add_argument('-s', '--stardate', help='stardate to run')
+ subparser.add_argument(
+ '-H', '--HPC', required=True, help='HPC to run experiment on it')
+ subparser.add_argument(
+ '-b', '--branch', help='branch of git to run (or revision from subversion)')
+
+ # Test
+ subparser = subparsers.add_parser(
+ 'test', description='test experiment')
+ subparser.add_argument('expid', help='experiment identifier')
+ subparser.add_argument(
+ '-c', '--chunks', required=True, help='chunks to run')
+ subparser.add_argument('-m', '--member', help='member to run')
+ subparser.add_argument('-s', '--stardate', help='stardate to run')
+ subparser.add_argument(
+ '-H', '--HPC', help='HPC to run experiment on it')
+ subparser.add_argument(
+ '-b', '--branch', help='branch of git to run (or revision from subversion)')
+
+ # Refresh
+ subparser = subparsers.add_parser(
+ 'refresh', description='refresh project directory for an experiment')
+ subparser.add_argument('expid', help='experiment identifier')
+ subparser.add_argument('-mc', '--model_conf', default=False, action='store_true',
+ help='overwrite model conf file')
+ subparser.add_argument('-jc', '--jobs_conf', default=False, action='store_true',
+ help='overwrite jobs conf file')
+
+ # Archive
+ subparser = subparsers.add_parser(
+ 'archive', description='archives an experiment')
+ subparser.add_argument('expid', help='experiment identifier')
+
+ # Unarchive
+ subparser = subparsers.add_parser(
+ 'unarchive', description='unarchives an experiment')
+ subparser.add_argument('expid', help='experiment identifier')
+
+ # Readme
+ subparsers.add_parser('readme', description='show readme')
+
+ # Changelog
+ subparsers.add_parser('changelog', description='show changelog')
+
+ args = parser.parse_args()
+
+ Log.set_console_level(args.logconsole)
+ Log.set_file_level(args.logfile)
+
+ if args.command == 'run':
+ return Autosubmit.run_experiment(args.expid, args.notransitive)
+ elif args.command == 'expid':
+ return Autosubmit.expid(args.HPC, args.description, args.copy, args.dummy, False,
+ args.operational) != ''
+ elif args.command == 'delete':
+ return Autosubmit.delete(args.expid, args.force)
+ elif args.command == 'monitor':
+ return Autosubmit.monitor(args.expid, args.output, args.list, args.filter_chunks, args.filter_status,
+ args.filter_type, args.hide, args.txt, args.group_by, args.expand,
+ args.expand_status, args.hide_groups, args.notransitive, args.check_wrapper, args.txt_logfiles)
+ elif args.command == 'stats':
+ return Autosubmit.statistics(args.expid, args.filter_type, args.filter_period, args.output, args.hide,
+ args.notransitive)
+ elif args.command == 'clean':
+ return Autosubmit.clean(args.expid, args.project, args.plot, args.stats)
+ elif args.command == 'recovery':
+ return Autosubmit.recovery(args.expid, args.noplot, args.save, args.all, args.hide, args.group_by,
+ args.expand, args.expand_status, args.notransitive, args.no_recover_logs)
+ elif args.command == 'check':
+ return Autosubmit.check(args.expid, args.notransitive)
+ elif args.command == 'inspect':
+ return Autosubmit.inspect(args.expid, args.list, args.filter_chunks, args.filter_status,
+ args.filter_type, args.notransitive, args.force, args.check_wrapper)
+ elif args.command == 'describe':
+ return Autosubmit.describe(args.expid)
+ elif args.command == 'migrate':
+ return Autosubmit.migrate(args.expid, args.offer, args.pickup)
+ elif args.command == 'create':
+ return Autosubmit.create(args.expid, args.noplot, args.hide, args.output, args.group_by, args.expand,
+ args.expand_status, args.notransitive, args.check_wrapper)
+ elif args.command == 'configure':
+ if not args.advanced or (args.advanced and dialog is None):
+ return Autosubmit.configure(args.advanced, args.databasepath, args.databasefilename,
+ args.localrootpath, args.platformsconfpath, args.jobsconfpath,
+ args.smtphostname, args.mailfrom, args.all, args.local)
+ else:
+ return Autosubmit.configure_dialog()
+ elif args.command == 'install':
+ return Autosubmit.install()
+ elif args.command == 'setstatus':
+ return Autosubmit.set_status(args.expid, args.noplot, args.save, args.status_final, args.list,
+ args.filter_chunks, args.filter_status, args.filter_type, args.hide,
+ args.group_by, args.expand, args.expand_status, args.notransitive, args.check_wrapper)
+ elif args.command == 'testcase':
+ return Autosubmit.testcase(args.copy, args.description, args.chunks, args.member, args.stardate,
+ args.HPC, args.branch)
+ elif args.command == 'test':
+ return Autosubmit.test(args.expid, args.chunks, args.member, args.stardate, args.HPC, args.branch)
+ elif args.command == 'refresh':
+ return Autosubmit.refresh(args.expid, args.model_conf, args.jobs_conf)
+ elif args.command == 'archive':
+ return Autosubmit.archive(args.expid)
+ elif args.command == 'unarchive':
+ return Autosubmit.unarchive(args.expid)
+
+ elif args.command == 'readme':
+ if os.path.isfile(Autosubmit.readme_path):
+ with open(Autosubmit.readme_path) as f:
+ print(f.read())
+ return True
+ return False
+ elif args.command == 'changelog':
+ if os.path.isfile(Autosubmit.changes_path):
+ with open(Autosubmit.changes_path) as f:
+ print(f.read())
+ return True
+ return False
+ except Exception as e:
+ from traceback import format_exc
+ Log.critical(
+ 'Unhandled exception on Autosubmit: {0}\n{1}', e, format_exc(10))
+
+ return False
+
+ @staticmethod
+ def _delete_expid(expid_delete):
+ """
+ Removes an experiment from path and database
+
+ :type expid_delete: str
+ :param expid_delete: identifier of the experiment to delete
+ """
+ if expid_delete == '' or expid_delete is None and not os.path.exists(os.path.join(BasicConfig.LOCAL_ROOT_DIR,
+ expid_delete)):
+ Log.info("Experiment directory does not exist.")
+ else:
+ Log.info("Removing experiment directory...")
+ ret = False
+ if pwd.getpwuid(os.stat(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid_delete)).st_uid).pw_name == os.getlogin():
+ try:
+
+ shutil.rmtree(os.path.join(
+ BasicConfig.LOCAL_ROOT_DIR, expid_delete))
+ except OSError as e:
+ Log.warning('Can not delete experiment folder: {0}', e)
+ return ret
+ Log.info("Deleting experiment from database...")
+ ret = delete_experiment(expid_delete)
+ if ret:
+ Log.result("Experiment {0} deleted".format(expid_delete))
+ else:
+ Log.warning(
+ "Current User is not the Owner {0} can not be deleted!", expid_delete)
+ return ret
+
+ @staticmethod
+ def expid(hpc, description, copy_id='', dummy=False, test=False, operational=False):
+ """
+ Creates a new experiment for given HPC
+
+ :param operational: if true, creates an operational experiment
+ :type operational: bool
+ :type hpc: str
+ :type description: str
+ :type copy_id: str
+ :type dummy: bool
+ :param hpc: name of the main HPC for the experiment
+ :param description: short experiment's description.
+ :param copy_id: experiment identifier of experiment to copy
+ :param dummy: if true, writes a default dummy configuration for testing
+ :param test: if true, creates an experiment for testing
+ :return: experiment identifier. If method fails, returns ''.
+ :rtype: str
+ """
+ BasicConfig.read()
+
+ log_path = os.path.join(
+ BasicConfig.LOCAL_ROOT_DIR, 'ASlogs', 'expid.log'.format(os.getuid()))
+ try:
+ Log.set_file(log_path)
+ except IOError as e:
+ Log.error("Can not create log file in path {0}: {1}".format(
+ log_path, e.message))
+ exp_id = None
+ if description is None:
+ Log.error("Missing experiment description.")
+ return ''
+ if hpc is None:
+ Log.error("Missing HPC.")
+ return ''
+ if not copy_id:
+ exp_id = new_experiment(
+ description, Autosubmit.autosubmit_version, test, operational)
+ if exp_id == '':
+ return ''
+ try:
+ os.mkdir(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id))
+
+ os.mkdir(os.path.join(
+ BasicConfig.LOCAL_ROOT_DIR, exp_id, 'conf'))
+ Log.info("Copying config files...")
+
+ # autosubmit config and experiment copied from AS.
+ files = resource_listdir('autosubmit.config', 'files')
+ for filename in files:
+ if resource_exists('autosubmit.config', 'files/' + filename):
+ index = filename.index('.')
+ new_filename = filename[:index] + \
+ "_" + exp_id + filename[index:]
+
+ if filename == 'platforms.conf' and BasicConfig.DEFAULT_PLATFORMS_CONF != '':
+ content = open(os.path.join(
+ BasicConfig.DEFAULT_PLATFORMS_CONF, filename)).read()
+ elif filename == 'jobs.conf' and BasicConfig.DEFAULT_JOBS_CONF != '':
+ content = open(os.path.join(
+ BasicConfig.DEFAULT_JOBS_CONF, filename)).read()
+ else:
+ content = resource_string(
+ 'autosubmit.config', 'files/' + filename)
+
+ conf_new_filename = os.path.join(
+ BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf", new_filename)
+ Log.debug(conf_new_filename)
+ open(conf_new_filename, 'w').write(content)
+ Autosubmit._prepare_conf_files(
+ exp_id, hpc, Autosubmit.autosubmit_version, dummy)
+ except (OSError, IOError) as e:
+ Log.error(
+ "Can not create experiment: {0}\nCleaning...".format(e))
+ Autosubmit._delete_expid(exp_id)
+ return ''
+ else:
+ try:
+ if os.path.exists(os.path.join(BasicConfig.LOCAL_ROOT_DIR, copy_id)):
+ exp_id = copy_experiment(
+ copy_id, description, Autosubmit.autosubmit_version, test, operational)
+ if exp_id == '':
+ return ''
+ dir_exp_id = os.path.join(
+ BasicConfig.LOCAL_ROOT_DIR, exp_id)
+ os.mkdir(dir_exp_id)
+ os.mkdir(dir_exp_id + '/conf')
+ Log.info("Copying previous experiment config directories")
+ conf_copy_id = os.path.join(
+ BasicConfig.LOCAL_ROOT_DIR, copy_id, "conf")
+ files = os.listdir(conf_copy_id)
+ for filename in files:
+ if os.path.isfile(os.path.join(conf_copy_id, filename)):
+ new_filename = filename.replace(copy_id, exp_id)
+ content = open(os.path.join(
+ conf_copy_id, filename), 'r').read()
+ open(os.path.join(dir_exp_id, "conf",
+ new_filename), 'w').write(content)
+ Autosubmit._prepare_conf_files(
+ exp_id, hpc, Autosubmit.autosubmit_version, dummy)
+ #####
+ autosubmit_config = AutosubmitConfig(
+ copy_id, BasicConfig, ConfigParserFactory())
+ if autosubmit_config.check_conf_files():
+ project_type = autosubmit_config.get_project_type()
+ if project_type == "git":
+ autosubmit_config.check_proj()
+ autosubmit_git = AutosubmitGit(copy_id[0])
+ Log.info("checking model version...")
+ if not autosubmit_git.check_commit(autosubmit_config):
+ return False
+ #####
+ else:
+ Log.critical(
+ "The previous experiment directory does not exist")
+ return ''
+ except (OSError, IOError) as e:
+ Log.error(
+ "Can not create experiment: {0}\nCleaning...".format(e))
+ Autosubmit._delete_expid(exp_id)
+ return ''
+
+ Log.debug("Creating temporal directory...")
+ exp_id_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id)
+ tmp_path = os.path.join(exp_id_path, "tmp")
+ os.mkdir(tmp_path)
+ os.chmod(tmp_path, 0o775)
+ os.mkdir(os.path.join(tmp_path, BasicConfig.LOCAL_ASLOG_DIR))
+ os.chmod(os.path.join(tmp_path, BasicConfig.LOCAL_ASLOG_DIR), 0o775)
+ Log.debug("Creating temporal remote directory...")
+ remote_tmp_path = os.path.join(tmp_path, "LOG_" + exp_id)
+ os.mkdir(remote_tmp_path)
+ os.chmod(remote_tmp_path, 0o775)
+
+ Log.debug("Creating pkl directory...")
+ os.mkdir(os.path.join(exp_id_path, "pkl"))
+
+ Log.debug("Creating plot directory...")
+ os.mkdir(os.path.join(exp_id_path, "plot"))
+ os.chmod(os.path.join(exp_id_path, "plot"), 0o775)
+ Log.result("Experiment registered successfully")
+ Log.user_warning("Remember to MODIFY the config files!")
+ return exp_id
+
+ @staticmethod
+ def delete(expid, force):
+ """
+ Deletes and experiment from database and experiment's folder
+
+ :type force: bool
+ :type expid: str
+ :param expid: identifier of the experiment to delete
+ :param force: if True, does not ask for confirmation
+
+ :returns: True if succesful, False if not
+ :rtype: bool
+ """
+ log_path = os.path.join(
+ BasicConfig.LOCAL_ROOT_DIR, "ASlogs", 'delete.log'.format(os.getuid()))
+ try:
+ Log.set_file(log_path)
+ except IOError as e:
+ Log.error("Can not create log file in path {0}: {1}".format(
+ log_path, e.message))
+
+ if os.path.exists(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid)):
+ if force or Autosubmit._user_yes_no_query("Do you want to delete " + expid + " ?"):
+ return Autosubmit._delete_expid(expid)
+ else:
+ Log.info("Quitting...")
+ return False
+ else:
+ Log.error("The experiment does not exist")
+ return True
+
+ @staticmethod
+ def _load_parameters(as_conf, job_list, platforms):
+ # Load parameters
+ Log.debug("Loading parameters...")
+ parameters = as_conf.load_parameters()
+ for platform_name in platforms:
+ platform = platforms[platform_name]
+ platform.add_parameters(parameters)
+
+ platform = platforms[as_conf.get_platform().lower()]
+ platform.add_parameters(parameters, True)
+
+ job_list.parameters = parameters
+
+ @staticmethod
+ def inspect(expid, lst, filter_chunks, filter_status, filter_section, notransitive=False, force=False, check_wrapper=False):
+ """
+ Generates cmd files experiment.
+
+ :type expid: str
+ :param expid: identifier of experiment to be run
+ :return: True if run to the end, False otherwise
+ :rtype: bool
+ """
+
+ if expid is None:
+ Log.critical("Missing experiment id")
+
+ BasicConfig.read()
+ exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid)
+ tmp_path = os.path.join(exp_path, BasicConfig.LOCAL_TMP_DIR)
+ if os.path.exists(os.path.join(tmp_path, 'autosubmit.lock')):
+ locked = True
+ else:
+ locked = False
+
+ if not os.path.exists(exp_path):
+ Log.critical(
+ "The directory %s is needed and does not exist" % exp_path)
+ Log.warning("Does an experiment with the given id exist?")
+ return 1
+ Log.info("Starting inspect command")
+ Log.set_file(os.path.join(
+ tmp_path, BasicConfig.LOCAL_ASLOG_DIR, 'generate.log'))
+ os.system('clear')
+ signal.signal(signal.SIGINT, signal_handler)
+ as_conf = AutosubmitConfig(expid, BasicConfig, ConfigParserFactory())
+ if not as_conf.check_conf_files():
+ Log.critical('Can not generate scripts with invalid configuration')
+ return False
+ project_type = as_conf.get_project_type()
+ if project_type != "none":
+ # Check proj configuration
+ as_conf.check_proj()
+ safetysleeptime = as_conf.get_safetysleeptime()
+ Log.debug("The Experiment name is: {0}", expid)
+ Log.debug("Sleep: {0}", safetysleeptime)
+ packages_persistence = JobPackagePersistence(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "pkl"),
+ "job_packages_" + expid)
+ os.chmod(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid,
+ "pkl", "job_packages_" + expid + ".db"), 0o664)
+
+ packages_persistence.reset_table(True)
+ job_list_original = Autosubmit.load_job_list(
+ expid, as_conf, notransitive=notransitive)
+ job_list = copy.deepcopy(job_list_original)
+ job_list.packages_dict = {}
+
+ Log.debug("Length of the jobs list: {0}", len(job_list))
+
+ # variables to be updated on the fly
+ safetysleeptime = as_conf.get_safetysleeptime()
+ Log.debug("Sleep: {0}", safetysleeptime)
+ # Generate
+ Log.info("Starting to generate cmd scripts")
+
+ if not isinstance(job_list, type([])):
+ jobs = []
+ jobs_cw = []
+ if check_wrapper and (not locked or (force and locked)):
+ Log.info("Generating all cmd script adapted for wrappers")
+ jobs = job_list.get_uncompleted()
+
+ jobs_cw = job_list.get_completed()
+ else:
+ if (force and not locked) or (force and locked):
+ Log.info("Overwritting all cmd scripts")
+ jobs = job_list.get_job_list()
+ elif locked:
+ Log.warning(
+ "There is a .lock file and not -f, generating only all unsubmitted cmd scripts")
+ jobs = job_list.get_unsubmitted()
+ else:
+ Log.info("Generating cmd scripts only for selected jobs")
+ if filter_chunks:
+ fc = filter_chunks
+ Log.debug(fc)
+ if fc == 'Any':
+ jobs = job_list.get_job_list()
+ else:
+ # noinspection PyTypeChecker
+ data = json.loads(Autosubmit._create_json(fc))
+ for date_json in data['sds']:
+ date = date_json['sd']
+ jobs_date = [j for j in job_list.get_job_list() if date2str(
+ j.date) == date]
+
+ for member_json in date_json['ms']:
+ member = member_json['m']
+ jobs_member = [j for j in jobs_date if j.member == member]
+
+ for chunk_json in member_json['cs']:
+ chunk = int(chunk_json)
+ jobs = jobs + \
+ [job for job in [j for j in jobs_member if j.chunk == chunk]]
+
+ elif filter_status:
+ Log.debug(
+ "Filtering jobs with status {0}", filter_status)
+ if filter_status == 'Any':
+ jobs = job_list.get_job_list()
+ else:
+ fs = Autosubmit._get_status(filter_status)
+ jobs = [job for job in [j for j in job_list.get_job_list() if j.status == fs]]
+
+ elif filter_section:
+ ft = filter_section
+ Log.debug(ft)
+
+ if ft == 'Any':
+ jobs = job_list.get_job_list()
+ else:
+ for job in job_list.get_job_list():
+ if job.section == ft:
+ jobs.append(job)
+ elif lst:
+ jobs_lst = lst.split()
+
+ if jobs == 'Any':
+ jobs = job_list.get_job_list()
+ else:
+ for job in job_list.get_job_list():
+ if job.name in jobs_lst:
+ jobs.append(job)
+ else:
+ jobs = job_list.get_job_list()
+ if isinstance(jobs, type([])):
+ referenced_jobs_to_remove = set()
+ for job in jobs:
+ for child in job.children:
+ if child not in jobs:
+ referenced_jobs_to_remove.add(child)
+ for parent in job.parents:
+ if parent not in jobs:
+ referenced_jobs_to_remove.add(parent)
+
+ for job in jobs:
+ job.status = Status.WAITING
+
+ Autosubmit.generate_scripts_andor_wrappers(
+ as_conf, job_list, jobs, packages_persistence, False)
+ if len(jobs_cw) > 0:
+ referenced_jobs_to_remove = set()
+ for job in jobs_cw:
+ for child in job.children:
+ if child not in jobs_cw:
+ referenced_jobs_to_remove.add(child)
+ for parent in job.parents:
+ if parent not in jobs_cw:
+ referenced_jobs_to_remove.add(parent)
+
+ for job in jobs_cw:
+ job.status = Status.WAITING
+ Autosubmit.generate_scripts_andor_wrappers(
+ as_conf, job_list, jobs_cw, packages_persistence, False)
+
+ Log.info("no more scripts to generate, now proceed to check them manually")
+ time.sleep(safetysleeptime)
+ return True
+
+ @staticmethod
+ def generate_scripts_andor_wrappers(as_conf, job_list, jobs_filtered, packages_persistence, only_wrappers=False):
+ """
+ as_conf: AutosubmitConfig object
+ job_list: JobList object, contains a list of jobs
+ jobs_filtered: list of jobs
+ packages_persistence: Database handler
+ only_wrappers: True
+ """
+ job_list._job_list = jobs_filtered
+ job_list.update_list(as_conf, False)
+ # Identifying the submitter and loading it
+ submitter = Autosubmit._get_submitter(as_conf)
+ # Function depending on the submitter
+ submitter.load_platforms(as_conf)
+ # Identifying HPC from config files
+ hpcarch = as_conf.get_platform()
+ #
+ Autosubmit._load_parameters(as_conf, job_list, submitter.platforms)
+ platforms_to_test = set()
+ for job in job_list.get_job_list():
+ if job.platform_name is None:
+ job.platform_name = hpcarch
+ # noinspection PyTypeChecker
+ job.platform = submitter.platforms[job.platform_name.lower()]
+ # noinspection PyTypeChecker
+ platforms_to_test.add(job.platform)
+ # case setstatus
+ job_list.check_scripts(as_conf)
+ job_list.update_list(as_conf, False)
+ Autosubmit._load_parameters(as_conf, job_list, submitter.platforms)
+ while job_list.get_active():
+ Autosubmit.submit_ready_jobs(
+ as_conf, job_list, platforms_to_test, packages_persistence, True, only_wrappers)
+
+ job_list.update_list(as_conf, False)
+
+ @staticmethod
+ def run_experiment(expid, notransitive=False):
+ """
+ Runs and experiment (submitting all the jobs properly and repeating its execution in case of failure).
+
+ :type expid: str
+ :param expid: identifier of experiment to be run
+ :return: True if run to the end, False otherwise
+ :rtype: bool
+ """
+ if expid is None:
+ Log.critical("Missing experiment id")
+
+ BasicConfig.read()
+ exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid)
+ tmp_path = os.path.join(exp_path, BasicConfig.LOCAL_TMP_DIR)
+ aslogs_path = os.path.join(tmp_path, BasicConfig.LOCAL_ASLOG_DIR)
+ if not os.path.exists(aslogs_path):
+ os.mkdir(aslogs_path)
+ os.chmod(aslogs_path, 0o775)
+ if not os.path.exists(exp_path):
+ Log.critical(
+ "The directory %s is needed and does not exist" % exp_path)
+ Log.warning("Does an experiment with the given id exist?")
+ return 1
+
+ # checking host whitelist
+ import platform
+ host = platform.node()
+ print(host)
+ if BasicConfig.ALLOWED_HOSTS and host not in BasicConfig.ALLOWED_HOSTS:
+ Log.info("\n Autosubmit run command is not allowed on this host")
+ return False
+
+ # checking if there is a lock file to avoid multiple running on the same expid
+ try:
+ with portalocker.Lock(os.path.join(tmp_path, 'autosubmit.lock'), timeout=1):
+ Log.info(
+ "Preparing .lock file to avoid multiple instances with same experiment id")
+
+ Log.set_file(os.path.join(aslogs_path, 'run.log'))
+ os.system('clear')
+
+ signal.signal(signal.SIGINT, signal_handler)
+
+ as_conf = AutosubmitConfig(
+ expid, BasicConfig, ConfigParserFactory())
+ if not as_conf.check_conf_files():
+ Log.critical('Can not run with invalid configuration')
+ return False
+
+ project_type = as_conf.get_project_type()
+ if project_type != "none":
+ # Check proj configuration
+ as_conf.check_proj()
+
+ hpcarch = as_conf.get_platform()
+
+ safetysleeptime = as_conf.get_safetysleeptime()
+ retrials = as_conf.get_retrials()
+
+ submitter = Autosubmit._get_submitter(as_conf)
+ submitter.load_platforms(as_conf)
+
+ Log.debug("The Experiment name is: {0}", expid)
+ Log.debug("Sleep: {0}", safetysleeptime)
+ Log.debug("Default retrials: {0}", retrials)
+
+ Log.info("Starting job submission...")
+
+ pkl_dir = os.path.join(
+ BasicConfig.LOCAL_ROOT_DIR, expid, 'pkl')
+ job_list = Autosubmit.load_job_list(
+ expid, as_conf, notransitive=notransitive)
+
+ Log.debug(
+ "Starting from job list restored from {0} files", pkl_dir)
+
+ Log.debug("Length of the jobs list: {0}", len(job_list))
+
+ Autosubmit._load_parameters(
+ as_conf, job_list, submitter.platforms)
+
+ # check the job list script creation
+ Log.debug("Checking experiment templates...")
+
+ platforms_to_test = set()
+ for job in job_list.get_job_list():
+ if job.platform_name is None:
+ job.platform_name = hpcarch
+ # noinspection PyTypeChecker
+ job.platform = submitter.platforms[job.platform_name.lower(
+ )]
+ # noinspection PyTypeChecker
+ platforms_to_test.add(job.platform)
+
+ job_list.check_scripts(as_conf)
+
+ packages_persistence = JobPackagePersistence(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "pkl"),
+ "job_packages_" + expid)
+
+ if as_conf.get_wrapper_type() != 'none':
+ os.chmod(os.path.join(BasicConfig.LOCAL_ROOT_DIR,
+ expid, "pkl", "job_packages_" + expid + ".db"), 0o664)
+ packages = packages_persistence.load()
+ for (exp_id, package_name, job_name) in packages:
+ if package_name not in job_list.packages_dict:
+ job_list.packages_dict[package_name] = []
+ job_list.packages_dict[package_name].append(
+ job_list.get_job_by_name(job_name))
+
+ for package_name, jobs in list(job_list.packages_dict.items()):
+ from job.job import WrapperJob
+ wrapper_job = WrapperJob(package_name, jobs[0].id, Status.SUBMITTED, 0, jobs,
+ None,
+ None, jobs[0].platform, as_conf)
+ job_list.job_package_map[jobs[0].id] = wrapper_job
+ job_list.update_list(as_conf)
+ job_list.save()
+ #########################
+ # AUTOSUBMIT - MAIN LOOP
+ #########################
+ # Main loop. Finishing when all jobs have been submitted
+ while job_list.get_active():
+ if Autosubmit.exit:
+ return 2
+ # reload parameters changes
+ Log.debug("Reloading parameters...")
+ as_conf.reload()
+ Autosubmit._load_parameters(
+ as_conf, job_list, submitter.platforms)
+ # variables to be updated on the fly
+ total_jobs = len(job_list.get_job_list())
+ Log.info(
+ "\n\n{0} of {1} jobs remaining ({2})".format(total_jobs - len(job_list.get_completed()),
+ total_jobs,
+ time.strftime("%H:%M")))
+ safetysleeptime = as_conf.get_safetysleeptime()
+ Log.debug("Sleep: {0}", safetysleeptime)
+ default_retrials = as_conf.get_retrials()
+ Log.debug("Number of retrials: {0}", default_retrials)
+
+ check_wrapper_jobs_sleeptime = as_conf.get_wrapper_check_time()
+ Log.debug('WRAPPER CHECK TIME = {0}'.format(
+ check_wrapper_jobs_sleeptime))
+
+ save = False
+
+ slurm = []
+ for platform in platforms_to_test:
+ list_jobid = ""
+ completed_joblist = []
+ list_prevStatus = []
+ queuing_jobs = job_list.get_in_queue_grouped_id(
+ platform)
+ for job_id, job in list(queuing_jobs.items()):
+ if job_list.job_package_map and job_id in job_list.job_package_map:
+ Log.debug(
+ 'Checking wrapper job with id ' + str(job_id))
+ wrapper_job = job_list.job_package_map[job_id]
+ check_wrapper = True
+ if wrapper_job.status == Status.RUNNING:
+ check_wrapper = True if datetime.timedelta.total_seconds(datetime.datetime.now(
+ ) - wrapper_job.checked_time) >= check_wrapper_jobs_sleeptime else False
+ if check_wrapper:
+ wrapper_job.checked_time = datetime.datetime.now()
+ platform.check_job(wrapper_job)
+ Log.info(
+ 'Wrapper job ' + wrapper_job.name + ' is ' + str(Status.VALUE_TO_KEY[wrapper_job.new_status]))
+ wrapper_job.check_status(
+ wrapper_job.new_status)
+ save = True
+ else:
+ Log.info(
+ "Waiting for wrapper check time: {0}\n", check_wrapper_jobs_sleeptime)
+ else:
+ job = job[0]
+ prev_status = job.status
+ if job.status == Status.FAILED:
+ continue
+
+ if platform.type == "slurm":
+ list_jobid += str(job_id) + ','
+ list_prevStatus.append(prev_status)
+ completed_joblist.append(job)
+ else:
+ platform.check_job(job)
+ if prev_status != job.update_status(as_conf.get_copy_remote_logs() == 'true'):
+ if as_conf.get_notifications() == 'true':
+ if Status.VALUE_TO_KEY[job.status] in job.notify_on:
+ Notifier.notify_status_change(MailNotifier(BasicConfig), expid, job.name,
+ Status.VALUE_TO_KEY[prev_status],
+ Status.VALUE_TO_KEY[job.status],
+ as_conf.get_mails_to())
+ save = True
+
+ if platform.type == "slurm" and list_jobid != "":
+ slurm.append(
+ [platform, list_jobid, list_prevStatus, completed_joblist])
+ # END LOOP
+ for platform_jobs in slurm:
+ platform = platform_jobs[0]
+ jobs_to_check = platform_jobs[1]
+ platform.check_Alljobs(
+ platform_jobs[3], jobs_to_check, as_conf.get_copy_remote_logs())
+
+ for j_Indx in range(0, len(platform_jobs[3])):
+ prev_status = platform_jobs[2][j_Indx]
+ job = platform_jobs[3][j_Indx]
+
+ if prev_status != job.update_status(as_conf.get_copy_remote_logs() == 'true'):
+ if as_conf.get_notifications() == 'true':
+ if Status.VALUE_TO_KEY[job.status] in job.notify_on:
+ Notifier.notify_status_change(MailNotifier(BasicConfig), expid, job.name,
+ Status.VALUE_TO_KEY[prev_status],
+ Status.VALUE_TO_KEY[job.status],
+ as_conf.get_mails_to())
+ save = True
+
+ if job_list.update_list(as_conf) or save:
+ job_list.save()
+
+ if Autosubmit.submit_ready_jobs(as_conf, job_list, platforms_to_test, packages_persistence):
+ job_list.save()
+
+ if Autosubmit.exit:
+ return 2
+ time.sleep(safetysleeptime)
+
+ Log.info("No more jobs to run.")
+ if len(job_list.get_failed()) > 0:
+ Log.info("Some jobs have failed and reached maximum retrials")
+ return False
+ else:
+ Log.result("Run successful")
+ return True
+
+ except portalocker.AlreadyLocked:
+ Autosubmit.show_lock_warning(expid)
+
+ except WrongTemplateException:
+ return False
+
+ @staticmethod
+ def submit_ready_jobs(as_conf, job_list, platforms_to_test, packages_persistence, inspect=False,
+ only_wrappers=False):
+ """
+ Gets READY jobs and send them to the platforms if there is available space on the queues
+
+ :param as_conf: autosubmit config object. \n
+ :type as_conf: AutosubmitConfig Object. \n
+ :param job_list: JobList as a single entity. \n
+ :type job_list: JobList() Object. \n
+ :param platforms_to_test: List of platforms that will be used in the experiment. \n
+ :type platforms_to_test: Set() of Platform() Object. e.g. EcPlatform(), LsfPlatform(), etc. \n
+ :return: True if at least one job was submitted, False otherwise
+ :rtype: bool
+ """
+ save = False
+
+ for platform in platforms_to_test:
+ Log.debug("\nJobs ready for {1}: {0}", len(
+ job_list.get_ready(platform)), platform.name)
+ packages_to_submit, remote_dependencies_dict = JobPackager(
+ as_conf, platform, job_list).build_packages()
+ if not inspect:
+ platform.open_submit_script()
+ valid_packages_to_submit = []
+ for package in packages_to_submit:
+ try:
+ if hasattr(package, "name"):
+ if remote_dependencies_dict and package.name in remote_dependencies_dict['dependencies']:
+ remote_dependency = remote_dependencies_dict['dependencies'][package.name]
+ remote_dependency_id = remote_dependencies_dict['name_to_id'][remote_dependency]
+ package.set_job_dependency(remote_dependency_id)
+ if not only_wrappers:
+ try:
+ package.submit(
+ as_conf, job_list.parameters, inspect)
+ valid_packages_to_submit.append(package)
+ except (IOError, OSError):
+ # write error file
+ continue
+ if only_wrappers or inspect:
+ for innerJob in package._jobs:
+ innerJob.status = Status.COMPLETED
+
+ if hasattr(package, "name"):
+ job_list.packages_dict[package.name] = package.jobs
+ from job.job import WrapperJob
+ wrapper_job = WrapperJob(package.name, package.jobs[0].id, Status.READY, 0,
+ package.jobs,
+ package._wallclock, package._num_processors,
+ package.platform, as_conf)
+ job_list.job_package_map[package.jobs[0].id] = wrapper_job
+ if remote_dependencies_dict and package.name in remote_dependencies_dict['name_to_id']:
+ remote_dependencies_dict['name_to_id'][package.name] = package.jobs[0].id
+ if isinstance(package, JobPackageThread):
+ packages_persistence.save(
+ package.name, package.jobs, package._expid, inspect)
+ save = True
+ except WrongTemplateException as e:
+ Log.error(
+ "Invalid parameter substitution in {0} template", e.job_name)
+ raise
+ except Exception:
+ Log.error(
+ "{0} submission failed due to Unknown error", platform.name)
+ raise
+
+ if platform.type == "slurm" and not inspect and not only_wrappers:
+ try:
+ save = True
+ if len(valid_packages_to_submit) > 0:
+ jobs_id = platform.submit_Script()
+ if jobs_id is None:
+ raise BaseException(
+ "Exiting AS being unable to get jobID")
+ i = 0
+ for package in valid_packages_to_submit:
+ for job in package.jobs:
+ job.id = str(jobs_id[i])
+ Log.info("{0} submitted", job.name)
+ job.status = Status.SUBMITTED
+ job.write_submit_time()
+ if hasattr(package, "name"):
+ job_list.packages_dict[package.name] = package.jobs
+ from job.job import WrapperJob
+ wrapper_job = WrapperJob(package.name, package.jobs[0].id, Status.SUBMITTED, 0,
+ package.jobs,
+ package._wallclock, package._num_processors,
+ package.platform, as_conf)
+ job_list.job_package_map[package.jobs[0].id] = wrapper_job
+ if remote_dependencies_dict and package.name in remote_dependencies_dict[
+ 'name_to_id']:
+ remote_dependencies_dict['name_to_id'][package.name] = package.jobs[0].id
+ if isinstance(package, JobPackageThread):
+ packages_persistence.save(
+ package.name, package.jobs, package._expid, inspect)
+ i += 1
+
+ except WrongTemplateException as e:
+ Log.error(
+ "Invalid parameter substitution in {0} template", e.job_name)
+ raise
+ except Exception:
+ Log.error("{0} submission failed", platform.name)
+ raise
+
+ return save
+
+ @staticmethod
+ def monitor(expid, file_format, lst, filter_chunks, filter_status, filter_section, hide, txt_only=False,
+ group_by=None, expand=list(), expand_status=list(), hide_groups=False, notransitive=False, check_wrapper=False, txt_logfiles=False):
+ """
+ Plots workflow graph for a given experiment with status of each job coded by node color.
+ Plot is created in experiment's plot folder with name __