diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 6769e21d99a63338394e47bc4c7d0aba1e88d5a5..0000000000000000000000000000000000000000 --- a/.gitignore +++ /dev/null @@ -1,160 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/#use-with-ide -.pdm.toml - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ \ No newline at end of file diff --git a/autosubmit_api.egg-info/PKG-INFO b/autosubmit_api.egg-info/PKG-INFO new file mode 100644 index 0000000000000000000000000000000000000000..31c238680622f81390d66c097e13a3e991728e12 --- /dev/null +++ b/autosubmit_api.egg-info/PKG-INFO @@ -0,0 +1,14 @@ +Metadata-Version: 2.1 +Name: autosubmit-api +Version: 1.0.27 +Summary: An extension to the Autosubmit package that serves its information as an API +Home-page: https://earth.bsc.es/gitlab/wuruchi/autosubmit_api +Author: Wilmer Uruchi +Author-email: wilmer.uruchi@bsc.es +License: GNU GPL +Keywords: autosubmit,API +Classifier: Development Status :: 4 - Beta +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) +Classifier: Operating System :: POSIX :: Linux +Classifier: Programming Language :: Python :: 3.7 diff --git a/autosubmit_api.egg-info/SOURCES.txt b/autosubmit_api.egg-info/SOURCES.txt new file mode 100644 index 0000000000000000000000000000000000000000..f0ccb7254c04d36adeeb15db22c11d7fdce57470 --- /dev/null +++ b/autosubmit_api.egg-info/SOURCES.txt @@ -0,0 +1,161 @@ +setup.py +autosubmit_api/__init__.py +autosubmit_api/app.py +autosubmit_api/get-pip.py +autosubmit_api.egg-info/PKG-INFO +autosubmit_api.egg-info/SOURCES.txt +autosubmit_api.egg-info/dependency_links.txt +autosubmit_api.egg-info/requires.txt +autosubmit_api.egg-info/top_level.txt +autosubmit_api/autosubmit_legacy/__init__.py +autosubmit_api/autosubmit_legacy/autosubmit.py +autosubmit_api/autosubmit_legacy/job/__init__.py +autosubmit_api/autosubmit_legacy/job/job.py +autosubmit_api/autosubmit_legacy/job/job_common.py +autosubmit_api/autosubmit_legacy/job/job_dict.py +autosubmit_api/autosubmit_legacy/job/job_exceptions.py +autosubmit_api/autosubmit_legacy/job/job_grouping.py +autosubmit_api/autosubmit_legacy/job/job_list.py +autosubmit_api/autosubmit_legacy/job/job_list_persistence.py +autosubmit_api/autosubmit_legacy/job/job_package_persistence.py +autosubmit_api/autosubmit_legacy/job/job_packager.py +autosubmit_api/autosubmit_legacy/job/job_packages.py +autosubmit_api/autosubmit_legacy/job/job_utils.py +autosubmit_api/autosubmit_legacy/platforms/__init__.py +autosubmit_api/autosubmit_legacy/platforms/ecmwf_adaptor.py +autosubmit_api/autosubmit_legacy/platforms/ecplatform.py +autosubmit_api/autosubmit_legacy/platforms/locplatform.py +autosubmit_api/autosubmit_legacy/platforms/lsfplatform.py +autosubmit_api/autosubmit_legacy/platforms/mn_adaptor.py +autosubmit_api/autosubmit_legacy/platforms/paramiko_platform.py +autosubmit_api/autosubmit_legacy/platforms/paramiko_submitter.py +autosubmit_api/autosubmit_legacy/platforms/pbsplatform.py +autosubmit_api/autosubmit_legacy/platforms/platform.py +autosubmit_api/autosubmit_legacy/platforms/psplatform.py +autosubmit_api/autosubmit_legacy/platforms/saga_platform.py +autosubmit_api/autosubmit_legacy/platforms/saga_submitter.py +autosubmit_api/autosubmit_legacy/platforms/sgeplatform.py +autosubmit_api/autosubmit_legacy/platforms/slurmplatform.py +autosubmit_api/autosubmit_legacy/platforms/submitter.py +autosubmit_api/autosubmit_legacy/platforms/headers/__init__.py +autosubmit_api/autosubmit_legacy/platforms/headers/ec_cca_header.py +autosubmit_api/autosubmit_legacy/platforms/headers/ec_header.py +autosubmit_api/autosubmit_legacy/platforms/headers/local_header.py +autosubmit_api/autosubmit_legacy/platforms/headers/lsf_header.py +autosubmit_api/autosubmit_legacy/platforms/headers/pbs10_header.py +autosubmit_api/autosubmit_legacy/platforms/headers/pbs11_header.py +autosubmit_api/autosubmit_legacy/platforms/headers/pbs12_header.py +autosubmit_api/autosubmit_legacy/platforms/headers/ps_header.py +autosubmit_api/autosubmit_legacy/platforms/headers/sge_header.py +autosubmit_api/autosubmit_legacy/platforms/headers/slurm_header.py +autosubmit_api/autosubmit_legacy/platforms/wrappers/__init__.py +autosubmit_api/autosubmit_legacy/platforms/wrappers/wrapper_builder.py +autosubmit_api/autosubmit_legacy/platforms/wrappers/wrapper_factory.py +autosubmit_api/builders/__init__.py +autosubmit_api/builders/basic_builder.py +autosubmit_api/builders/configuration_facade_builder.py +autosubmit_api/builders/experiment_history_builder.py +autosubmit_api/builders/joblist_helper_builder.py +autosubmit_api/builders/joblist_loader_builder.py +autosubmit_api/builders/pkl_organizer_builder.py +autosubmit_api/common/__init__.py +autosubmit_api/common/utils.py +autosubmit_api/common/utils_for_testing.py +autosubmit_api/components/__init__.py +autosubmit_api/components/experiment/__init__.py +autosubmit_api/components/experiment/configuration_facade.py +autosubmit_api/components/experiment/pkl_organizer.py +autosubmit_api/components/experiment/test.py +autosubmit_api/components/jobs/__init__.py +autosubmit_api/components/jobs/job_factory.py +autosubmit_api/components/jobs/job_support.py +autosubmit_api/components/jobs/joblist_helper.py +autosubmit_api/components/jobs/joblist_loader.py +autosubmit_api/components/jobs/test.py +autosubmit_api/components/jobs/utils.py +autosubmit_api/components/representations/__init__.py +autosubmit_api/components/representations/graph/__init__.py +autosubmit_api/components/representations/graph/edge.py +autosubmit_api/components/representations/graph/graph.py +autosubmit_api/components/representations/graph/test.py +autosubmit_api/components/representations/tree/__init__.py +autosubmit_api/components/representations/tree/test.py +autosubmit_api/components/representations/tree/tree.py +autosubmit_api/config/__init__.py +autosubmit_api/config/basicConfig.py +autosubmit_api/config/config_common.py +autosubmit_api/database/__init__.py +autosubmit_api/database/autosubmit.py +autosubmit_api/database/db_common.py +autosubmit_api/database/db_jobdata.py +autosubmit_api/database/db_manager.py +autosubmit_api/database/db_structure.py +autosubmit_api/experiment/__init__.py +autosubmit_api/experiment/as_times_db_manager.py +autosubmit_api/experiment/common_db_requests.py +autosubmit_api/experiment/common_requests.py +autosubmit_api/experiment/experiment_common.py +autosubmit_api/experiment/experiment_db_manager.py +autosubmit_api/experiment/test.py +autosubmit_api/experiment/utils.py +autosubmit_api/git/__init__.py +autosubmit_api/git/autosubmit_git.py +autosubmit_api/history/__init__.py +autosubmit_api/history/experiment_history.py +autosubmit_api/history/experiment_status.py +autosubmit_api/history/experiment_status_manager.py +autosubmit_api/history/internal_logging.py +autosubmit_api/history/strategies.py +autosubmit_api/history/test.py +autosubmit_api/history/test_job_history.py +autosubmit_api/history/test_strategies.py +autosubmit_api/history/test_utils.py +autosubmit_api/history/utils.py +autosubmit_api/history/data_classes/__init__.py +autosubmit_api/history/data_classes/experiment_run.py +autosubmit_api/history/data_classes/job_data.py +autosubmit_api/history/database_managers/__init__.py +autosubmit_api/history/database_managers/database_manager.py +autosubmit_api/history/database_managers/database_models.py +autosubmit_api/history/database_managers/experiment_history_db_manager.py +autosubmit_api/history/database_managers/experiment_status_db_manager.py +autosubmit_api/history/database_managers/test.py +autosubmit_api/history/platform_monitor/__init__.py +autosubmit_api/history/platform_monitor/platform_monitor.py +autosubmit_api/history/platform_monitor/platform_utils.py +autosubmit_api/history/platform_monitor/slurm_monitor.py +autosubmit_api/history/platform_monitor/slurm_monitor_item.py +autosubmit_api/history/platform_monitor/test.py +autosubmit_api/monitor/__init__.py +autosubmit_api/monitor/diagram.py +autosubmit_api/monitor/monitor.py +autosubmit_api/monitor/utils.py +autosubmit_api/notifications/__init__.py +autosubmit_api/notifications/mail_notifier.py +autosubmit_api/notifications/notifier.py +autosubmit_api/performance/__init__.py +autosubmit_api/performance/performance_metrics.py +autosubmit_api/performance/utils.py +autosubmit_api/statistics/__init__.py +autosubmit_api/statistics/job_stat.py +autosubmit_api/statistics/statistics.py +autosubmit_api/statistics/stats_summary.py +autosubmit_api/statistics/test.py +autosubmit_api/statistics/utils.py +autosubmit_api/workers/__init__.py +autosubmit_api/workers/populate_details_db.py +autosubmit_api/workers/populate_graph.py +autosubmit_api/workers/populate_queue_run_times.py +autosubmit_api/workers/populate_running_experiments.py +autosubmit_api/workers/test.py +autosubmit_api/workers/test_esarchive.py +autosubmit_api/workers/verify_complete.py +autosubmit_api/workers/business/__init__.py +autosubmit_api/workers/business/populate_times.py +autosubmit_api/workers/business/process_graph_drawings.py +autosubmit_api/workers/deprecated/__init__.py +autosubmit_api/workers/deprecated/fix_historic.py +autosubmit_api/workers/deprecated/fix_historic_energy.py +autosubmit_api/workers/populate_details/__init__.py +autosubmit_api/workers/populate_details/populate.py +autosubmit_api/workers/populate_details/test.py \ No newline at end of file diff --git a/autosubmit_api.egg-info/dependency_links.txt b/autosubmit_api.egg-info/dependency_links.txt new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/autosubmit_api.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/autosubmit_api.egg-info/requires.txt b/autosubmit_api.egg-info/requires.txt new file mode 100644 index 0000000000000000000000000000000000000000..d8ff9e05a6fd86e529fee22dcb82571fd4c49046 --- /dev/null +++ b/autosubmit_api.egg-info/requires.txt @@ -0,0 +1,11 @@ +jwt==1.3.1 +requests==2.28.1 +flask_cors==3.0.10 +bscearth.utils==0.5.2 +pysqlite3==0.4.7 +numpy==1.21.6 +pydotplus==2.0.2 +portalocker==2.6.0 +networkx==2.6.3 +scipy==1.7.3 +paramiko==2.12.0 diff --git a/autosubmit_api.egg-info/top_level.txt b/autosubmit_api.egg-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..7ecd916442b11be3fd54d6ad68e3a2ef05838217 --- /dev/null +++ b/autosubmit_api.egg-info/top_level.txt @@ -0,0 +1 @@ +autosubmit_api diff --git a/autosubmit_api/__init__.pyc b/autosubmit_api/__init__.pyc new file mode 100644 index 0000000000000000000000000000000000000000..224467877f8104b9b5e89aa172bda46b60e446b9 Binary files /dev/null and b/autosubmit_api/__init__.pyc differ diff --git a/autosubmit_api/app.pyc b/autosubmit_api/app.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f9d49de8c66b7fcd5eac888dc667509e09d6ed79 Binary files /dev/null and b/autosubmit_api/app.pyc differ diff --git a/autosubmit_api/autosubmit_api.egg-info/PKG-INFO b/autosubmit_api/autosubmit_api.egg-info/PKG-INFO new file mode 100644 index 0000000000000000000000000000000000000000..754166785604aece4732b5dbe08990418dfd8e2c --- /dev/null +++ b/autosubmit_api/autosubmit_api.egg-info/PKG-INFO @@ -0,0 +1,16 @@ +Metadata-Version: 1.1 +Name: autosubmit-api +Version: 1.0.0 +Summary: An extension to the Autosubmit package that serves its information as an API +Home-page: https://earth.bsc.es/gitlab/wuruchi/autosubmit_api +Author: Wilmer Uruchi +Author-email: wilmer.uruchi@bsc.es +License: GNU GPL +Description: UNKNOWN +Keywords: autosubmit,API +Platform: UNKNOWN +Classifier: Development Status :: 4 - Beta +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) +Classifier: Operating System :: POSIX :: Linux +Classifier: Programming Language :: Python :: 3.7 diff --git a/autosubmit_api/autosubmit_api.egg-info/SOURCES.txt b/autosubmit_api/autosubmit_api.egg-info/SOURCES.txt new file mode 100644 index 0000000000000000000000000000000000000000..861769ff0cfc65c3d1facd06b41e6c194716d681 --- /dev/null +++ b/autosubmit_api/autosubmit_api.egg-info/SOURCES.txt @@ -0,0 +1,156 @@ +.egg-info/PKG-INFO +.egg-info/SOURCES.txt +.egg-info/dependency_links.txt +.egg-info/requires.txt +.egg-info/top_level.txt +autosubmit_legacy/__init__.py +autosubmit_legacy/autosubmit.py +autosubmit_legacy/job/__init__.py +autosubmit_legacy/job/job.py +autosubmit_legacy/job/job_common.py +autosubmit_legacy/job/job_dict.py +autosubmit_legacy/job/job_exceptions.py +autosubmit_legacy/job/job_grouping.py +autosubmit_legacy/job/job_list.py +autosubmit_legacy/job/job_list_persistence.py +autosubmit_legacy/job/job_package_persistence.py +autosubmit_legacy/job/job_packager.py +autosubmit_legacy/job/job_packages.py +autosubmit_legacy/job/job_utils.py +autosubmit_legacy/platforms/__init__.py +autosubmit_legacy/platforms/ecmwf_adaptor.py +autosubmit_legacy/platforms/ecplatform.py +autosubmit_legacy/platforms/locplatform.py +autosubmit_legacy/platforms/lsfplatform.py +autosubmit_legacy/platforms/mn_adaptor.py +autosubmit_legacy/platforms/paramiko_platform.py +autosubmit_legacy/platforms/paramiko_submitter.py +autosubmit_legacy/platforms/pbsplatform.py +autosubmit_legacy/platforms/platform.py +autosubmit_legacy/platforms/psplatform.py +autosubmit_legacy/platforms/saga_platform.py +autosubmit_legacy/platforms/saga_submitter.py +autosubmit_legacy/platforms/sgeplatform.py +autosubmit_legacy/platforms/slurmplatform.py +autosubmit_legacy/platforms/submitter.py +autosubmit_legacy/platforms/headers/__init__.py +autosubmit_legacy/platforms/headers/ec_cca_header.py +autosubmit_legacy/platforms/headers/ec_header.py +autosubmit_legacy/platforms/headers/local_header.py +autosubmit_legacy/platforms/headers/lsf_header.py +autosubmit_legacy/platforms/headers/pbs10_header.py +autosubmit_legacy/platforms/headers/pbs11_header.py +autosubmit_legacy/platforms/headers/pbs12_header.py +autosubmit_legacy/platforms/headers/ps_header.py +autosubmit_legacy/platforms/headers/sge_header.py +autosubmit_legacy/platforms/headers/slurm_header.py +autosubmit_legacy/platforms/wrappers/__init__.py +autosubmit_legacy/platforms/wrappers/wrapper_builder.py +autosubmit_legacy/platforms/wrappers/wrapper_factory.py +builders/__init__.py +builders/basic_builder.py +builders/configuration_facade_builder.py +builders/experiment_history_builder.py +builders/joblist_helper_builder.py +builders/joblist_loader_builder.py +builders/pkl_organizer_builder.py +common/__init__.py +common/utils.py +common/utils_for_testing.py +components/__init__.py +components/experiment/__init__.py +components/experiment/configuration_facade.py +components/experiment/pkl_organizer.py +components/experiment/test.py +components/jobs/__init__.py +components/jobs/job_factory.py +components/jobs/job_support.py +components/jobs/joblist_helper.py +components/jobs/joblist_loader.py +components/jobs/test.py +components/jobs/utils.py +components/representations/__init__.py +components/representations/graph/__init__.py +components/representations/graph/edge.py +components/representations/graph/graph.py +components/representations/graph/test.py +components/representations/tree/__init__.py +components/representations/tree/test.py +components/representations/tree/tree.py +config/__init__.py +config/basicConfig.py +config/config_common.py +database/__init__.py +database/db_common.py +database/db_jobdata.py +database/db_manager.py +database/db_structure.py +experiment/__init__.py +experiment/as_times_db_manager.py +experiment/common_db_requests.py +experiment/common_requests.py +experiment/experiment_common.py +experiment/experiment_db_manager.py +experiment/test.py +experiment/utils.py +git/__init__.py +git/autosubmit_git.py +history/__init__.py +history/experiment_history.py +history/experiment_status.py +history/experiment_status_manager.py +history/internal_logging.py +history/strategies.py +history/test.py +history/test_job_history.py +history/test_strategies.py +history/test_utils.py +history/utils.py +history/data_classes/__init__.py +history/data_classes/experiment_run.py +history/data_classes/job_data.py +history/database_managers/__init__.py +history/database_managers/database_manager.py +history/database_managers/database_models.py +history/database_managers/experiment_history_db_manager.py +history/database_managers/experiment_status_db_manager.py +history/database_managers/test.py +history/platform_monitor/__init__.py +history/platform_monitor/platform_monitor.py +history/platform_monitor/platform_utils.py +history/platform_monitor/slurm_monitor.py +history/platform_monitor/slurm_monitor_item.py +history/platform_monitor/test.py +monitor/__init__.py +monitor/diagram.py +monitor/monitor.py +monitor/utils.py +notifications/__init__.py +notifications/mail_notifier.py +notifications/notifier.py +performance/__init__.py +performance/performance_metrics.py +performance/utils.py +statistics/__init__.py +statistics/job_stat.py +statistics/statistics.py +statistics/stats_summary.py +statistics/test.py +statistics/utils.py +workers/__init__.py +workers/populate_details_db.py +workers/populate_graph.py +workers/populate_queue_run_times.py +workers/populate_running_experiments.py +workers/test.py +workers/test_esarchive.py +workers/verify_complete.py +workers/business/__init__.py +workers/business/populate_times.py +workers/business/process_graph_drawings.py +workers/deprecated/__init__.py +workers/deprecated/fix_historic.py +workers/deprecated/fix_historic_energy.py +workers/populate_details/__init__.py +workers/populate_details/populate.py +workers/populate_details/test.py \ No newline at end of file diff --git a/autosubmit_api/autosubmit_api.egg-info/dependency_links.txt b/autosubmit_api/autosubmit_api.egg-info/dependency_links.txt new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/autosubmit_api/autosubmit_api.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/autosubmit_api/autosubmit_api.egg-info/requires.txt b/autosubmit_api/autosubmit_api.egg-info/requires.txt new file mode 100644 index 0000000000000000000000000000000000000000..d48668615a82bcef4d7bdaf81c0326244541e470 --- /dev/null +++ b/autosubmit_api/autosubmit_api.egg-info/requires.txt @@ -0,0 +1,37 @@ +bscearth.utils==0.5.2 +argparse<2,>=1.2 +python-dateutil>2 +pydotplus>=2 +pyparsing>=2.0.1 +numpy +matplotlib +paramiko==2.6.0 +mock>=1.3.0 +portalocker>=0.5.7 +networkx +bscearth.utils +Flask==1.0.4 +Flask-Cors==3.0.8 +Flask-Jsonpify==1.5.0 +Flask-RESTful==0.3.7 +SQLAlchemy==1.3.11 +PyJWT==1.7.1 +Flask==1.1.1 +Flask-Cors==3.0.8 +Flask-Jsonpify==1.5.0 +Flask-RESTful==0.3.7 +gunicorn==19.9.0 +mock==3.0.5 +networkx==2.2 +numpy==1.16.4 +paramiko==1.15.0 +portalocker==0.5.7 +pydotplus==2.0.2 +pydot==1.4.1 +regex==2019.6.8 +requests==2.22.0 +graphviz==0.13 +enum34==1.1.6 +typing==3.7.4.3 +radical.saga==0.70.0 +scipy==1.2.2 diff --git a/autosubmit_api/autosubmit_api.egg-info/top_level.txt b/autosubmit_api/autosubmit_api.egg-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..25b178aab7083c3934c3d3f0b90bdc5ac4e5988d --- /dev/null +++ b/autosubmit_api/autosubmit_api.egg-info/top_level.txt @@ -0,0 +1,14 @@ +autosubmit_legacy +builders +common +components +config +database +experiment +git +history +monitor +notifications +performance +statistics +workers diff --git a/autosubmit_api/autosubmit_legacy/__init__.pyc b/autosubmit_api/autosubmit_legacy/__init__.pyc new file mode 100644 index 0000000000000000000000000000000000000000..49bdbde9f260188e7702232471cd86878c1b2c07 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/__init__.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/autosubmit.pyc b/autosubmit_api/autosubmit_legacy/autosubmit.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f8cfe1132f1d4ded653b4c2c646d0ff8d14ca596 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/autosubmit.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/job/__init__.pyc b/autosubmit_api/autosubmit_legacy/job/__init__.pyc new file mode 100644 index 0000000000000000000000000000000000000000..325c7874fe54b1de27c286e0f053df817ffd2faa Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/__init__.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/job/job.pyc b/autosubmit_api/autosubmit_legacy/job/job.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ae608934554368795529ad102160520af40b4a98 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/job.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/job/job_common.pyc b/autosubmit_api/autosubmit_legacy/job/job_common.pyc new file mode 100644 index 0000000000000000000000000000000000000000..60a7501d0a1c9a3fd5e7da7cf30f8f9edbbd0445 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/job_common.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/job/job_dict.pyc b/autosubmit_api/autosubmit_legacy/job/job_dict.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c058fefac0e19e828d9991d3c56b870793bdff15 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/job_dict.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/job/job_exceptions.pyc b/autosubmit_api/autosubmit_legacy/job/job_exceptions.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ba4cb9e314daa9d575fd84140e2594532327f081 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/job_exceptions.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/job/job_grouping.pyc b/autosubmit_api/autosubmit_legacy/job/job_grouping.pyc new file mode 100644 index 0000000000000000000000000000000000000000..124384bd4d54e37d5aaaab0fe330bbbe3f7e344d Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/job_grouping.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/job/job_list.pyc b/autosubmit_api/autosubmit_legacy/job/job_list.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b7c1a0681b5f1b58b8205e53c781a8a8ef650a21 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/job_list.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/job/job_list_persistence.pyc b/autosubmit_api/autosubmit_legacy/job/job_list_persistence.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d3a3d5d5a4492947b21a39d919f9ed26060c4086 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/job_list_persistence.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/job/job_package_persistence.pyc b/autosubmit_api/autosubmit_legacy/job/job_package_persistence.pyc new file mode 100644 index 0000000000000000000000000000000000000000..83c8873d6fa7175cfb71b6858695cf81ffa85f3e Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/job_package_persistence.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/job/job_packager.pyc b/autosubmit_api/autosubmit_legacy/job/job_packager.pyc new file mode 100644 index 0000000000000000000000000000000000000000..69ae92fd04e3311bb9303bb0f9aab4bc3f31814f Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/job_packager.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/job/job_packages.pyc b/autosubmit_api/autosubmit_legacy/job/job_packages.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c5894a181ffc59481e96af25e42dca05271297ee Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/job_packages.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/job/job_utils.pyc b/autosubmit_api/autosubmit_legacy/job/job_utils.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c2e68e53adcc9e578bd95350ebdc15b28c4c524a Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/job_utils.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/job/tree.pyc b/autosubmit_api/autosubmit_legacy/job/tree.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7653d608b704d17781ec27ea28c25f5853887735 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/job/tree.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/__init__.pyc b/autosubmit_api/autosubmit_legacy/platforms/__init__.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e19ab6bbc3f392d6c279be2e83c4bd12108a9cc0 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/__init__.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/ecmwf_adaptor.pyc b/autosubmit_api/autosubmit_legacy/platforms/ecmwf_adaptor.pyc new file mode 100644 index 0000000000000000000000000000000000000000..69e9dec946c6d64914c2b15d5ee82ea0e82f7149 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/ecmwf_adaptor.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/ecplatform.pyc b/autosubmit_api/autosubmit_legacy/platforms/ecplatform.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c93c83f385cf8d33e774e1ce5d2b7dc2db9950ff Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/ecplatform.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/headers/__init__.pyc b/autosubmit_api/autosubmit_legacy/platforms/headers/__init__.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9fd954a2a3d9c2b945c67b52bb63da939b73639c Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/headers/__init__.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/headers/ec_cca_header.pyc b/autosubmit_api/autosubmit_legacy/platforms/headers/ec_cca_header.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cd001ae52cf62d862be142ece3def116df8d539b Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/headers/ec_cca_header.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/headers/ec_header.pyc b/autosubmit_api/autosubmit_legacy/platforms/headers/ec_header.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e324bc0a7d9836e19035d629da4f1abf8c2263bb Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/headers/ec_header.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/headers/local_header.pyc b/autosubmit_api/autosubmit_legacy/platforms/headers/local_header.pyc new file mode 100644 index 0000000000000000000000000000000000000000..daeddc94b247145bf10f977dc2bba99eac31fce8 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/headers/local_header.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/headers/lsf_header.pyc b/autosubmit_api/autosubmit_legacy/platforms/headers/lsf_header.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ffb6ec747b282fd3b3275967bfd34b1695cd4593 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/headers/lsf_header.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/headers/pbs10_header.pyc b/autosubmit_api/autosubmit_legacy/platforms/headers/pbs10_header.pyc new file mode 100644 index 0000000000000000000000000000000000000000..62a925d9f0a85cc729a99a20f1872770619154b1 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/headers/pbs10_header.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/headers/pbs11_header.pyc b/autosubmit_api/autosubmit_legacy/platforms/headers/pbs11_header.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3081967dc161d1d62789c9a4a53adb8344cb7f21 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/headers/pbs11_header.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/headers/pbs12_header.pyc b/autosubmit_api/autosubmit_legacy/platforms/headers/pbs12_header.pyc new file mode 100644 index 0000000000000000000000000000000000000000..eb960f9c8e978a82b78db95b6e1d1adfe4b7fc52 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/headers/pbs12_header.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/headers/ps_header.pyc b/autosubmit_api/autosubmit_legacy/platforms/headers/ps_header.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e76bc5ffcb3777a25da78e75ad082a46b25636b3 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/headers/ps_header.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/headers/sge_header.pyc b/autosubmit_api/autosubmit_legacy/platforms/headers/sge_header.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b74a22bb3265c58d69aeb77beb79c838d4d0c0ba Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/headers/sge_header.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/headers/slurm_header.pyc b/autosubmit_api/autosubmit_legacy/platforms/headers/slurm_header.pyc new file mode 100644 index 0000000000000000000000000000000000000000..592852ba36080ec12f5dc1cf98ac91cad35e10c2 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/headers/slurm_header.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/locplatform.pyc b/autosubmit_api/autosubmit_legacy/platforms/locplatform.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d01dc7cab7a9a0076550e7f328a12d2c522d84a2 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/locplatform.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/lsfplatform.pyc b/autosubmit_api/autosubmit_legacy/platforms/lsfplatform.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d4c08f54ac4ad112b069a136b1b611e506830ef1 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/lsfplatform.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/mn_adaptor.pyc b/autosubmit_api/autosubmit_legacy/platforms/mn_adaptor.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8f77b751b4f9a9368163501cb152dd7322e0fd79 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/mn_adaptor.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/paramiko_platform.pyc b/autosubmit_api/autosubmit_legacy/platforms/paramiko_platform.pyc new file mode 100644 index 0000000000000000000000000000000000000000..095b8b8402fe7411f8eff751bddf3ff89d1c4eb5 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/paramiko_platform.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/paramiko_submitter.pyc b/autosubmit_api/autosubmit_legacy/platforms/paramiko_submitter.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b56f668024f120a9455cb8fabfbf10987a2d2dad Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/paramiko_submitter.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/pbsplatform.pyc b/autosubmit_api/autosubmit_legacy/platforms/pbsplatform.pyc new file mode 100644 index 0000000000000000000000000000000000000000..99f206314ad773100ad50c86a958e0d05f322695 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/pbsplatform.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/platform.pyc b/autosubmit_api/autosubmit_legacy/platforms/platform.pyc new file mode 100644 index 0000000000000000000000000000000000000000..12dcdfa4a5267dfa05f4feb6079b0c508f61990f Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/platform.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/psplatform.pyc b/autosubmit_api/autosubmit_legacy/platforms/psplatform.pyc new file mode 100644 index 0000000000000000000000000000000000000000..13b26c82a77ccd22e91eeec18e3a3fca2dffaa87 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/psplatform.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/saga_platform.pyc b/autosubmit_api/autosubmit_legacy/platforms/saga_platform.pyc new file mode 100644 index 0000000000000000000000000000000000000000..61ae638d8f0de2a0db2f5ed7d61b0a83344ff030 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/saga_platform.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/saga_submitter.pyc b/autosubmit_api/autosubmit_legacy/platforms/saga_submitter.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a06901982eea2f87835817cc89160e15088da0e2 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/saga_submitter.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/sgeplatform.pyc b/autosubmit_api/autosubmit_legacy/platforms/sgeplatform.pyc new file mode 100644 index 0000000000000000000000000000000000000000..35fb5ba153e47709d7707a025de6c701f1da7642 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/sgeplatform.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/slurmplatform.pyc b/autosubmit_api/autosubmit_legacy/platforms/slurmplatform.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e06a8d820c3b99a8b902cb79a6637df788831fe5 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/slurmplatform.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/submitter.pyc b/autosubmit_api/autosubmit_legacy/platforms/submitter.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a6457b9b3f1aa54dbc8c75117d7b2833f68d70e5 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/submitter.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/wrappers/__init__.pyc b/autosubmit_api/autosubmit_legacy/platforms/wrappers/__init__.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7cdcbb3f33a32e1423681f466f99897db9326a32 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/wrappers/__init__.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/wrappers/wrapper_builder.pyc b/autosubmit_api/autosubmit_legacy/platforms/wrappers/wrapper_builder.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8d6581211c0b4bcacd394070b3f873d748f0a448 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/wrappers/wrapper_builder.pyc differ diff --git a/autosubmit_api/autosubmit_legacy/platforms/wrappers/wrapper_factory.pyc b/autosubmit_api/autosubmit_legacy/platforms/wrappers/wrapper_factory.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a8853878c5776519f36a9c4d4afd6bec46419f76 Binary files /dev/null and b/autosubmit_api/autosubmit_legacy/platforms/wrappers/wrapper_factory.pyc differ diff --git a/autosubmit_api/build/lib/autosubmit_legacy/__init__.py b/autosubmit_api/build/lib/autosubmit_legacy/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/autosubmit_api/build/lib/autosubmit_legacy/autosubmit.py b/autosubmit_api/build/lib/autosubmit_legacy/autosubmit.py new file mode 100644 index 0000000000000000000000000000000000000000..4a7c4dba46018119fbda5636654d4babf5aac686 --- /dev/null +++ b/autosubmit_api/build/lib/autosubmit_legacy/autosubmit.py @@ -0,0 +1,3498 @@ +#!/usr/bin/env python + +# Copyright 2017 Earth Sciences Department, BSC-CNS + +# This file is part of Autosubmit. + +# Autosubmit is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# Autosubmit is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with Autosubmit. If not, see . +# pipeline_test + + +import traceback +from pyparsing import nestedExpr +from collections import defaultdict +from distutils.util import strtobool +from pkg_resources import require, resource_listdir, resource_exists, resource_string +import portalocker +import datetime +import signal +import random +import re +import shutil +import sys +import pwd +import os +import copy +import time +import tarfile +import json +import subprocess +import argparse + +sys.path.insert(0, os.path.abspath('.')) +from ..config.basicConfig import BasicConfig +from ..config.config_common import AutosubmitConfig +from bscearth.utils.config_parser import ConfigParserFactory +from .job.job_common import Status +from ..git.autosubmit_git import AutosubmitGit +from .job.job_list import JobList +from .job.job_packages import JobPackageThread +from .job.job_package_persistence import JobPackagePersistence +from .job.job_list_persistence import JobListPersistenceDb +from .job.job_list_persistence import JobListPersistencePkl +from .job.job_grouping import JobGrouping +from bscearth.utils.log import Log +from ..database.db_common import create_db +from ..experiment.experiment_common import new_experiment +from ..experiment.experiment_common import copy_experiment +from ..database.db_common import delete_experiment +from ..database.db_common import get_autosubmit_version +from ..monitor.monitor import Monitor +from bscearth.utils.date import date2str +from ..notifications.mail_notifier import MailNotifier +from ..notifications.notifier import Notifier +from .platforms.saga_submitter import SagaSubmitter +from .platforms.paramiko_submitter import ParamikoSubmitter +from .job.job_exceptions import WrongTemplateException +from .job.job_packager import JobPackager +from .platforms.paramiko_platform import ParamikoTimeout +""" +Main module for autosubmit. Only contains an interface class to all functionality implemented on autosubmit +""" + +try: + # noinspection PyCompatibility + from configparser import SafeConfigParser +except ImportError: + # noinspection PyCompatibility + from configparser import SafeConfigParser + +# It is Python dialog available? (optional dependency) +try: + import dialog +except Exception: + dialog = None + + +# noinspection PyPackageRequirements +# noinspection PyPackageRequirements +# noinspection PyPackageRequirements + +# noinspection PyUnusedLocal + + +def signal_handler(signal_received, frame): + """ + Used to handle interrupt signals, allowing autosubmit to clean before exit + + :param signal_received: + :param frame: + """ + Log.info('Autosubmit will interrupt at the next safe occasion') + Autosubmit.exit = True + + +class Autosubmit: + """ + Interface class for autosubmit. + """ + # sys.setrecursionlimit(500000) + # # Get the version number from the relevant file. If not, from autosubmit package + # scriptdir = os.path.abspath(os.path.dirname(__file__)) + + # if not os.path.exists(os.path.join(scriptdir, 'VERSION')): + # scriptdir = os.path.join(scriptdir, os.path.pardir) + + # version_path = os.path.join(scriptdir, 'VERSION') + # readme_path = os.path.join(scriptdir, 'README') + # changes_path = os.path.join(scriptdir, 'CHANGELOG') + # if os.path.isfile(version_path): + # with open(version_path) as f: + # autosubmit_version = f.read().strip() + # else: + # autosubmit_version = require("autosubmitAPIwu")[0].version + + exit = False + + @staticmethod + def parse_args(): + """ + Parse arguments given to an executable and start execution of command given + """ + try: + BasicConfig.read() + + parser = argparse.ArgumentParser( + description='Main executable for autosubmit. ') + parser.add_argument('-v', '--version', action='version', version=Autosubmit.autosubmit_version, + help="returns autosubmit's version number and exit") + parser.add_argument('-lf', '--logfile', choices=('EVERYTHING', 'DEBUG', 'INFO', 'RESULT', 'USER_WARNING', + 'WARNING', 'ERROR', 'CRITICAL', 'NO_LOG'), + default='DEBUG', type=str, + help="sets file's log level.") + parser.add_argument('-lc', '--logconsole', choices=('EVERYTHING', 'DEBUG', 'INFO', 'RESULT', 'USER_WARNING', + 'WARNING', 'ERROR', 'CRITICAL', 'NO_LOG'), + default='INFO', type=str, + help="sets console's log level") + + subparsers = parser.add_subparsers(dest='command') + + # Run + subparser = subparsers.add_parser( + 'run', description="runs specified experiment") + subparser.add_argument('expid', help='experiment identifier') + subparser.add_argument('-nt', '--notransitive', action='store_true', + default=False, help='Disable transitive reduction') + + # Expid + subparser = subparsers.add_parser( + 'expid', description="Creates a new experiment") + group = subparser.add_mutually_exclusive_group() + group.add_argument( + '-y', '--copy', help='makes a copy of the specified experiment') + group.add_argument('-dm', '--dummy', action='store_true', + help='creates a new experiment with default values, usually for testing') + group.add_argument('-op', '--operational', action='store_true', + help='creates a new experiment with operational experiment id') + subparser.add_argument('-H', '--HPC', required=True, + help='specifies the HPC to use for the experiment') + subparser.add_argument('-d', '--description', type=str, required=True, + help='sets a description for the experiment to store in the database.') + + # Delete + subparser = subparsers.add_parser( + 'delete', description="delete specified experiment") + subparser.add_argument('expid', help='experiment identifier') + subparser.add_argument( + '-f', '--force', action='store_true', help='deletes experiment without confirmation') + + # Monitor + subparser = subparsers.add_parser( + 'monitor', description="plots specified experiment") + subparser.add_argument('expid', help='experiment identifier') + subparser.add_argument('-o', '--output', choices=('pdf', 'png', 'ps', 'svg'), default='pdf', + help='chooses type of output for generated plot') + subparser.add_argument('-group_by', choices=('date', 'member', 'chunk', 'split', 'automatic'), default=None, + help='Groups the jobs automatically or by date, member, chunk or split') + subparser.add_argument('-expand', type=str, + help='Supply the list of dates/members/chunks to filter the list of jobs. Default = "Any". ' + 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"') + subparser.add_argument( + '-expand_status', type=str, help='Select the statuses to be expanded') + subparser.add_argument('--hide_groups', action='store_true', + default=False, help='Hides the groups from the plot') + subparser.add_argument('-cw', '--check_wrapper', action='store_true', + default=False, help='Generate possible wrapper in the current workflow') + + group2 = subparser.add_mutually_exclusive_group(required=False) + + group.add_argument('-fs', '--filter_status', type=str, + choices=('Any', 'READY', 'COMPLETED', + 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN'), + help='Select the original status to filter the list of jobs') + group = subparser.add_mutually_exclusive_group(required=False) + group.add_argument('-fl', '--list', type=str, + help='Supply the list of job names to be filtered. Default = "Any". ' + 'LIST = "b037_20101101_fc3_21_sim b037_20111101_fc4_26_sim"') + group.add_argument('-fc', '--filter_chunks', type=str, + help='Supply the list of chunks to filter the list of jobs. Default = "Any". ' + 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"') + group.add_argument('-fs', '--filter_status', type=str, + choices=('Any', 'READY', 'COMPLETED', + 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN'), + help='Select the original status to filter the list of jobs') + group.add_argument('-ft', '--filter_type', type=str, + help='Select the job type to filter the list of jobs') + subparser.add_argument('--hide', action='store_true', default=False, + help='hides plot window') + group2.add_argument('--txt', action='store_true', default=False, + help='Generates only txt status file') + + group2.add_argument('-txtlog', '--txt_logfiles', action='store_true', default=False, + help='Generates only txt status file(AS < 3.12b behaviour)') + + subparser.add_argument('-nt', '--notransitive', action='store_true', + default=False, help='Disable transitive reduction') + + # Stats + subparser = subparsers.add_parser( + 'stats', description="plots statistics for specified experiment") + subparser.add_argument('expid', help='experiment identifier') + subparser.add_argument('-ft', '--filter_type', type=str, help='Select the job type to filter ' + 'the list of jobs') + subparser.add_argument('-fp', '--filter_period', type=int, help='Select the period to filter jobs ' + 'from current time to the past ' + 'in number of hours back') + subparser.add_argument('-o', '--output', choices=('pdf', 'png', 'ps', 'svg'), default='pdf', + help='type of output for generated plot') + subparser.add_argument('--hide', action='store_true', default=False, + help='hides plot window') + subparser.add_argument('-nt', '--notransitive', action='store_true', + default=False, help='Disable transitive reduction') + + # Clean + subparser = subparsers.add_parser( + 'clean', description="clean specified experiment") + subparser.add_argument('expid', help='experiment identifier') + subparser.add_argument( + '-pr', '--project', action="store_true", help='clean project') + subparser.add_argument('-p', '--plot', action="store_true", + help='clean plot, only 2 last will remain') + subparser.add_argument('-s', '--stats', action="store_true", + help='clean stats, only last will remain') + + # Recovery + subparser = subparsers.add_parser( + 'recovery', description="recover specified experiment") + subparser.add_argument( + 'expid', type=str, help='experiment identifier') + subparser.add_argument( + '-np', '--noplot', action='store_true', default=False, help='omit plot') + subparser.add_argument('--all', action="store_true", default=False, + help='Get completed files to synchronize pkl') + subparser.add_argument( + '-s', '--save', action="store_true", default=False, help='Save changes to disk') + subparser.add_argument('--hide', action='store_true', default=False, + help='hides plot window') + subparser.add_argument('-group_by', choices=('date', 'member', 'chunk', 'split', 'automatic'), default=None, + help='Groups the jobs automatically or by date, member, chunk or split') + subparser.add_argument('-expand', type=str, + help='Supply the list of dates/members/chunks to filter the list of jobs. Default = "Any". ' + 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"') + subparser.add_argument( + '-expand_status', type=str, help='Select the statuses to be expanded') + subparser.add_argument('-nt', '--notransitive', action='store_true', + default=False, help='Disable transitive reduction') + subparser.add_argument('-nl', '--no_recover_logs', action='store_true', default=False, + help='Disable logs recovery') + # Migrate + subparser = subparsers.add_parser( + 'migrate', description="Migrate experiments from current user to another") + subparser.add_argument('expid', help='experiment identifier') + group = subparser.add_mutually_exclusive_group(required=True) + group.add_argument('-o', '--offer', action="store_true", + default=False, help='Offer experiment') + group.add_argument('-p', '--pickup', action="store_true", + default=False, help='Pick-up released experiment') + + # Inspect + subparser = subparsers.add_parser( + 'inspect', description="Generate all .cmd files") + subparser.add_argument('expid', help='experiment identifier') + subparser.add_argument('-nt', '--notransitive', action='store_true', + default=False, help='Disable transitive reduction') + subparser.add_argument( + '-f', '--force', action="store_true", help='Overwrite all cmd') + subparser.add_argument('-cw', '--check_wrapper', action='store_true', + default=False, help='Generate possible wrapper in the current workflow') + + group.add_argument('-fs', '--filter_status', type=str, + choices=('Any', 'READY', 'COMPLETED', + 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN'), + help='Select the original status to filter the list of jobs') + group = subparser.add_mutually_exclusive_group(required=False) + group.add_argument('-fl', '--list', type=str, + help='Supply the list of job names to be filtered. Default = "Any". ' + 'LIST = "b037_20101101_fc3_21_sim b037_20111101_fc4_26_sim"') + group.add_argument('-fc', '--filter_chunks', type=str, + help='Supply the list of chunks to filter the list of jobs. Default = "Any". ' + 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"') + group.add_argument('-fs', '--filter_status', type=str, + choices=('Any', 'READY', 'COMPLETED', + 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN'), + help='Select the original status to filter the list of jobs') + group.add_argument('-ft', '--filter_type', type=str, + help='Select the job type to filter the list of jobs') + + # Check + subparser = subparsers.add_parser( + 'check', description="check configuration for specified experiment") + subparser.add_argument('expid', help='experiment identifier') + subparser.add_argument('-nt', '--notransitive', action='store_true', + default=False, help='Disable transitive reduction') + # Describe + subparser = subparsers.add_parser( + 'describe', description="Show details for specified experiment") + subparser.add_argument('expid', help='experiment identifier') + + # Create + subparser = subparsers.add_parser( + 'create', description="create specified experiment joblist") + subparser.add_argument('expid', help='experiment identifier') + subparser.add_argument( + '-np', '--noplot', action='store_true', default=False, help='omit plot') + subparser.add_argument('--hide', action='store_true', default=False, + help='hides plot window') + subparser.add_argument('-o', '--output', choices=('pdf', 'png', 'ps', 'svg'), default='pdf', + help='chooses type of output for generated plot') + subparser.add_argument('-group_by', choices=('date', 'member', 'chunk', 'split', 'automatic'), default=None, + help='Groups the jobs automatically or by date, member, chunk or split') + subparser.add_argument('-expand', type=str, + help='Supply the list of dates/members/chunks to filter the list of jobs. Default = "Any". ' + 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"') + subparser.add_argument( + '-expand_status', type=str, help='Select the statuses to be expanded') + subparser.add_argument('-nt', '--notransitive', action='store_true', + default=False, help='Disable transitive reduction') + subparser.add_argument('-cw', '--check_wrapper', action='store_true', + default=False, help='Generate possible wrapper in the current workflow') + + # Configure + subparser = subparsers.add_parser('configure', description="configure database and path for autosubmit. It " + "can be done at machine, user or local level." + "If no arguments specified configure will " + "display dialog boxes (if installed)") + subparser.add_argument( + '--advanced', action="store_true", help="Open advanced configuration of autosubmit") + subparser.add_argument('-db', '--databasepath', default=None, help='path to database. If not supplied, ' + 'it will prompt for it') + subparser.add_argument( + '-dbf', '--databasefilename', default=None, help='database filename') + subparser.add_argument('-lr', '--localrootpath', default=None, help='path to store experiments. If not ' + 'supplied, it will prompt for it') + subparser.add_argument('-pc', '--platformsconfpath', default=None, help='path to platforms.conf file to ' + 'use by default. Optional') + subparser.add_argument('-jc', '--jobsconfpath', default=None, help='path to jobs.conf file to use by ' + 'default. Optional') + subparser.add_argument( + '-sm', '--smtphostname', default=None, help='STMP server hostname. Optional') + subparser.add_argument( + '-mf', '--mailfrom', default=None, help='Notifications sender address. Optional') + group = subparser.add_mutually_exclusive_group() + group.add_argument('--all', action="store_true", + help='configure for all users') + group.add_argument('--local', action="store_true", help='configure only for using Autosubmit from this ' + 'path') + + # Install + subparsers.add_parser( + 'install', description='install database for autosubmit on the configured folder') + + # Set status + subparser = subparsers.add_parser( + 'setstatus', description="sets job status for an experiment") + subparser.add_argument('expid', help='experiment identifier') + subparser.add_argument( + '-np', '--noplot', action='store_true', default=False, help='omit plot') + subparser.add_argument( + '-s', '--save', action="store_true", default=False, help='Save changes to disk') + + subparser.add_argument('-t', '--status_final', + choices=('READY', 'COMPLETED', 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN', + 'QUEUING', 'RUNNING'), + required=True, + help='Supply the target status') + group = subparser.add_mutually_exclusive_group(required=True) + group.add_argument('-fl', '--list', type=str, + help='Supply the list of job names to be changed. Default = "Any". ' + 'LIST = "b037_20101101_fc3_21_sim b037_20111101_fc4_26_sim"') + group.add_argument('-fc', '--filter_chunks', type=str, + help='Supply the list of chunks to change the status. Default = "Any". ' + 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"') + group.add_argument('-fs', '--filter_status', type=str, + help='Select the status (one or more) to filter the list of jobs.' + "Valid values = ['Any', 'READY', 'COMPLETED', 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN']") + group.add_argument('-ft', '--filter_type', type=str, + help='Select the job type to filter the list of jobs') + + subparser.add_argument('--hide', action='store_true', default=False, + help='hides plot window') + subparser.add_argument('-group_by', choices=('date', 'member', 'chunk', 'split', 'automatic'), default=None, + help='Groups the jobs automatically or by date, member, chunk or split') + subparser.add_argument('-expand', type=str, + help='Supply the list of dates/members/chunks to filter the list of jobs. Default = "Any". ' + 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"') + subparser.add_argument( + '-expand_status', type=str, help='Select the statuses to be expanded') + subparser.add_argument('-nt', '--notransitive', action='store_true', + default=False, help='Disable transitive reduction') + subparser.add_argument('-cw', '--check_wrapper', action='store_true', + default=False, help='Generate possible wrapper in the current workflow') + + # Test Case + subparser = subparsers.add_parser( + 'testcase', description='create test case experiment') + subparser.add_argument( + '-y', '--copy', help='makes a copy of the specified experiment') + subparser.add_argument( + '-d', '--description', required=True, help='description of the test case') + subparser.add_argument('-c', '--chunks', help='chunks to run') + subparser.add_argument('-m', '--member', help='member to run') + subparser.add_argument('-s', '--stardate', help='stardate to run') + subparser.add_argument( + '-H', '--HPC', required=True, help='HPC to run experiment on it') + subparser.add_argument( + '-b', '--branch', help='branch of git to run (or revision from subversion)') + + # Test + subparser = subparsers.add_parser( + 'test', description='test experiment') + subparser.add_argument('expid', help='experiment identifier') + subparser.add_argument( + '-c', '--chunks', required=True, help='chunks to run') + subparser.add_argument('-m', '--member', help='member to run') + subparser.add_argument('-s', '--stardate', help='stardate to run') + subparser.add_argument( + '-H', '--HPC', help='HPC to run experiment on it') + subparser.add_argument( + '-b', '--branch', help='branch of git to run (or revision from subversion)') + + # Refresh + subparser = subparsers.add_parser( + 'refresh', description='refresh project directory for an experiment') + subparser.add_argument('expid', help='experiment identifier') + subparser.add_argument('-mc', '--model_conf', default=False, action='store_true', + help='overwrite model conf file') + subparser.add_argument('-jc', '--jobs_conf', default=False, action='store_true', + help='overwrite jobs conf file') + + # Archive + subparser = subparsers.add_parser( + 'archive', description='archives an experiment') + subparser.add_argument('expid', help='experiment identifier') + + # Unarchive + subparser = subparsers.add_parser( + 'unarchive', description='unarchives an experiment') + subparser.add_argument('expid', help='experiment identifier') + + # Readme + subparsers.add_parser('readme', description='show readme') + + # Changelog + subparsers.add_parser('changelog', description='show changelog') + + args = parser.parse_args() + + Log.set_console_level(args.logconsole) + Log.set_file_level(args.logfile) + + if args.command == 'run': + return Autosubmit.run_experiment(args.expid, args.notransitive) + elif args.command == 'expid': + return Autosubmit.expid(args.HPC, args.description, args.copy, args.dummy, False, + args.operational) != '' + elif args.command == 'delete': + return Autosubmit.delete(args.expid, args.force) + elif args.command == 'monitor': + return Autosubmit.monitor(args.expid, args.output, args.list, args.filter_chunks, args.filter_status, + args.filter_type, args.hide, args.txt, args.group_by, args.expand, + args.expand_status, args.hide_groups, args.notransitive, args.check_wrapper, args.txt_logfiles) + elif args.command == 'stats': + return Autosubmit.statistics(args.expid, args.filter_type, args.filter_period, args.output, args.hide, + args.notransitive) + elif args.command == 'clean': + return Autosubmit.clean(args.expid, args.project, args.plot, args.stats) + elif args.command == 'recovery': + return Autosubmit.recovery(args.expid, args.noplot, args.save, args.all, args.hide, args.group_by, + args.expand, args.expand_status, args.notransitive, args.no_recover_logs) + elif args.command == 'check': + return Autosubmit.check(args.expid, args.notransitive) + elif args.command == 'inspect': + return Autosubmit.inspect(args.expid, args.list, args.filter_chunks, args.filter_status, + args.filter_type, args.notransitive, args.force, args.check_wrapper) + elif args.command == 'describe': + return Autosubmit.describe(args.expid) + elif args.command == 'migrate': + return Autosubmit.migrate(args.expid, args.offer, args.pickup) + elif args.command == 'create': + return Autosubmit.create(args.expid, args.noplot, args.hide, args.output, args.group_by, args.expand, + args.expand_status, args.notransitive, args.check_wrapper) + elif args.command == 'configure': + if not args.advanced or (args.advanced and dialog is None): + return Autosubmit.configure(args.advanced, args.databasepath, args.databasefilename, + args.localrootpath, args.platformsconfpath, args.jobsconfpath, + args.smtphostname, args.mailfrom, args.all, args.local) + else: + return Autosubmit.configure_dialog() + elif args.command == 'install': + return Autosubmit.install() + elif args.command == 'setstatus': + return Autosubmit.set_status(args.expid, args.noplot, args.save, args.status_final, args.list, + args.filter_chunks, args.filter_status, args.filter_type, args.hide, + args.group_by, args.expand, args.expand_status, args.notransitive, args.check_wrapper) + elif args.command == 'testcase': + return Autosubmit.testcase(args.copy, args.description, args.chunks, args.member, args.stardate, + args.HPC, args.branch) + elif args.command == 'test': + return Autosubmit.test(args.expid, args.chunks, args.member, args.stardate, args.HPC, args.branch) + elif args.command == 'refresh': + return Autosubmit.refresh(args.expid, args.model_conf, args.jobs_conf) + elif args.command == 'archive': + return Autosubmit.archive(args.expid) + elif args.command == 'unarchive': + return Autosubmit.unarchive(args.expid) + + elif args.command == 'readme': + if os.path.isfile(Autosubmit.readme_path): + with open(Autosubmit.readme_path) as f: + print(f.read()) + return True + return False + elif args.command == 'changelog': + if os.path.isfile(Autosubmit.changes_path): + with open(Autosubmit.changes_path) as f: + print(f.read()) + return True + return False + except Exception as e: + from traceback import format_exc + Log.critical( + 'Unhandled exception on Autosubmit: {0}\n{1}', e, format_exc(10)) + + return False + + @staticmethod + def _delete_expid(expid_delete): + """ + Removes an experiment from path and database + + :type expid_delete: str + :param expid_delete: identifier of the experiment to delete + """ + if expid_delete == '' or expid_delete is None and not os.path.exists(os.path.join(BasicConfig.LOCAL_ROOT_DIR, + expid_delete)): + Log.info("Experiment directory does not exist.") + else: + Log.info("Removing experiment directory...") + ret = False + if pwd.getpwuid(os.stat(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid_delete)).st_uid).pw_name == os.getlogin(): + try: + + shutil.rmtree(os.path.join( + BasicConfig.LOCAL_ROOT_DIR, expid_delete)) + except OSError as e: + Log.warning('Can not delete experiment folder: {0}', e) + return ret + Log.info("Deleting experiment from database...") + ret = delete_experiment(expid_delete) + if ret: + Log.result("Experiment {0} deleted".format(expid_delete)) + else: + Log.warning( + "Current User is not the Owner {0} can not be deleted!", expid_delete) + return ret + + @staticmethod + def expid(hpc, description, copy_id='', dummy=False, test=False, operational=False): + """ + Creates a new experiment for given HPC + + :param operational: if true, creates an operational experiment + :type operational: bool + :type hpc: str + :type description: str + :type copy_id: str + :type dummy: bool + :param hpc: name of the main HPC for the experiment + :param description: short experiment's description. + :param copy_id: experiment identifier of experiment to copy + :param dummy: if true, writes a default dummy configuration for testing + :param test: if true, creates an experiment for testing + :return: experiment identifier. If method fails, returns ''. + :rtype: str + """ + BasicConfig.read() + + log_path = os.path.join( + BasicConfig.LOCAL_ROOT_DIR, 'ASlogs', 'expid.log'.format(os.getuid())) + try: + Log.set_file(log_path) + except IOError as e: + Log.error("Can not create log file in path {0}: {1}".format( + log_path, e.message)) + exp_id = None + if description is None: + Log.error("Missing experiment description.") + return '' + if hpc is None: + Log.error("Missing HPC.") + return '' + if not copy_id: + exp_id = new_experiment( + description, Autosubmit.autosubmit_version, test, operational) + if exp_id == '': + return '' + try: + os.mkdir(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id)) + + os.mkdir(os.path.join( + BasicConfig.LOCAL_ROOT_DIR, exp_id, 'conf')) + Log.info("Copying config files...") + + # autosubmit config and experiment copied from AS. + files = resource_listdir('autosubmit.config', 'files') + for filename in files: + if resource_exists('autosubmit.config', 'files/' + filename): + index = filename.index('.') + new_filename = filename[:index] + \ + "_" + exp_id + filename[index:] + + if filename == 'platforms.conf' and BasicConfig.DEFAULT_PLATFORMS_CONF != '': + content = open(os.path.join( + BasicConfig.DEFAULT_PLATFORMS_CONF, filename)).read() + elif filename == 'jobs.conf' and BasicConfig.DEFAULT_JOBS_CONF != '': + content = open(os.path.join( + BasicConfig.DEFAULT_JOBS_CONF, filename)).read() + else: + content = resource_string( + 'autosubmit.config', 'files/' + filename) + + conf_new_filename = os.path.join( + BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf", new_filename) + Log.debug(conf_new_filename) + open(conf_new_filename, 'w').write(content) + Autosubmit._prepare_conf_files( + exp_id, hpc, Autosubmit.autosubmit_version, dummy) + except (OSError, IOError) as e: + Log.error( + "Can not create experiment: {0}\nCleaning...".format(e)) + Autosubmit._delete_expid(exp_id) + return '' + else: + try: + if os.path.exists(os.path.join(BasicConfig.LOCAL_ROOT_DIR, copy_id)): + exp_id = copy_experiment( + copy_id, description, Autosubmit.autosubmit_version, test, operational) + if exp_id == '': + return '' + dir_exp_id = os.path.join( + BasicConfig.LOCAL_ROOT_DIR, exp_id) + os.mkdir(dir_exp_id) + os.mkdir(dir_exp_id + '/conf') + Log.info("Copying previous experiment config directories") + conf_copy_id = os.path.join( + BasicConfig.LOCAL_ROOT_DIR, copy_id, "conf") + files = os.listdir(conf_copy_id) + for filename in files: + if os.path.isfile(os.path.join(conf_copy_id, filename)): + new_filename = filename.replace(copy_id, exp_id) + content = open(os.path.join( + conf_copy_id, filename), 'r').read() + open(os.path.join(dir_exp_id, "conf", + new_filename), 'w').write(content) + Autosubmit._prepare_conf_files( + exp_id, hpc, Autosubmit.autosubmit_version, dummy) + ##### + autosubmit_config = AutosubmitConfig( + copy_id, BasicConfig, ConfigParserFactory()) + if autosubmit_config.check_conf_files(): + project_type = autosubmit_config.get_project_type() + if project_type == "git": + autosubmit_config.check_proj() + autosubmit_git = AutosubmitGit(copy_id[0]) + Log.info("checking model version...") + if not autosubmit_git.check_commit(autosubmit_config): + return False + ##### + else: + Log.critical( + "The previous experiment directory does not exist") + return '' + except (OSError, IOError) as e: + Log.error( + "Can not create experiment: {0}\nCleaning...".format(e)) + Autosubmit._delete_expid(exp_id) + return '' + + Log.debug("Creating temporal directory...") + exp_id_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id) + tmp_path = os.path.join(exp_id_path, "tmp") + os.mkdir(tmp_path) + os.chmod(tmp_path, 0o775) + os.mkdir(os.path.join(tmp_path, BasicConfig.LOCAL_ASLOG_DIR)) + os.chmod(os.path.join(tmp_path, BasicConfig.LOCAL_ASLOG_DIR), 0o775) + Log.debug("Creating temporal remote directory...") + remote_tmp_path = os.path.join(tmp_path, "LOG_" + exp_id) + os.mkdir(remote_tmp_path) + os.chmod(remote_tmp_path, 0o775) + + Log.debug("Creating pkl directory...") + os.mkdir(os.path.join(exp_id_path, "pkl")) + + Log.debug("Creating plot directory...") + os.mkdir(os.path.join(exp_id_path, "plot")) + os.chmod(os.path.join(exp_id_path, "plot"), 0o775) + Log.result("Experiment registered successfully") + Log.user_warning("Remember to MODIFY the config files!") + return exp_id + + @staticmethod + def delete(expid, force): + """ + Deletes and experiment from database and experiment's folder + + :type force: bool + :type expid: str + :param expid: identifier of the experiment to delete + :param force: if True, does not ask for confirmation + + :returns: True if succesful, False if not + :rtype: bool + """ + log_path = os.path.join( + BasicConfig.LOCAL_ROOT_DIR, "ASlogs", 'delete.log'.format(os.getuid())) + try: + Log.set_file(log_path) + except IOError as e: + Log.error("Can not create log file in path {0}: {1}".format( + log_path, e.message)) + + if os.path.exists(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid)): + if force or Autosubmit._user_yes_no_query("Do you want to delete " + expid + " ?"): + return Autosubmit._delete_expid(expid) + else: + Log.info("Quitting...") + return False + else: + Log.error("The experiment does not exist") + return True + + @staticmethod + def _load_parameters(as_conf, job_list, platforms): + # Load parameters + Log.debug("Loading parameters...") + parameters = as_conf.load_parameters() + for platform_name in platforms: + platform = platforms[platform_name] + platform.add_parameters(parameters) + + platform = platforms[as_conf.get_platform().lower()] + platform.add_parameters(parameters, True) + + job_list.parameters = parameters + + @staticmethod + def inspect(expid, lst, filter_chunks, filter_status, filter_section, notransitive=False, force=False, check_wrapper=False): + """ + Generates cmd files experiment. + + :type expid: str + :param expid: identifier of experiment to be run + :return: True if run to the end, False otherwise + :rtype: bool + """ + + if expid is None: + Log.critical("Missing experiment id") + + BasicConfig.read() + exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid) + tmp_path = os.path.join(exp_path, BasicConfig.LOCAL_TMP_DIR) + if os.path.exists(os.path.join(tmp_path, 'autosubmit.lock')): + locked = True + else: + locked = False + + if not os.path.exists(exp_path): + Log.critical( + "The directory %s is needed and does not exist" % exp_path) + Log.warning("Does an experiment with the given id exist?") + return 1 + Log.info("Starting inspect command") + Log.set_file(os.path.join( + tmp_path, BasicConfig.LOCAL_ASLOG_DIR, 'generate.log')) + os.system('clear') + signal.signal(signal.SIGINT, signal_handler) + as_conf = AutosubmitConfig(expid, BasicConfig, ConfigParserFactory()) + if not as_conf.check_conf_files(): + Log.critical('Can not generate scripts with invalid configuration') + return False + project_type = as_conf.get_project_type() + if project_type != "none": + # Check proj configuration + as_conf.check_proj() + safetysleeptime = as_conf.get_safetysleeptime() + Log.debug("The Experiment name is: {0}", expid) + Log.debug("Sleep: {0}", safetysleeptime) + packages_persistence = JobPackagePersistence(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "pkl"), + "job_packages_" + expid) + os.chmod(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, + "pkl", "job_packages_" + expid + ".db"), 0o664) + + packages_persistence.reset_table(True) + job_list_original = Autosubmit.load_job_list( + expid, as_conf, notransitive=notransitive) + job_list = copy.deepcopy(job_list_original) + job_list.packages_dict = {} + + Log.debug("Length of the jobs list: {0}", len(job_list)) + + # variables to be updated on the fly + safetysleeptime = as_conf.get_safetysleeptime() + Log.debug("Sleep: {0}", safetysleeptime) + # Generate + Log.info("Starting to generate cmd scripts") + + if not isinstance(job_list, type([])): + jobs = [] + jobs_cw = [] + if check_wrapper and (not locked or (force and locked)): + Log.info("Generating all cmd script adapted for wrappers") + jobs = job_list.get_uncompleted() + + jobs_cw = job_list.get_completed() + else: + if (force and not locked) or (force and locked): + Log.info("Overwritting all cmd scripts") + jobs = job_list.get_job_list() + elif locked: + Log.warning( + "There is a .lock file and not -f, generating only all unsubmitted cmd scripts") + jobs = job_list.get_unsubmitted() + else: + Log.info("Generating cmd scripts only for selected jobs") + if filter_chunks: + fc = filter_chunks + Log.debug(fc) + if fc == 'Any': + jobs = job_list.get_job_list() + else: + # noinspection PyTypeChecker + data = json.loads(Autosubmit._create_json(fc)) + for date_json in data['sds']: + date = date_json['sd'] + jobs_date = [j for j in job_list.get_job_list() if date2str( + j.date) == date] + + for member_json in date_json['ms']: + member = member_json['m'] + jobs_member = [j for j in jobs_date if j.member == member] + + for chunk_json in member_json['cs']: + chunk = int(chunk_json) + jobs = jobs + \ + [job for job in [j for j in jobs_member if j.chunk == chunk]] + + elif filter_status: + Log.debug( + "Filtering jobs with status {0}", filter_status) + if filter_status == 'Any': + jobs = job_list.get_job_list() + else: + fs = Autosubmit._get_status(filter_status) + jobs = [job for job in [j for j in job_list.get_job_list() if j.status == fs]] + + elif filter_section: + ft = filter_section + Log.debug(ft) + + if ft == 'Any': + jobs = job_list.get_job_list() + else: + for job in job_list.get_job_list(): + if job.section == ft: + jobs.append(job) + elif lst: + jobs_lst = lst.split() + + if jobs == 'Any': + jobs = job_list.get_job_list() + else: + for job in job_list.get_job_list(): + if job.name in jobs_lst: + jobs.append(job) + else: + jobs = job_list.get_job_list() + if isinstance(jobs, type([])): + referenced_jobs_to_remove = set() + for job in jobs: + for child in job.children: + if child not in jobs: + referenced_jobs_to_remove.add(child) + for parent in job.parents: + if parent not in jobs: + referenced_jobs_to_remove.add(parent) + + for job in jobs: + job.status = Status.WAITING + + Autosubmit.generate_scripts_andor_wrappers( + as_conf, job_list, jobs, packages_persistence, False) + if len(jobs_cw) > 0: + referenced_jobs_to_remove = set() + for job in jobs_cw: + for child in job.children: + if child not in jobs_cw: + referenced_jobs_to_remove.add(child) + for parent in job.parents: + if parent not in jobs_cw: + referenced_jobs_to_remove.add(parent) + + for job in jobs_cw: + job.status = Status.WAITING + Autosubmit.generate_scripts_andor_wrappers( + as_conf, job_list, jobs_cw, packages_persistence, False) + + Log.info("no more scripts to generate, now proceed to check them manually") + time.sleep(safetysleeptime) + return True + + @staticmethod + def generate_scripts_andor_wrappers(as_conf, job_list, jobs_filtered, packages_persistence, only_wrappers=False): + """ + as_conf: AutosubmitConfig object + job_list: JobList object, contains a list of jobs + jobs_filtered: list of jobs + packages_persistence: Database handler + only_wrappers: True + """ + job_list._job_list = jobs_filtered + job_list.update_list(as_conf, False) + # Identifying the submitter and loading it + submitter = Autosubmit._get_submitter(as_conf) + # Function depending on the submitter + submitter.load_platforms(as_conf) + # Identifying HPC from config files + hpcarch = as_conf.get_platform() + # + Autosubmit._load_parameters(as_conf, job_list, submitter.platforms) + platforms_to_test = set() + for job in job_list.get_job_list(): + if job.platform_name is None: + job.platform_name = hpcarch + # noinspection PyTypeChecker + job.platform = submitter.platforms[job.platform_name.lower()] + # noinspection PyTypeChecker + platforms_to_test.add(job.platform) + # case setstatus + job_list.check_scripts(as_conf) + job_list.update_list(as_conf, False) + Autosubmit._load_parameters(as_conf, job_list, submitter.platforms) + while job_list.get_active(): + Autosubmit.submit_ready_jobs( + as_conf, job_list, platforms_to_test, packages_persistence, True, only_wrappers) + + job_list.update_list(as_conf, False) + + @staticmethod + def run_experiment(expid, notransitive=False): + """ + Runs and experiment (submitting all the jobs properly and repeating its execution in case of failure). + + :type expid: str + :param expid: identifier of experiment to be run + :return: True if run to the end, False otherwise + :rtype: bool + """ + if expid is None: + Log.critical("Missing experiment id") + + BasicConfig.read() + exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid) + tmp_path = os.path.join(exp_path, BasicConfig.LOCAL_TMP_DIR) + aslogs_path = os.path.join(tmp_path, BasicConfig.LOCAL_ASLOG_DIR) + if not os.path.exists(aslogs_path): + os.mkdir(aslogs_path) + os.chmod(aslogs_path, 0o775) + if not os.path.exists(exp_path): + Log.critical( + "The directory %s is needed and does not exist" % exp_path) + Log.warning("Does an experiment with the given id exist?") + return 1 + + # checking host whitelist + import platform + host = platform.node() + print(host) + if BasicConfig.ALLOWED_HOSTS and host not in BasicConfig.ALLOWED_HOSTS: + Log.info("\n Autosubmit run command is not allowed on this host") + return False + + # checking if there is a lock file to avoid multiple running on the same expid + try: + with portalocker.Lock(os.path.join(tmp_path, 'autosubmit.lock'), timeout=1): + Log.info( + "Preparing .lock file to avoid multiple instances with same experiment id") + + Log.set_file(os.path.join(aslogs_path, 'run.log')) + os.system('clear') + + signal.signal(signal.SIGINT, signal_handler) + + as_conf = AutosubmitConfig( + expid, BasicConfig, ConfigParserFactory()) + if not as_conf.check_conf_files(): + Log.critical('Can not run with invalid configuration') + return False + + project_type = as_conf.get_project_type() + if project_type != "none": + # Check proj configuration + as_conf.check_proj() + + hpcarch = as_conf.get_platform() + + safetysleeptime = as_conf.get_safetysleeptime() + retrials = as_conf.get_retrials() + + submitter = Autosubmit._get_submitter(as_conf) + submitter.load_platforms(as_conf) + + Log.debug("The Experiment name is: {0}", expid) + Log.debug("Sleep: {0}", safetysleeptime) + Log.debug("Default retrials: {0}", retrials) + + Log.info("Starting job submission...") + + pkl_dir = os.path.join( + BasicConfig.LOCAL_ROOT_DIR, expid, 'pkl') + job_list = Autosubmit.load_job_list( + expid, as_conf, notransitive=notransitive) + + Log.debug( + "Starting from job list restored from {0} files", pkl_dir) + + Log.debug("Length of the jobs list: {0}", len(job_list)) + + Autosubmit._load_parameters( + as_conf, job_list, submitter.platforms) + + # check the job list script creation + Log.debug("Checking experiment templates...") + + platforms_to_test = set() + for job in job_list.get_job_list(): + if job.platform_name is None: + job.platform_name = hpcarch + # noinspection PyTypeChecker + job.platform = submitter.platforms[job.platform_name.lower( + )] + # noinspection PyTypeChecker + platforms_to_test.add(job.platform) + + job_list.check_scripts(as_conf) + + packages_persistence = JobPackagePersistence(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "pkl"), + "job_packages_" + expid) + + if as_conf.get_wrapper_type() != 'none': + os.chmod(os.path.join(BasicConfig.LOCAL_ROOT_DIR, + expid, "pkl", "job_packages_" + expid + ".db"), 0o664) + packages = packages_persistence.load() + for (exp_id, package_name, job_name) in packages: + if package_name not in job_list.packages_dict: + job_list.packages_dict[package_name] = [] + job_list.packages_dict[package_name].append( + job_list.get_job_by_name(job_name)) + + for package_name, jobs in list(job_list.packages_dict.items()): + from job.job import WrapperJob + wrapper_job = WrapperJob(package_name, jobs[0].id, Status.SUBMITTED, 0, jobs, + None, + None, jobs[0].platform, as_conf) + job_list.job_package_map[jobs[0].id] = wrapper_job + job_list.update_list(as_conf) + job_list.save() + ######################### + # AUTOSUBMIT - MAIN LOOP + ######################### + # Main loop. Finishing when all jobs have been submitted + while job_list.get_active(): + if Autosubmit.exit: + return 2 + # reload parameters changes + Log.debug("Reloading parameters...") + as_conf.reload() + Autosubmit._load_parameters( + as_conf, job_list, submitter.platforms) + # variables to be updated on the fly + total_jobs = len(job_list.get_job_list()) + Log.info( + "\n\n{0} of {1} jobs remaining ({2})".format(total_jobs - len(job_list.get_completed()), + total_jobs, + time.strftime("%H:%M"))) + safetysleeptime = as_conf.get_safetysleeptime() + Log.debug("Sleep: {0}", safetysleeptime) + default_retrials = as_conf.get_retrials() + Log.debug("Number of retrials: {0}", default_retrials) + + check_wrapper_jobs_sleeptime = as_conf.get_wrapper_check_time() + Log.debug('WRAPPER CHECK TIME = {0}'.format( + check_wrapper_jobs_sleeptime)) + + save = False + + slurm = [] + for platform in platforms_to_test: + list_jobid = "" + completed_joblist = [] + list_prevStatus = [] + queuing_jobs = job_list.get_in_queue_grouped_id( + platform) + for job_id, job in list(queuing_jobs.items()): + if job_list.job_package_map and job_id in job_list.job_package_map: + Log.debug( + 'Checking wrapper job with id ' + str(job_id)) + wrapper_job = job_list.job_package_map[job_id] + check_wrapper = True + if wrapper_job.status == Status.RUNNING: + check_wrapper = True if datetime.timedelta.total_seconds(datetime.datetime.now( + ) - wrapper_job.checked_time) >= check_wrapper_jobs_sleeptime else False + if check_wrapper: + wrapper_job.checked_time = datetime.datetime.now() + platform.check_job(wrapper_job) + Log.info( + 'Wrapper job ' + wrapper_job.name + ' is ' + str(Status.VALUE_TO_KEY[wrapper_job.new_status])) + wrapper_job.check_status( + wrapper_job.new_status) + save = True + else: + Log.info( + "Waiting for wrapper check time: {0}\n", check_wrapper_jobs_sleeptime) + else: + job = job[0] + prev_status = job.status + if job.status == Status.FAILED: + continue + + if platform.type == "slurm": + list_jobid += str(job_id) + ',' + list_prevStatus.append(prev_status) + completed_joblist.append(job) + else: + platform.check_job(job) + if prev_status != job.update_status(as_conf.get_copy_remote_logs() == 'true'): + if as_conf.get_notifications() == 'true': + if Status.VALUE_TO_KEY[job.status] in job.notify_on: + Notifier.notify_status_change(MailNotifier(BasicConfig), expid, job.name, + Status.VALUE_TO_KEY[prev_status], + Status.VALUE_TO_KEY[job.status], + as_conf.get_mails_to()) + save = True + + if platform.type == "slurm" and list_jobid != "": + slurm.append( + [platform, list_jobid, list_prevStatus, completed_joblist]) + # END LOOP + for platform_jobs in slurm: + platform = platform_jobs[0] + jobs_to_check = platform_jobs[1] + platform.check_Alljobs( + platform_jobs[3], jobs_to_check, as_conf.get_copy_remote_logs()) + + for j_Indx in range(0, len(platform_jobs[3])): + prev_status = platform_jobs[2][j_Indx] + job = platform_jobs[3][j_Indx] + + if prev_status != job.update_status(as_conf.get_copy_remote_logs() == 'true'): + if as_conf.get_notifications() == 'true': + if Status.VALUE_TO_KEY[job.status] in job.notify_on: + Notifier.notify_status_change(MailNotifier(BasicConfig), expid, job.name, + Status.VALUE_TO_KEY[prev_status], + Status.VALUE_TO_KEY[job.status], + as_conf.get_mails_to()) + save = True + + if job_list.update_list(as_conf) or save: + job_list.save() + + if Autosubmit.submit_ready_jobs(as_conf, job_list, platforms_to_test, packages_persistence): + job_list.save() + + if Autosubmit.exit: + return 2 + time.sleep(safetysleeptime) + + Log.info("No more jobs to run.") + if len(job_list.get_failed()) > 0: + Log.info("Some jobs have failed and reached maximum retrials") + return False + else: + Log.result("Run successful") + return True + + except portalocker.AlreadyLocked: + Autosubmit.show_lock_warning(expid) + + except WrongTemplateException: + return False + + @staticmethod + def submit_ready_jobs(as_conf, job_list, platforms_to_test, packages_persistence, inspect=False, + only_wrappers=False): + """ + Gets READY jobs and send them to the platforms if there is available space on the queues + + :param as_conf: autosubmit config object. \n + :type as_conf: AutosubmitConfig Object. \n + :param job_list: JobList as a single entity. \n + :type job_list: JobList() Object. \n + :param platforms_to_test: List of platforms that will be used in the experiment. \n + :type platforms_to_test: Set() of Platform() Object. e.g. EcPlatform(), LsfPlatform(), etc. \n + :return: True if at least one job was submitted, False otherwise + :rtype: bool + """ + save = False + + for platform in platforms_to_test: + Log.debug("\nJobs ready for {1}: {0}", len( + job_list.get_ready(platform)), platform.name) + packages_to_submit, remote_dependencies_dict = JobPackager( + as_conf, platform, job_list).build_packages() + if not inspect: + platform.open_submit_script() + valid_packages_to_submit = [] + for package in packages_to_submit: + try: + if hasattr(package, "name"): + if remote_dependencies_dict and package.name in remote_dependencies_dict['dependencies']: + remote_dependency = remote_dependencies_dict['dependencies'][package.name] + remote_dependency_id = remote_dependencies_dict['name_to_id'][remote_dependency] + package.set_job_dependency(remote_dependency_id) + if not only_wrappers: + try: + package.submit( + as_conf, job_list.parameters, inspect) + valid_packages_to_submit.append(package) + except (IOError, OSError): + # write error file + continue + if only_wrappers or inspect: + for innerJob in package._jobs: + innerJob.status = Status.COMPLETED + + if hasattr(package, "name"): + job_list.packages_dict[package.name] = package.jobs + from job.job import WrapperJob + wrapper_job = WrapperJob(package.name, package.jobs[0].id, Status.READY, 0, + package.jobs, + package._wallclock, package._num_processors, + package.platform, as_conf) + job_list.job_package_map[package.jobs[0].id] = wrapper_job + if remote_dependencies_dict and package.name in remote_dependencies_dict['name_to_id']: + remote_dependencies_dict['name_to_id'][package.name] = package.jobs[0].id + if isinstance(package, JobPackageThread): + packages_persistence.save( + package.name, package.jobs, package._expid, inspect) + save = True + except WrongTemplateException as e: + Log.error( + "Invalid parameter substitution in {0} template", e.job_name) + raise + except Exception: + Log.error( + "{0} submission failed due to Unknown error", platform.name) + raise + + if platform.type == "slurm" and not inspect and not only_wrappers: + try: + save = True + if len(valid_packages_to_submit) > 0: + jobs_id = platform.submit_Script() + if jobs_id is None: + raise BaseException( + "Exiting AS being unable to get jobID") + i = 0 + for package in valid_packages_to_submit: + for job in package.jobs: + job.id = str(jobs_id[i]) + Log.info("{0} submitted", job.name) + job.status = Status.SUBMITTED + job.write_submit_time() + if hasattr(package, "name"): + job_list.packages_dict[package.name] = package.jobs + from job.job import WrapperJob + wrapper_job = WrapperJob(package.name, package.jobs[0].id, Status.SUBMITTED, 0, + package.jobs, + package._wallclock, package._num_processors, + package.platform, as_conf) + job_list.job_package_map[package.jobs[0].id] = wrapper_job + if remote_dependencies_dict and package.name in remote_dependencies_dict[ + 'name_to_id']: + remote_dependencies_dict['name_to_id'][package.name] = package.jobs[0].id + if isinstance(package, JobPackageThread): + packages_persistence.save( + package.name, package.jobs, package._expid, inspect) + i += 1 + + except WrongTemplateException as e: + Log.error( + "Invalid parameter substitution in {0} template", e.job_name) + raise + except Exception: + Log.error("{0} submission failed", platform.name) + raise + + return save + + @staticmethod + def monitor(expid, file_format, lst, filter_chunks, filter_status, filter_section, hide, txt_only=False, + group_by=None, expand=list(), expand_status=list(), hide_groups=False, notransitive=False, check_wrapper=False, txt_logfiles=False): + """ + Plots workflow graph for a given experiment with status of each job coded by node color. + Plot is created in experiment's plot folder with name __