diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a8414e32dfbf872096c9769d5350744d8dce433..0f51a45141eb4bd2f4dc14b0fcb856114cfcf642 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # CHANGELOG +### Pre-release v4.0.0b7 - Release date: 2024-05-14 + +* Reduced writing time in status updater background task +* Improved documentation +* Removed `listexp` view +* Minor fixes + ### Pre-release v4.0.0b6 - Release date: 2024-04-19 * Added filter by autosubmit version in `/v4/experiments` diff --git a/autosubmit_api/__init__.py b/autosubmit_api/__init__.py index e7eae9a30091fe0cf2d717b94857d259a82c4e09..2eeabb72a6e3aa7797442fbd31b95425370317ca 100644 --- a/autosubmit_api/__init__.py +++ b/autosubmit_api/__init__.py @@ -17,6 +17,6 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . -__version__ = "4.0.0b6" +__version__ = "4.0.0b7" __author__ = "Luiggi Tenorio, Bruno P. Kinoshita, Cristian Gutiérrez, Julian Berlin, Wilmer Uruchi" __credits__ = "Barcelona Supercomputing Center" \ No newline at end of file diff --git a/autosubmit_api/bgtasks/tasks/status_updater.py b/autosubmit_api/bgtasks/tasks/status_updater.py index 186be6fccb18c783a2c3c9079d9389a66b3b23c8..971cd1f14c7eea3a43228c8fb5c9d362878a858b 100644 --- a/autosubmit_api/bgtasks/tasks/status_updater.py +++ b/autosubmit_api/bgtasks/tasks/status_updater.py @@ -1,7 +1,7 @@ from datetime import datetime import os import time -from typing import List +from typing import Dict, List from sqlalchemy import select from autosubmit_api.bgtasks.bgtask import BackgroundTaskTemplate @@ -50,6 +50,15 @@ class StatusUpdater(BackgroundTaskTemplate): query_result = conn.execute(tables.experiment_table.select()).all() return [ExperimentModel.model_validate(row._mapping) for row in query_result] + @classmethod + def _get_current_status(cls) -> Dict[str, str]: + """ + Get the current status of the experiments + """ + with create_as_times_db_engine().connect() as conn: + query_result = conn.execute(tables.experiment_status_table.select()).all() + return {row.name: row.status for row in query_result} + @classmethod def _check_exp_running(cls, expid: str) -> bool: """ @@ -113,7 +122,20 @@ class StatusUpdater(BackgroundTaskTemplate): # Read experiments table exp_list = cls._get_experiments() + # Read current status of all experiments + current_status = cls._get_current_status() + # Check every experiment status & update for experiment in exp_list: is_running = cls._check_exp_running(experiment.name) - cls._update_experiment_status(experiment, is_running) + new_status = ( + RunningStatus.RUNNING if is_running else RunningStatus.NOT_RUNNING + ) + if ( + current_status.get(experiment.name, RunningStatus.NOT_RUNNING) + != new_status + ): + cls.logger.info( + f"[{cls.id}] Updating status of {experiment.name} to {new_status}" + ) + cls._update_experiment_status(experiment, is_running) diff --git a/autosubmit_api/database/common.py b/autosubmit_api/database/common.py index 6891df5403feed41ba4a06ac72b50a3cd10274a7..defe7056fce0dd39124123d138f33849032eea0a 100644 --- a/autosubmit_api/database/common.py +++ b/autosubmit_api/database/common.py @@ -1,6 +1,15 @@ import os from typing import Any -from sqlalchemy import Connection, Engine, Select, create_engine, select, text, func +from sqlalchemy import ( + Connection, + Engine, + NullPool, + Select, + create_engine, + select, + text, + func, +) from autosubmit_api.builders import BaseBuilder from autosubmit_api.logger import logger from autosubmit_api.config.basicConfig import APIBasicConfig @@ -14,7 +23,7 @@ class AttachedDatabaseConnBuilder(BaseBuilder): def __init__(self) -> None: super().__init__(False) APIBasicConfig.read() - self.engine = create_engine("sqlite://") + self.engine = create_engine("sqlite://", poolclass=NullPool) self._product = self.engine.connect() def attach_db(self, path: str, name: str): @@ -51,7 +60,9 @@ def create_autosubmit_db_engine() -> Engine: Create an engine for the autosubmit DDBB. Usually named autosubmit.db """ APIBasicConfig.read() - return create_engine(f"sqlite:///{ os.path.abspath(APIBasicConfig.DB_PATH)}") + return create_engine( + f"sqlite:///{ os.path.abspath(APIBasicConfig.DB_PATH)}", poolclass=NullPool + ) def create_as_times_db_engine() -> Engine: @@ -60,7 +71,7 @@ def create_as_times_db_engine() -> Engine: """ APIBasicConfig.read() db_path = os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB) - return create_engine(f"sqlite:///{ os.path.abspath(db_path)}") + return create_engine(f"sqlite:///{ os.path.abspath(db_path)}", poolclass=NullPool) def execute_with_limit_offset( diff --git a/autosubmit_api/database/db_common.py b/autosubmit_api/database/db_common.py index 40443b2afee46b7e2bf9ce4d59ab20b45b3b719b..6eec21ad01cca983e74344c65c9afb7cebffaf44 100644 --- a/autosubmit_api/database/db_common.py +++ b/autosubmit_api/database/db_common.py @@ -205,10 +205,10 @@ def search_experiment_by_id(query, exp_type=None, only_active=None, owner=None): 'Connection to database could not be established: {0}', e.message) return False if owner: - query = "SELECT id,name,user,created,model,branch,hpc,description FROM listexp WHERE user='{0}'".format(owner) + query = "SELECT id,name,user,created,model,branch,hpc,description FROM experiment e left join details d on e.id = d.exp_id WHERE user='{0}'".format(owner) # print(query) else: - query = "SELECT id,name,user,created,model,branch,hpc,description FROM listexp WHERE (name LIKE '" + query + \ + query = "SELECT id,name,user,created,model,branch,hpc,description FROM experiment e left join details d on e.id = d.exp_id WHERE (name LIKE '" + query + \ "%' OR description LIKE '%" + query + \ "%' OR user LIKE '%" + query + "%')" if exp_type and len(exp_type) > 0: @@ -302,7 +302,7 @@ def get_current_running_exp(): Log.error( 'Connection to database could not be established: {0}', e.message) return False - query = "SELECT id,name,user,created,model,branch,hpc,description FROM listexp" + query = "SELECT id,name,user,created,model,branch,hpc,description FROM experiment e left join details d on e.id = d.exp_id" APIBasicConfig.read() # print(query) cursor.execute(query) diff --git a/autosubmit_api/experiment/common_db_requests.py b/autosubmit_api/experiment/common_db_requests.py index 187cd45f8aa8f4771383588c2f2fa25da2cd4c47..e8aa22fa4e2add6edad197839a2becf0a0287401 100644 --- a/autosubmit_api/experiment/common_db_requests.py +++ b/autosubmit_api/experiment/common_db_requests.py @@ -110,9 +110,10 @@ def get_specific_experiment_status(expid): tables.experiment_status_table.c.name == expid ) ).one_or_none() + if row: + return (row.name, row.status) except Exception as exc: logger.error(f"Exception while reading experiment_status for {expid}: {exc}") logger.error(traceback.format_exc()) - if row: - return (row.name, row.status) + return (expid, "NOT RUNNING") diff --git a/autosubmit_api/experiment/common_requests.py b/autosubmit_api/experiment/common_requests.py index f5a909502217aadfac338d06a6cd857b949a4262..f730d58d0fafc31c8523d10a09201e8a41a1bb2c 100644 --- a/autosubmit_api/experiment/common_requests.py +++ b/autosubmit_api/experiment/common_requests.py @@ -59,7 +59,7 @@ from autosubmit_api.builders.experiment_history_builder import ExperimentHistory from autosubmit_api.builders.configuration_facade_builder import ConfigurationFacadeDirector, AutosubmitConfigurationFacadeBuilder from autosubmit_api.builders.joblist_loader_builder import JobListLoaderBuilder, JobListLoaderDirector from autosubmit_api.components.jobs.job_support import JobSupport -from typing import Dict, Any, Optional +from typing import Dict, Any, Optional, Tuple import locale from autosubmitconfigparser.config.configcommon import AutosubmitConfig as Autosubmit4Config @@ -69,8 +69,7 @@ SAFE_TIME_LIMIT = 300 SAFE_TIME_LIMIT_STATUS = 180 -def get_experiment_stats(expid, filter_period, filter_type): - # type: (str, int, str) -> Dict[str, Any] +def get_experiment_stats(expid: str, filter_period: int, filter_type: str) -> Dict[str, Any]: """ Lite version of the stats generator from Autosubmit autosubmit.py """ @@ -151,8 +150,14 @@ def get_experiment_data(expid): 'db_historic_version': "NA"} try: autosubmit_config_facade = ConfigurationFacadeDirector(AutosubmitConfigurationFacadeBuilder(expid)).build_autosubmit_configuration_facade() - _, experiment_status = DbRequests.get_specific_experiment_status(expid) - result["running"] = (experiment_status == "RUNNING") + try: + _, experiment_status = DbRequests.get_specific_experiment_status(expid) + result["running"] = (experiment_status == "RUNNING") + except Exception as exc: + logger.warning((traceback.format_exc())) + logger.warning((f"Warning: Error in get_experiment_data: {exc}. Trying to get the status exhaustively.")) + _, _, is_running, _, _ = _is_exp_running(expid) + result["running"] = is_running result["path"] = autosubmit_config_facade.experiment_path result["owner_id"] = autosubmit_config_facade.get_owner_id() result["owner"] = autosubmit_config_facade.get_owner_name() @@ -181,7 +186,7 @@ def get_experiment_data(expid): except Exception as exp: result["error"] = True result["error_message"] = str(exp) - print((traceback.format_exc())) + logger.error((traceback.format_exc())) return result @@ -199,13 +204,12 @@ def get_current_status_log_plus(expid): "log_path": log_path} -def _is_exp_running(expid, time_condition=300): +def _is_exp_running(expid: str, time_condition=300) -> Tuple[bool, str, bool, int, str]: """ Tests if experiment is running :param expid: Experiment name :param time_condition: Time constraint, 120 by default. Represents max seconds before an experiment is considered as NOT RUNNING :return: (error (true if error is found, false otherwise), error_message, is_running (true if running, false otherwise), timediff, path_to_log) - :rtype: tuple (bool, string, bool, int) """ is_running = False error = False @@ -225,14 +229,13 @@ def _is_exp_running(expid, time_condition=300): expid, APIBasicConfig, ConfigParserFactory()) as_conf.reload() current_version = as_conf.get_version() - except Exception as exp: - # print(exp) - pass + except Exception as exc: + logger.warning("Error in getting current_version: " + str(exc)) look_old_folder = True if current_version is not None and (str(current_version).startswith( "3.11") or str(current_version).startswith("3.9") or str(current_version).startswith("3.12")) else False - pathlog_first = pathlog_aslog if look_old_folder == False else pathlog_tmp - pathlog_second = pathlog_aslog if look_old_folder == True else pathlog_tmp + pathlog_first = pathlog_aslog if look_old_folder is False else pathlog_tmp + pathlog_second = pathlog_aslog if look_old_folder is True else pathlog_tmp # print("Experiment {0} version {1} \nLook {2} \nLook {3}".format( # expid, current_version, pathlog_first, pathlog_second)) # print(pathlog) @@ -313,8 +316,8 @@ def get_experiment_summary(expid, log): job_package_reader = JobPackageReader(expid) try: job_package_reader.read() - except: - logger.warning("Failed to read job_packages") + except Exception as exc: + logger.warning(f"Failed to read job_packages: {exc}") # Basic data job_running_to_seconds = dict() job_running_to_runtext = dict() @@ -359,7 +362,6 @@ def get_experiment_summary(expid, log): queue_seconds = job_info.queue_time if job_info else 0 running_seconds = job_info.run_time if job_info else 0 status = job_info.status if job_info else "UNKNOWN" - energy = job_info.energy if job_info else 0 # Identifying SIM name_components = job_name.split('_') if "SIM" in name_components: @@ -536,7 +538,6 @@ def get_job_log(expid, logfile, nlines=150): error = False error_message = "" logcontent = [] - reading = "" APIBasicConfig.read() exp_paths = ExperimentPaths(expid) logfilepath = os.path.join(exp_paths.tmp_log_dir, logfile) @@ -752,7 +753,6 @@ def get_experiment_tree_structured(expid, log): """ try: - notransitive = False APIBasicConfig.read() # TODO: Encapsulate this following 2 lines or move to the parent function in app.py @@ -812,18 +812,12 @@ def get_experiment_counters(expid): # TODO: Update to current representation standards and classes def get_quick_view(expid): """ Lighter View """ - pkl_file_name = "" error = False error_message = "" #quick_view = list() view_data = [] quick_tree_view = deque() - source = JobList.get_sourcetag() - target = JobList.get_targettag() - sync = JobList.get_synctag() - check_mark = JobList.get_checkmark() jobs_in_pkl = {} - fakeAllJobs = [] total_count = completed_count = failed_count = running_count = queuing_count = 0 try: APIBasicConfig.read() @@ -840,8 +834,8 @@ def get_quick_view(expid): job_package_reader.read() job_to_package = job_package_reader.job_to_package package_to_package_id = job_package_reader.package_to_package_id - except: - logger.warning("Failed to read job_packages") + except Exception as exc: + logger.warning(f"Failed to read job_packages: {exc}") logger.debug(("Retrieving packages {0} seconds.".format( str(time.time() - now_)))) @@ -872,9 +866,9 @@ def get_quick_view(expid): status_text = str(common_utils.Status.VALUE_TO_KEY[status_code]) jobs_in_pkl[job_name] = ( status_code, status_color, status_text, out, err, priority, id_number) - except Exception as exp: + except Exception as exc: raise Exception( - "Autosubmit API couldn't open pkl file. If you are sure that your experiment is running correctly, try again.") + f"Autosubmit API couldn't open pkl file. If you are sure that your experiment is running correctly, try again: {exc}") total_count = len(list(jobs_in_pkl.keys())) @@ -884,9 +878,9 @@ def get_quick_view(expid): job_name] wrapper_tag = "" wrapper_id = 0 - wrapper_name = "" + # wrapper_name = "" if job_name in list(job_to_package.keys()): - wrapper_name = job_to_package[job_name] + # wrapper_name = job_to_package[job_name] wrapper_id = package_to_package_id[job_to_package[job_name]] wrapper_tag = " Wrapped " + \ wrapper_id + "" @@ -909,8 +903,8 @@ def get_quick_view(expid): # return {} # quick_tree_view = list(quick_tree_view) - except Exception as exp: - error_message = "Exception: {0}".format(str(exp)) + except Exception as exc: + error_message = "Exception: {0}".format(str(exc)) error = True logger.error(error_message) logger.error(traceback.format_exc()) @@ -989,7 +983,7 @@ def get_current_configuration_by_expid(expid: str, user_id: Optional[str]): experimentRun = historicalDatabase.get_max_id_experiment_run() currentMetadata = json.loads( experimentRun.metadata) if experimentRun and experimentRun.metadata else None - currentRunId = experimentRun.run_id if experimentRun else None + # currentRunId = experimentRun.run_id if experimentRun else None # Main keys = conf, exp, jobs, platforms, proj # Can we ignore proj by now? Including it. # TODO: Define which keys should be included in the answer @@ -1199,7 +1193,7 @@ def test_esarchive_status(): clatency = result[4] DbRequests.insert_archive_status( status, alatency, abandwith, clatency, cbandwidth, rtime) - except Exception as exp: + except Exception: print((traceback.format_exc())) # error_message = str(exp) @@ -1225,9 +1219,9 @@ def get_last_test_archive_status(): rtime) > 1 else None else: str_status = "OFFLINE" - except Exception as exp: + except Exception as exc: error = True - error_message = "" + error_message = exc return {"status": str_status, "error": error, @@ -1247,14 +1241,14 @@ def enforceLocal(log): try: try: locale.setlocale(locale.LC_ALL, 'en_GB.UTF-8') - except Exception as e: + except Exception: try: locale.setlocale(locale.LC_ALL, 'C.utf8') - except Exception as e: + except Exception: try: locale.setlocale(locale.LC_ALL, 'en_GB') - except Exception as e: + except Exception: locale.setlocale(locale.LC_ALL, 'es_ES') - except Exception as e: + except Exception: log.info("Locale C.utf8 is not found, using '{0}' as fallback".format("C")) locale.setlocale(locale.LC_ALL, 'C') diff --git a/docs/source/_static/openapi.json b/docs/source/_static/openapi.json index d5b24179325fc518a550325b286d3860813e05a3..9b19e49345f9b9d5cbecab77287f8c897e735f60 100644 --- a/docs/source/_static/openapi.json +++ b/docs/source/_static/openapi.json @@ -9,177 +9,772 @@ } }, "paths": { + "/v3/login": { + "post": { + "tags": ["v3"], + "summary": "Login", + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/tokentest": { + "get": { + "tags": ["v3"], + "summary": "Token test", + "responses": { + "200": { + "description": "Success" + } + } + }, + "post": { + "tags": ["v3"], + "summary": "Token test", + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/updatedesc": { + "get": { + "tags": ["v3"], + "summary": "Update description", + "responses": { + "200": { + "description": "Success" + } + } + }, + "post": { + "tags": ["v3"], + "summary": "Update description", + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/cconfig/{expid}": { + "get": { + "tags": ["v3"], + "summary": "Get current configuration", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/expinfo/{expid}": { + "get": { + "tags": ["v3"], + "summary": "Get experiment info", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/expcount/{expid}": { + "get": { + "tags": ["v3"], + "summary": "Get experiment counters", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/searchowner/{owner}/{exptype}/{onlyactive}": { + "get": { + "tags": ["v3"], + "summary": "Search owner", + "parameters": [ + { + "name": "owner", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "exptype", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "onlyactive", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/searchowner/{owner}": { + "get": { + "tags": ["v3"], + "summary": "Search owner", + "parameters": [ + { + "name": "owner", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/search/{expid}/{exptype}/{onlyactive}": { + "get": { + "tags": ["v3"], + "summary": "Search experiment", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "exptype", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "onlyactive", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/search/{expid}": { + "get": { + "tags": ["v3"], + "summary": "Search experiment", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, "/v3/running": { "get": { - "tags": [ - "v3" + "tags": ["v3"], + "summary": "Running experiments", + "operationId": "get-v3-running", + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "experiment": { + "type": "array", + "items": { + "type": "object", + "properties": { + "completed": { + "type": "integer" + }, + "description": { + "type": "string" + }, + "failed": { + "type": "integer" + }, + "hpc": { + "type": "string" + }, + "id": { + "type": "integer" + }, + "modified": { + "type": "string" + }, + "name": { + "type": "string" + }, + "queuing": { + "type": "integer" + }, + "running": { + "type": "integer" + }, + "status": { + "type": "string" + }, + "submitted": { + "type": "integer" + }, + "suspended": { + "type": "integer" + }, + "total": { + "type": "integer" + }, + "user": { + "type": "string" + }, + "version": { + "type": "string" + }, + "wrapper": { + "type": "string", + "nullable": true + } + } + } + } + } + }, + "examples": { + "Example 1": { + "value": { + "experiment": [ + { + "completed": 25, + "description": "Testing AS 4.0", + "failed": 8, + "hpc": "MARENOSTRUM4", + "id": 8192, + "modified": "2022-10-03 10:05:27", + "name": "a4yd", + "queuing": 0, + "running": 0, + "status": "RUNNING", + "submitted": 0, + "suspended": 0, + "total": 385, + "user": "root", + "version": "4.0.0b0", + "wrapper": null + }, + { + "completed": 88, + "description": "TEST_AS_4.0", + "failed": 0, + "hpc": "MARENOSTRUM4", + "id": 8202, + "modified": "2022-11-30 11:50:29", + "name": "t0an", + "queuing": 0, + "running": 0, + "status": "RUNNING", + "submitted": 0, + "suspended": 0, + "total": 91, + "user": "root", + "version": "4.0.0b0", + "wrapper": null + } + ] + } + } + } + } + } + } + } + } + }, + "/v3/runs/{expid}": { + "get": { + "tags": ["v3"], + "summary": "Get runs", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/ifrun/{expid}": { + "get": { + "tags": ["v3"], + "summary": "Get if running", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/logrun/{expid}": { + "get": { + "tags": ["v3"], + "summary": "Get status and log path experiment", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/summary/{expid}": { + "get": { + "tags": ["v3"], + "summary": "Get experiment summary", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/shutdown/{route}": { + "get": { + "tags": ["v3"], + "summary": "Shutdown route", + "parameters": [ + { + "name": "route", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/performance/{expid}": { + "get": { + "tags": ["v3"], + "summary": "Get experiment performance metrics", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/graph/{expid}/{layout}/{grouped}": { + "get": { + "tags": ["v3"], + "summary": "Get graph format", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "layout", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "grouped", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/tree/{expid}": { + "get": { + "tags": ["v3"], + "summary": "Get experiment tree", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/quick/{expid}": { + "get": { + "tags": ["v3"], + "summary": "Get quick view data", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/exprun/{expid}": { + "get": { + "tags": ["v3"], + "summary": "Get experiment run log", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/joblog/{logfile}": { + "get": { + "tags": ["v3"], + "summary": "Get job log from path", + "parameters": [ + { + "name": "logfile", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/pklinfo/{expid}/{timeStamp}": { + "get": { + "tags": ["v3"], + "summary": "Get experiment pklinfo", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "timeStamp", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/pkltreeinfo/{expid}/{timeStamp}": { + "get": { + "tags": ["v3"], + "summary": "Get experiment tree pklinfo", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "timeStamp", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/stats/{expid}/{filter_period}/{filter_type}": { + "get": { + "tags": ["v3"], + "summary": "Get experiment statistics", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "filter_period", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "filter_type", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/history/{expid}/{jobname}": { + "get": { + "tags": ["v3"], + "summary": "Get experiment job history", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "jobname", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } ], - "summary": "Running experiments", - "operationId": "get-v3-running", "responses": { "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "experiment": { - "type": "array", - "items": { - "type": "object", - "properties": { - "completed": { - "type": "integer" - }, - "description": { - "type": "string" - }, - "failed": { - "type": "integer" - }, - "hpc": { - "type": "string" - }, - "id": { - "type": "integer" - }, - "modified": { - "type": "string" - }, - "name": { - "type": "string" - }, - "queuing": { - "type": "integer" - }, - "running": { - "type": "integer" - }, - "status": { - "type": "string" - }, - "submitted": { - "type": "integer" - }, - "suspended": { - "type": "integer" - }, - "total": { - "type": "integer" - }, - "user": { - "type": "string" - }, - "version": { - "type": "string" - }, - "wrapper": { - "type": "string", - "nullable": true - } - } - } - } - }, - "x-examples": { - "Example 1": { - "experiment": [ - { - "completed": 25, - "description": "Testing AS 4.0", - "failed": 8, - "hpc": "MARENOSTRUM4", - "id": 8192, - "modified": "2022-10-03 10:05:27", - "name": "a4yd", - "queuing": 0, - "running": 0, - "status": "RUNNING", - "submitted": 0, - "suspended": 0, - "total": 385, - "user": "root", - "version": "4.0.0b0", - "wrapper": null - }, - { - "completed": 88, - "description": "TEST_AS_4.0", - "failed": 0, - "hpc": "MARENOSTRUM4", - "id": 8202, - "modified": "2022-11-30 11:50:29", - "name": "t0an", - "queuing": 0, - "running": 0, - "status": "RUNNING", - "submitted": 0, - "suspended": 0, - "total": 91, - "user": "root", - "version": "4.0.0b0", - "wrapper": null - } - ] - } - } - }, - "examples": { - "Example 1": { - "value": { - "experiment": [ - { - "completed": 25, - "description": "Testing AS 4.0", - "failed": 8, - "hpc": "MARENOSTRUM4", - "id": 8192, - "modified": "2022-10-03 10:05:27", - "name": "a4yd", - "queuing": 0, - "running": 0, - "status": "RUNNING", - "submitted": 0, - "suspended": 0, - "total": 385, - "user": "root", - "version": "4.0.0b0", - "wrapper": null - }, - { - "completed": 88, - "description": "TEST_AS_4.0", - "failed": 0, - "hpc": "MARENOSTRUM4", - "id": 8202, - "modified": "2022-11-30 11:50:29", - "name": "t0an", - "queuing": 0, - "running": 0, - "status": "RUNNING", - "submitted": 0, - "suspended": 0, - "total": 91, - "user": "root", - "version": "4.0.0b0", - "wrapper": null - } - ] - } - } - } - } + "description": "Success" + } + } + } + }, + "/v3/rundetail/{expid}/{runid}": { + "get": { + "tags": ["v3"], + "summary": "Get experiment run job detail", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" } + }, + { + "name": "runid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/filestatus/": { + "get": { + "tags": ["v3"], + "summary": "[LEGACY] Get file status", + "responses": { + "200": { + "description": "Success" } } } }, "/v4/experiments": { "get": { - "tags": [ - "v4" - ], + "tags": ["v4"], "summary": "Search experiments", "description": "", "operationId": "get-v4-experiments", @@ -210,11 +805,7 @@ "in": "query", "schema": { "type": "string", - "enum": [ - "test", - "operational", - "experiment" - ] + "enum": ["test", "operational", "experiment"] } }, { @@ -229,11 +820,7 @@ "in": "query", "schema": { "type": "string", - "enum": [ - "expid", - "created", - "description" - ] + "enum": ["expid", "created", "description"] } }, { @@ -346,55 +933,6 @@ } } } - }, - "x-examples": { - "Example 1": { - "experiments": [ - { - "completed": 0, - "description": "DE Model App", - "failed": 0, - "hpc": null, - "id": 9, - "modified": null, - "name": "a005", - "queuing": 0, - "running": 0, - "status": "NOT RUNNING", - "submitted": 0, - "suspended": 0, - "total": 0, - "user": null, - "version": "Unknown", - "wrapper": null - }, - { - "completed": 115, - "description": "DE downstream", - "failed": 0, - "hpc": "LOCAL", - "id": 11, - "modified": "2023-10-26 11:44:07", - "name": "a006", - "queuing": 0, - "running": 0, - "status": "NOT RUNNING", - "submitted": 0, - "suspended": 0, - "total": 115, - "user": "ltenorio", - "version": "4.0.95", - "wrapper": null - } - ], - "pagination": { - "page": 1, - "page_items": 2, - "page_size": 2, - "total_items": 9, - "total_pages": 5 - } - } } }, "examples": { @@ -456,9 +994,7 @@ }, "/v4/experiments/{expid}": { "get": { - "tags": [ - "v4" - ], + "tags": ["v4"], "summary": "Get experiment info", "operationId": "get-v4-experiments-expid", "parameters": [ @@ -471,14 +1007,16 @@ } } ], - "responses": {} + "responses": { + "200": { + "description": "OK" + } + } } }, "/v4/experiments/{expid}/jobs": { "get": { - "tags": [ - "v4" - ], + "tags": ["v4"], "summary": "List experiment jobs", "operationId": "get-v4-experiments-expid-jobs", "parameters": [ @@ -488,10 +1026,7 @@ "description": "Data view selector", "schema": { "type": "string", - "enum": [ - "base", - "quick" - ] + "enum": ["base", "quick"] } }, { @@ -548,9 +1083,7 @@ }, "/v4/experiments/{expid}/wrappers": { "get": { - "tags": [ - "v4" - ], + "tags": ["v4"], "summary": "List experiment wrappers", "operationId": "get-v4-experiments-expid-wrappers", "parameters": [ @@ -588,34 +1121,6 @@ } } } - }, - "x-examples": { - "Example 1": { - "wrappers": [ - { - "job_names": [ - "a6zi_18500101_fc0_1_SIM", - "a6zi_18500101_fc0_2_SIM" - ], - "wrapper_name": "a6zi_ASThread_17108563159474_528_10" - }, - { - "job_names": [ - "a6zi_18500101_fc0_3_SIM", - "a6zi_18500101_fc0_4_SIM", - "a6zi_18500101_fc0_5_SIM", - "a6zi_18500101_fc0_6_SIM", - "a6zi_18500101_fc0_7_SIM", - "a6zi_18500101_fc0_8_SIM", - "a6zi_18500101_fc0_9_SIM", - "a6zi_18500101_fc0_10_SIM", - "a6zi_18500101_fc0_11_SIM", - "a6zi_18500101_fc0_12_SIM" - ], - "wrapper_name": "a6zi_ASThread_17108816522924_528_10" - } - ] - } } } } @@ -626,9 +1131,7 @@ }, "/v4/auth/verify-token": { "get": { - "tags": [ - "v4" - ], + "tags": ["v4"], "summary": "Verify JWT Token", "operationId": "get-v4-auth-verify-token", "parameters": [ @@ -701,9 +1204,7 @@ }, "/v4/auth/cas/v2/login": { "get": { - "tags": [ - "v4" - ], + "tags": ["v4"], "summary": "CAS v2 Login", "operationId": "get-v4-auth-cas-v2-login", "parameters": [ @@ -803,9 +1304,7 @@ }, "/v4/auth/oauth2/github/login": { "get": { - "tags": [ - "v4" - ], + "tags": ["v4"], "summary": "GitHub Oauth2 App Login", "operationId": "get-v4-auth-oauth2-github-login", "parameters": [ @@ -867,10 +1366,12 @@ "type": "string" }, "token": { - "type": "string" + "type": "string", + "nullable": true }, "user": { - "type": "string" + "type": "string", + "nullable": true } } }, @@ -950,23 +1451,8 @@ "err_path_remote": { "type": "string" } - }, - "x-examples": { - "Example 1": { - "name": "a008_20200120_fc0_2_SIM", - "status": "COMPLETED", - "priority": 4, - "section": "SIM", - "date": "2020-01-20T00:00:00", - "member": "fc0", - "chunk": 2, - "out_path_local": "a008_20200120_fc0_2_SIM.20231027153508.out", - "err_path_local": "a008_20200120_fc0_2_SIM.20231027153508.err", - "out_path_remote": "a008_20200120_fc0_2_SIM.20231027153508.out", - "err_path_remote": "a008_20200120_fc0_2_SIM.20231027153508.err" - } } } } } -} \ No newline at end of file +} diff --git a/docs/source/conf.py b/docs/source/conf.py index 1af25c71cb3198d165a3593b3f06d1085a168e82..b591128366f419e93478fe067d4b57f699488efa 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -24,6 +24,10 @@ redoc = [ 'spec': '_static/openapi.json', # 'spec': 'https://petstore3.swagger.io/api/v3/openapi.json', # 'spec': 'https://petstore.swagger.io/v2/swagger.json' + 'opts': { + 'suppress-warnings': True, + 'hide-hostname': True + } } ] diff --git a/openapi.json b/openapi.json index d5b24179325fc518a550325b286d3860813e05a3..9b19e49345f9b9d5cbecab77287f8c897e735f60 100644 --- a/openapi.json +++ b/openapi.json @@ -9,177 +9,772 @@ } }, "paths": { + "/v3/login": { + "post": { + "tags": ["v3"], + "summary": "Login", + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/tokentest": { + "get": { + "tags": ["v3"], + "summary": "Token test", + "responses": { + "200": { + "description": "Success" + } + } + }, + "post": { + "tags": ["v3"], + "summary": "Token test", + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/updatedesc": { + "get": { + "tags": ["v3"], + "summary": "Update description", + "responses": { + "200": { + "description": "Success" + } + } + }, + "post": { + "tags": ["v3"], + "summary": "Update description", + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/cconfig/{expid}": { + "get": { + "tags": ["v3"], + "summary": "Get current configuration", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/expinfo/{expid}": { + "get": { + "tags": ["v3"], + "summary": "Get experiment info", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/expcount/{expid}": { + "get": { + "tags": ["v3"], + "summary": "Get experiment counters", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/searchowner/{owner}/{exptype}/{onlyactive}": { + "get": { + "tags": ["v3"], + "summary": "Search owner", + "parameters": [ + { + "name": "owner", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "exptype", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "onlyactive", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/searchowner/{owner}": { + "get": { + "tags": ["v3"], + "summary": "Search owner", + "parameters": [ + { + "name": "owner", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/search/{expid}/{exptype}/{onlyactive}": { + "get": { + "tags": ["v3"], + "summary": "Search experiment", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "exptype", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "onlyactive", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/search/{expid}": { + "get": { + "tags": ["v3"], + "summary": "Search experiment", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, "/v3/running": { "get": { - "tags": [ - "v3" + "tags": ["v3"], + "summary": "Running experiments", + "operationId": "get-v3-running", + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "experiment": { + "type": "array", + "items": { + "type": "object", + "properties": { + "completed": { + "type": "integer" + }, + "description": { + "type": "string" + }, + "failed": { + "type": "integer" + }, + "hpc": { + "type": "string" + }, + "id": { + "type": "integer" + }, + "modified": { + "type": "string" + }, + "name": { + "type": "string" + }, + "queuing": { + "type": "integer" + }, + "running": { + "type": "integer" + }, + "status": { + "type": "string" + }, + "submitted": { + "type": "integer" + }, + "suspended": { + "type": "integer" + }, + "total": { + "type": "integer" + }, + "user": { + "type": "string" + }, + "version": { + "type": "string" + }, + "wrapper": { + "type": "string", + "nullable": true + } + } + } + } + } + }, + "examples": { + "Example 1": { + "value": { + "experiment": [ + { + "completed": 25, + "description": "Testing AS 4.0", + "failed": 8, + "hpc": "MARENOSTRUM4", + "id": 8192, + "modified": "2022-10-03 10:05:27", + "name": "a4yd", + "queuing": 0, + "running": 0, + "status": "RUNNING", + "submitted": 0, + "suspended": 0, + "total": 385, + "user": "root", + "version": "4.0.0b0", + "wrapper": null + }, + { + "completed": 88, + "description": "TEST_AS_4.0", + "failed": 0, + "hpc": "MARENOSTRUM4", + "id": 8202, + "modified": "2022-11-30 11:50:29", + "name": "t0an", + "queuing": 0, + "running": 0, + "status": "RUNNING", + "submitted": 0, + "suspended": 0, + "total": 91, + "user": "root", + "version": "4.0.0b0", + "wrapper": null + } + ] + } + } + } + } + } + } + } + } + }, + "/v3/runs/{expid}": { + "get": { + "tags": ["v3"], + "summary": "Get runs", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/ifrun/{expid}": { + "get": { + "tags": ["v3"], + "summary": "Get if running", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/logrun/{expid}": { + "get": { + "tags": ["v3"], + "summary": "Get status and log path experiment", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/summary/{expid}": { + "get": { + "tags": ["v3"], + "summary": "Get experiment summary", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/shutdown/{route}": { + "get": { + "tags": ["v3"], + "summary": "Shutdown route", + "parameters": [ + { + "name": "route", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/performance/{expid}": { + "get": { + "tags": ["v3"], + "summary": "Get experiment performance metrics", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/graph/{expid}/{layout}/{grouped}": { + "get": { + "tags": ["v3"], + "summary": "Get graph format", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "layout", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "grouped", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/tree/{expid}": { + "get": { + "tags": ["v3"], + "summary": "Get experiment tree", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/quick/{expid}": { + "get": { + "tags": ["v3"], + "summary": "Get quick view data", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/exprun/{expid}": { + "get": { + "tags": ["v3"], + "summary": "Get experiment run log", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/joblog/{logfile}": { + "get": { + "tags": ["v3"], + "summary": "Get job log from path", + "parameters": [ + { + "name": "logfile", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/pklinfo/{expid}/{timeStamp}": { + "get": { + "tags": ["v3"], + "summary": "Get experiment pklinfo", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "timeStamp", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/pkltreeinfo/{expid}/{timeStamp}": { + "get": { + "tags": ["v3"], + "summary": "Get experiment tree pklinfo", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "timeStamp", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/stats/{expid}/{filter_period}/{filter_type}": { + "get": { + "tags": ["v3"], + "summary": "Get experiment statistics", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "filter_period", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "filter_type", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/history/{expid}/{jobname}": { + "get": { + "tags": ["v3"], + "summary": "Get experiment job history", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "jobname", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } ], - "summary": "Running experiments", - "operationId": "get-v3-running", "responses": { "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "experiment": { - "type": "array", - "items": { - "type": "object", - "properties": { - "completed": { - "type": "integer" - }, - "description": { - "type": "string" - }, - "failed": { - "type": "integer" - }, - "hpc": { - "type": "string" - }, - "id": { - "type": "integer" - }, - "modified": { - "type": "string" - }, - "name": { - "type": "string" - }, - "queuing": { - "type": "integer" - }, - "running": { - "type": "integer" - }, - "status": { - "type": "string" - }, - "submitted": { - "type": "integer" - }, - "suspended": { - "type": "integer" - }, - "total": { - "type": "integer" - }, - "user": { - "type": "string" - }, - "version": { - "type": "string" - }, - "wrapper": { - "type": "string", - "nullable": true - } - } - } - } - }, - "x-examples": { - "Example 1": { - "experiment": [ - { - "completed": 25, - "description": "Testing AS 4.0", - "failed": 8, - "hpc": "MARENOSTRUM4", - "id": 8192, - "modified": "2022-10-03 10:05:27", - "name": "a4yd", - "queuing": 0, - "running": 0, - "status": "RUNNING", - "submitted": 0, - "suspended": 0, - "total": 385, - "user": "root", - "version": "4.0.0b0", - "wrapper": null - }, - { - "completed": 88, - "description": "TEST_AS_4.0", - "failed": 0, - "hpc": "MARENOSTRUM4", - "id": 8202, - "modified": "2022-11-30 11:50:29", - "name": "t0an", - "queuing": 0, - "running": 0, - "status": "RUNNING", - "submitted": 0, - "suspended": 0, - "total": 91, - "user": "root", - "version": "4.0.0b0", - "wrapper": null - } - ] - } - } - }, - "examples": { - "Example 1": { - "value": { - "experiment": [ - { - "completed": 25, - "description": "Testing AS 4.0", - "failed": 8, - "hpc": "MARENOSTRUM4", - "id": 8192, - "modified": "2022-10-03 10:05:27", - "name": "a4yd", - "queuing": 0, - "running": 0, - "status": "RUNNING", - "submitted": 0, - "suspended": 0, - "total": 385, - "user": "root", - "version": "4.0.0b0", - "wrapper": null - }, - { - "completed": 88, - "description": "TEST_AS_4.0", - "failed": 0, - "hpc": "MARENOSTRUM4", - "id": 8202, - "modified": "2022-11-30 11:50:29", - "name": "t0an", - "queuing": 0, - "running": 0, - "status": "RUNNING", - "submitted": 0, - "suspended": 0, - "total": 91, - "user": "root", - "version": "4.0.0b0", - "wrapper": null - } - ] - } - } - } - } + "description": "Success" + } + } + } + }, + "/v3/rundetail/{expid}/{runid}": { + "get": { + "tags": ["v3"], + "summary": "Get experiment run job detail", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" } + }, + { + "name": "runid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v3/filestatus/": { + "get": { + "tags": ["v3"], + "summary": "[LEGACY] Get file status", + "responses": { + "200": { + "description": "Success" } } } }, "/v4/experiments": { "get": { - "tags": [ - "v4" - ], + "tags": ["v4"], "summary": "Search experiments", "description": "", "operationId": "get-v4-experiments", @@ -210,11 +805,7 @@ "in": "query", "schema": { "type": "string", - "enum": [ - "test", - "operational", - "experiment" - ] + "enum": ["test", "operational", "experiment"] } }, { @@ -229,11 +820,7 @@ "in": "query", "schema": { "type": "string", - "enum": [ - "expid", - "created", - "description" - ] + "enum": ["expid", "created", "description"] } }, { @@ -346,55 +933,6 @@ } } } - }, - "x-examples": { - "Example 1": { - "experiments": [ - { - "completed": 0, - "description": "DE Model App", - "failed": 0, - "hpc": null, - "id": 9, - "modified": null, - "name": "a005", - "queuing": 0, - "running": 0, - "status": "NOT RUNNING", - "submitted": 0, - "suspended": 0, - "total": 0, - "user": null, - "version": "Unknown", - "wrapper": null - }, - { - "completed": 115, - "description": "DE downstream", - "failed": 0, - "hpc": "LOCAL", - "id": 11, - "modified": "2023-10-26 11:44:07", - "name": "a006", - "queuing": 0, - "running": 0, - "status": "NOT RUNNING", - "submitted": 0, - "suspended": 0, - "total": 115, - "user": "ltenorio", - "version": "4.0.95", - "wrapper": null - } - ], - "pagination": { - "page": 1, - "page_items": 2, - "page_size": 2, - "total_items": 9, - "total_pages": 5 - } - } } }, "examples": { @@ -456,9 +994,7 @@ }, "/v4/experiments/{expid}": { "get": { - "tags": [ - "v4" - ], + "tags": ["v4"], "summary": "Get experiment info", "operationId": "get-v4-experiments-expid", "parameters": [ @@ -471,14 +1007,16 @@ } } ], - "responses": {} + "responses": { + "200": { + "description": "OK" + } + } } }, "/v4/experiments/{expid}/jobs": { "get": { - "tags": [ - "v4" - ], + "tags": ["v4"], "summary": "List experiment jobs", "operationId": "get-v4-experiments-expid-jobs", "parameters": [ @@ -488,10 +1026,7 @@ "description": "Data view selector", "schema": { "type": "string", - "enum": [ - "base", - "quick" - ] + "enum": ["base", "quick"] } }, { @@ -548,9 +1083,7 @@ }, "/v4/experiments/{expid}/wrappers": { "get": { - "tags": [ - "v4" - ], + "tags": ["v4"], "summary": "List experiment wrappers", "operationId": "get-v4-experiments-expid-wrappers", "parameters": [ @@ -588,34 +1121,6 @@ } } } - }, - "x-examples": { - "Example 1": { - "wrappers": [ - { - "job_names": [ - "a6zi_18500101_fc0_1_SIM", - "a6zi_18500101_fc0_2_SIM" - ], - "wrapper_name": "a6zi_ASThread_17108563159474_528_10" - }, - { - "job_names": [ - "a6zi_18500101_fc0_3_SIM", - "a6zi_18500101_fc0_4_SIM", - "a6zi_18500101_fc0_5_SIM", - "a6zi_18500101_fc0_6_SIM", - "a6zi_18500101_fc0_7_SIM", - "a6zi_18500101_fc0_8_SIM", - "a6zi_18500101_fc0_9_SIM", - "a6zi_18500101_fc0_10_SIM", - "a6zi_18500101_fc0_11_SIM", - "a6zi_18500101_fc0_12_SIM" - ], - "wrapper_name": "a6zi_ASThread_17108816522924_528_10" - } - ] - } } } } @@ -626,9 +1131,7 @@ }, "/v4/auth/verify-token": { "get": { - "tags": [ - "v4" - ], + "tags": ["v4"], "summary": "Verify JWT Token", "operationId": "get-v4-auth-verify-token", "parameters": [ @@ -701,9 +1204,7 @@ }, "/v4/auth/cas/v2/login": { "get": { - "tags": [ - "v4" - ], + "tags": ["v4"], "summary": "CAS v2 Login", "operationId": "get-v4-auth-cas-v2-login", "parameters": [ @@ -803,9 +1304,7 @@ }, "/v4/auth/oauth2/github/login": { "get": { - "tags": [ - "v4" - ], + "tags": ["v4"], "summary": "GitHub Oauth2 App Login", "operationId": "get-v4-auth-oauth2-github-login", "parameters": [ @@ -867,10 +1366,12 @@ "type": "string" }, "token": { - "type": "string" + "type": "string", + "nullable": true }, "user": { - "type": "string" + "type": "string", + "nullable": true } } }, @@ -950,23 +1451,8 @@ "err_path_remote": { "type": "string" } - }, - "x-examples": { - "Example 1": { - "name": "a008_20200120_fc0_2_SIM", - "status": "COMPLETED", - "priority": 4, - "section": "SIM", - "date": "2020-01-20T00:00:00", - "member": "fc0", - "chunk": 2, - "out_path_local": "a008_20200120_fc0_2_SIM.20231027153508.out", - "err_path_local": "a008_20200120_fc0_2_SIM.20231027153508.err", - "out_path_remote": "a008_20200120_fc0_2_SIM.20231027153508.out", - "err_path_remote": "a008_20200120_fc0_2_SIM.20231027153508.err" - } } } } } -} \ No newline at end of file +} diff --git a/setup.py b/setup.py index 8eb87b2f76389b0717447cd8068c1bcba08d5453..06ebda7f003432a77fe97c0595d3875631211173 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,6 @@ install_requires = [ "portalocker~=2.6.0", "networkx~=2.6.3", "scipy~=1.7.3", - "paramiko~=2.12.0", "python-dotenv", "autosubmitconfigparser~=1.0.48", "autosubmit>=3.13", @@ -38,8 +37,7 @@ install_requires = [ "gunicorn", "pydantic~=2.5.2", "SQLAlchemy~=2.0.23", - "python-cas>=1.6.0", - "Authlib>=1.3.0" + "python-cas>=1.6.0" ] # Test dependencies diff --git a/tests/bgtasks/test_status_updater.py b/tests/bgtasks/test_status_updater.py index 6d2302956f97ffeb4eb5df736b4e3dfa6fbdece4..c6975a9b5f8fb5e4a72886e4e025b9c40a7db729 100644 --- a/tests/bgtasks/test_status_updater.py +++ b/tests/bgtasks/test_status_updater.py @@ -6,22 +6,25 @@ from autosubmit_api.database.common import ( ) from autosubmit_api.history.database_managers.database_models import RunningStatus -class TestStatusUpdater: +class TestStatusUpdater: def test_same_tables(self, fixture_mock_basic_config): prepare_db() - + + with create_as_times_db_engine().connect() as conn: + exps_status = conn.execute(tables.experiment_status_table.delete()) + StatusUpdater.run() - + with create_autosubmit_db_engine().connect() as conn: experiments = conn.execute(tables.experiment_table.select()).all() - + with create_as_times_db_engine().connect() as conn: exps_status = conn.execute(tables.experiment_status_table.select()).all() assert len(experiments) == len(exps_status) assert set([x.id for x in experiments]) == set([x.exp_id for x in exps_status]) assert set([x.name for x in experiments]) == set([x.name for x in exps_status]) - + for e_st in exps_status: assert e_st.status in [RunningStatus.RUNNING, RunningStatus.NOT_RUNNING] diff --git a/tests/experiments/autosubmit.db b/tests/experiments/autosubmit.db index fdca9705b5a01cac55d77fb350ce34b868e060df..472787336d33e91b529d921869dcc82dea543426 100644 Binary files a/tests/experiments/autosubmit.db and b/tests/experiments/autosubmit.db differ diff --git a/tests/experiments/metadata/graph/graph_data_a003.db b/tests/experiments/metadata/graph/graph_data_a003.db index 41b2c4f84c4a700e3a35394a2aea257543e762d4..1862073cccbab88a43d010644c5c6316b4202aa1 100755 Binary files a/tests/experiments/metadata/graph/graph_data_a003.db and b/tests/experiments/metadata/graph/graph_data_a003.db differ diff --git a/tests/test_database.py b/tests/test_database.py new file mode 100644 index 0000000000000000000000000000000000000000..518523b3705220b6a9d8bcb41fe485f215c7b72b --- /dev/null +++ b/tests/test_database.py @@ -0,0 +1,29 @@ +import os + +from sqlalchemy import select + +from autosubmit_api.database import common, tables + + +def count_pid_lsof(pid): + openfiles = os.popen(f"lsof -p {pid} | grep db").read() + print(openfiles) + return len([x for x in openfiles.strip("\n ").split("\n") if len(x.strip()) > 0]) + + +class TestDatabase: + + def test_open_files(self, fixture_mock_basic_config): + current_pid = os.getpid() + + counter = count_pid_lsof(current_pid) + + engine = common.create_autosubmit_db_engine() + + assert counter == count_pid_lsof(current_pid) + + with engine.connect() as conn: + conn.execute(select(tables.ExperimentTable)) + assert counter + 1 == count_pid_lsof(current_pid) + + assert counter == count_pid_lsof(current_pid)