diff --git a/CHANGELOG.md b/CHANGELOG.md index d5fc8df2fc0d6376725a2cf5b1652d48f256ab97..f5417483fbbdb4242063925140e4cac11c83f6b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # CHANGELOG +### Pre-release v4.0.1b3 - Release date: TBD + +* Added more CLI options related to gunicorn workers +* Fixed inconsistency in performance metrics calculations due to incorrect chunk size +* General code improvements + ### Pre-release v4.0.1b2 - Release date: 2024-07-18 * Fix threshold value on outlier detection algorithm. diff --git a/autosubmit_api/blueprints/v4.py b/autosubmit_api/blueprints/v4.py index f4092e52d7edea9b3fb234f7f95f08c388b54422..dd619800c31115bf1c362929f81c0853a8d92ba4 100644 --- a/autosubmit_api/blueprints/v4.py +++ b/autosubmit_api/blueprints/v4.py @@ -21,9 +21,6 @@ def create_v4_blueprint(): # blueprint.route("/experiments//description", methods=["PUT"])( # v4_views.experiment_description_view # ) - # blueprint.route("/experiments//config")( - # v3_views.get_current_configuration - # ) # blueprint.route("/experiments//info")(v3_views.exp_info) # blueprint.route("/experiments//status-counters")( # v3_views.exp_counters @@ -47,6 +44,18 @@ def create_v4_blueprint(): "/experiments//wrappers", view_func=v4_views.ExperimentWrappersView.as_view("ExperimentWrappersView"), ) + blueprint.add_url_rule( + "/experiments//filesystem-config", + view_func=v4_views.ExperimentFSConfigView.as_view("ExperimentFSConfigView"), + ) + blueprint.add_url_rule( + "/experiments//runs", + view_func=v4_views.ExperimentRunsView.as_view("ExperimentRunsView"), + ) + blueprint.add_url_rule( + "experiments//runs//config", + view_func=v4_views.ExperimentRunConfigView.as_view("ExperimentRunConfigView"), + ) # blueprint.route("/experiments//runs")(v3_views.get_runs) # blueprint.route("/experiments//check-running")( diff --git a/autosubmit_api/history/experiment_history.py b/autosubmit_api/history/experiment_history.py index b3dcbce87589bb2f3dcff88c8a3f961af13ef24f..635be988e5ce28c6d80d68570a9fcc868f4c8850 100644 --- a/autosubmit_api/history/experiment_history.py +++ b/autosubmit_api/history/experiment_history.py @@ -101,6 +101,12 @@ class ExperimentHistory(): "err": job_data_dc.err }) return result + + def get_experiment_runs(self) -> List[ExperimentRun]: + """ + Gets all the experiment runs + """ + return self.manager.get_experiment_runs_dcs() def get_all_jobs_last_run_dict(self) -> Dict[str, Optional[ExperimentRun]]: """ diff --git a/autosubmit_api/views/v4.py b/autosubmit_api/views/v4.py index 3cdf2238f357c6874d32b30659c45ed672aff6f5..a772856508bed9504461ce2aa2beaccac07595b5 100644 --- a/autosubmit_api/views/v4.py +++ b/autosubmit_api/views/v4.py @@ -1,10 +1,11 @@ from collections import deque -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from enum import Enum from http import HTTPStatus +import json import math import traceback -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional from flask import redirect, request from flask.views import MethodView import jwt @@ -17,17 +18,22 @@ from autosubmit_api.builders.experiment_history_builder import ( ExperimentHistoryDirector, ) from autosubmit_api.common.utils import Status +from autosubmit_api.config.basicConfig import APIBasicConfig +from autosubmit_api.config.confConfigStrategy import confConfigStrategy +from autosubmit_api.config.config_common import AutosubmitConfigResolver from autosubmit_api.database import tables from autosubmit_api.database.common import ( create_main_db_conn, execute_with_limit_offset, ) +from autosubmit_api.database.db_jobdata import JobDataStructure from autosubmit_api.database.queries import generate_query_listexp_extended from autosubmit_api.logger import logger, with_log_run_times from cas import CASClient from autosubmit_api import config from autosubmit_api.persistance.job_package_reader import JobPackageReader from autosubmit_api.persistance.pkl_reader import PklReader +from bscearth.utils.config_parser import ConfigParserFactory PAGINATION_LIMIT_DEFAULT = 12 @@ -412,7 +418,6 @@ class ExperimentWrappersView(MethodView): decorators = [with_auth_token(), with_log_run_times(logger, "WRAPPERS")] def get(self, expid: str, user_id: Optional[str] = None): - job_package_reader = JobPackageReader(expid) job_package_reader.read() @@ -424,3 +429,114 @@ class ExperimentWrappersView(MethodView): logger.debug(wrappers) return {"wrappers": wrappers} + + +class ExperimentFSConfigView(MethodView): + decorators = [with_auth_token(), with_log_run_times(logger, "EXP_FS_CONFIG")] + + @staticmethod + def _format_config_response( + config: Dict[str, Any], is_as3: bool = False + ) -> Dict[str, Any]: + """ + Format the config response, removing some keys if it's an AS3 config + Also, add a key to indicate if the config is empty + :param config: The config to format + :param is_as3: If the config is an AS3 config + """ + ALLOWED_CONFIG_KEYS = ["conf", "exp", "jobs", "platforms", "proj"] + formatted_config = { + key: config[key] + for key in config + if not is_as3 or (key.lower() in ALLOWED_CONFIG_KEYS) + } + formatted_config["contains_nones"] = not config or ( + None in list(config.values()) + ) + return formatted_config + + def get(self, expid: str, user_id: Optional[str] = None): + """ + Get the filesystem config of an experiment + """ + # Read the config + APIBasicConfig.read() + as_config = AutosubmitConfigResolver( + expid, APIBasicConfig, ConfigParserFactory() + ) + is_as3 = isinstance(as_config._configWrapper, confConfigStrategy) + as_config.reload() + curr_fs_config: Dict[str, Any] = as_config.get_full_config_as_dict() + + # Format the response + response = { + "config": ExperimentFSConfigView._format_config_response( + curr_fs_config, is_as3 + ) + } + return response, HTTPStatus.OK + + +class ExperimentRunsView(MethodView): + decorators = [with_auth_token(), with_log_run_times(logger, "EXP_RUNS")] + + def get(self, expid: str, user_id: Optional[str] = None): + """ + List all the runs of an experiment + It returns minimal information about the runs + """ + try: + experiment_history = ExperimentHistoryDirector( + ExperimentHistoryBuilder(expid) + ).build_reader_experiment_history() + exp_runs = experiment_history.get_experiment_runs() + except Exception: + logger.error("Error while getting experiment runs") + logger.error(traceback.format_exc()) + return { + "message": "Error while getting experiment runs" + }, HTTPStatus.INTERNAL_SERVER_ERROR + + # Format the response + response = {"runs": []} + for run in exp_runs: + response["runs"].append( + { + "run_id": run.run_id, + "start": datetime.fromtimestamp(run.start, timezone.utc).isoformat( + timespec="seconds" + ) + if run.start > 0 + else None, + "finish": datetime.fromtimestamp( + run.finish, timezone.utc + ).isoformat(timespec="seconds") + if run.finish > 0 + else None, + } + ) + + return response, HTTPStatus.OK + + +class ExperimentRunConfigView(MethodView): + decorators = [with_auth_token(), with_log_run_times(logger, "EXP_RUN_CONFIG")] + + def get(self, expid: str, run_id: str, user_id: Optional[str] = None): + """ + Get the config of a specific run of an experiment + """ + historical_db = JobDataStructure(expid, APIBasicConfig) + experiment_run = historical_db.get_experiment_run_by_id(run_id=run_id) + metadata = ( + json.loads(experiment_run.metadata) + if experiment_run and experiment_run.metadata + else {} + ) + + # Format the response + response = { + "run_id": experiment_run.run_id if experiment_run else None, + "config": ExperimentFSConfigView._format_config_response(metadata), + } + return response, HTTPStatus.OK diff --git a/openapi.json b/openapi.json index 9b19e49345f9b9d5cbecab77287f8c897e735f60..26dd05791149e8410206e4f4840d773ce07bd69b 100644 --- a/openapi.json +++ b/openapi.json @@ -11,7 +11,9 @@ "paths": { "/v3/login": { "post": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Login", "responses": { "200": { @@ -22,7 +24,9 @@ }, "/v3/tokentest": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Token test", "responses": { "200": { @@ -31,7 +35,9 @@ } }, "post": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Token test", "responses": { "200": { @@ -42,7 +48,9 @@ }, "/v3/updatedesc": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Update description", "responses": { "200": { @@ -51,7 +59,9 @@ } }, "post": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Update description", "responses": { "200": { @@ -62,7 +72,9 @@ }, "/v3/cconfig/{expid}": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Get current configuration", "parameters": [ { @@ -83,7 +95,9 @@ }, "/v3/expinfo/{expid}": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Get experiment info", "parameters": [ { @@ -104,7 +118,9 @@ }, "/v3/expcount/{expid}": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Get experiment counters", "parameters": [ { @@ -125,7 +141,9 @@ }, "/v3/searchowner/{owner}/{exptype}/{onlyactive}": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Search owner", "parameters": [ { @@ -162,7 +180,9 @@ }, "/v3/searchowner/{owner}": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Search owner", "parameters": [ { @@ -183,7 +203,9 @@ }, "/v3/search/{expid}/{exptype}/{onlyactive}": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Search experiment", "parameters": [ { @@ -220,7 +242,9 @@ }, "/v3/search/{expid}": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Search experiment", "parameters": [ { @@ -241,7 +265,9 @@ }, "/v3/running": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Running experiments", "operationId": "get-v3-running", "responses": { @@ -363,7 +389,9 @@ }, "/v3/runs/{expid}": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Get runs", "parameters": [ { @@ -384,7 +412,9 @@ }, "/v3/ifrun/{expid}": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Get if running", "parameters": [ { @@ -405,7 +435,9 @@ }, "/v3/logrun/{expid}": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Get status and log path experiment", "parameters": [ { @@ -426,7 +458,9 @@ }, "/v3/summary/{expid}": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Get experiment summary", "parameters": [ { @@ -447,7 +481,9 @@ }, "/v3/shutdown/{route}": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Shutdown route", "parameters": [ { @@ -468,7 +504,9 @@ }, "/v3/performance/{expid}": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Get experiment performance metrics", "parameters": [ { @@ -489,7 +527,9 @@ }, "/v3/graph/{expid}/{layout}/{grouped}": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Get graph format", "parameters": [ { @@ -526,7 +566,9 @@ }, "/v3/tree/{expid}": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Get experiment tree", "parameters": [ { @@ -547,7 +589,9 @@ }, "/v3/quick/{expid}": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Get quick view data", "parameters": [ { @@ -568,7 +612,9 @@ }, "/v3/exprun/{expid}": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Get experiment run log", "parameters": [ { @@ -589,7 +635,9 @@ }, "/v3/joblog/{logfile}": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Get job log from path", "parameters": [ { @@ -610,7 +658,9 @@ }, "/v3/pklinfo/{expid}/{timeStamp}": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Get experiment pklinfo", "parameters": [ { @@ -639,7 +689,9 @@ }, "/v3/pkltreeinfo/{expid}/{timeStamp}": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Get experiment tree pklinfo", "parameters": [ { @@ -668,7 +720,9 @@ }, "/v3/stats/{expid}/{filter_period}/{filter_type}": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Get experiment statistics", "parameters": [ { @@ -705,7 +759,9 @@ }, "/v3/history/{expid}/{jobname}": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Get experiment job history", "parameters": [ { @@ -734,7 +790,9 @@ }, "/v3/rundetail/{expid}/{runid}": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "Get experiment run job detail", "parameters": [ { @@ -763,7 +821,9 @@ }, "/v3/filestatus/": { "get": { - "tags": ["v3"], + "tags": [ + "v3" + ], "summary": "[LEGACY] Get file status", "responses": { "200": { @@ -774,7 +834,9 @@ }, "/v4/experiments": { "get": { - "tags": ["v4"], + "tags": [ + "v4" + ], "summary": "Search experiments", "description": "", "operationId": "get-v4-experiments", @@ -805,7 +867,11 @@ "in": "query", "schema": { "type": "string", - "enum": ["test", "operational", "experiment"] + "enum": [ + "test", + "operational", + "experiment" + ] } }, { @@ -820,7 +886,11 @@ "in": "query", "schema": { "type": "string", - "enum": ["expid", "created", "description"] + "enum": [ + "expid", + "created", + "description" + ] } }, { @@ -994,7 +1064,9 @@ }, "/v4/experiments/{expid}": { "get": { - "tags": ["v4"], + "tags": [ + "v4" + ], "summary": "Get experiment info", "operationId": "get-v4-experiments-expid", "parameters": [ @@ -1016,7 +1088,9 @@ }, "/v4/experiments/{expid}/jobs": { "get": { - "tags": ["v4"], + "tags": [ + "v4" + ], "summary": "List experiment jobs", "operationId": "get-v4-experiments-expid-jobs", "parameters": [ @@ -1026,7 +1100,10 @@ "description": "Data view selector", "schema": { "type": "string", - "enum": ["base", "quick"] + "enum": [ + "base", + "quick" + ] } }, { @@ -1083,7 +1160,9 @@ }, "/v4/experiments/{expid}/wrappers": { "get": { - "tags": ["v4"], + "tags": [ + "v4" + ], "summary": "List experiment wrappers", "operationId": "get-v4-experiments-expid-wrappers", "parameters": [ @@ -1131,7 +1210,9 @@ }, "/v4/auth/verify-token": { "get": { - "tags": ["v4"], + "tags": [ + "v4" + ], "summary": "Verify JWT Token", "operationId": "get-v4-auth-verify-token", "parameters": [ @@ -1204,7 +1285,9 @@ }, "/v4/auth/cas/v2/login": { "get": { - "tags": ["v4"], + "tags": [ + "v4" + ], "summary": "CAS v2 Login", "operationId": "get-v4-auth-cas-v2-login", "parameters": [ @@ -1304,7 +1387,9 @@ }, "/v4/auth/oauth2/github/login": { "get": { - "tags": ["v4"], + "tags": [ + "v4" + ], "summary": "GitHub Oauth2 App Login", "operationId": "get-v4-auth-oauth2-github-login", "parameters": [ @@ -1390,6 +1475,139 @@ } } } + }, + "/v4/experiments/{expid}/filesystem-config": { + "get": { + "tags": [ + "v4" + ], + "summary": "Get Filesystem current configuration", + "operationId": "get-experiments-expid-filesystem-config", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "config": { + "type": "object" + } + } + } + } + } + } + } + } + }, + "/v4/experiments/{expid}/runs": { + "get": { + "tags": [ + "v4" + ], + "summary": "List runs", + "operationId": "get-v4-experiments-expid-runs", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "runs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "run_id": { + "type": "integer" + }, + "start": { + "type": "string" + }, + "finish": { + "type": "string" + } + } + } + } + } + } + } + } + } + } + } + }, + "/v4/experiments/{expid}/runs/{run_id}/config": { + "get": { + "tags": [ + "v4" + ], + "summary": "Get configuration of an experiment run", + "operationId": "get-v4-experiments-expid-runs-run_id-config", + "parameters": [ + { + "name": "expid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "run_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "run_id": { + "type": "integer" + }, + "config": { + "type": "object" + } + } + } + } + } + } + } + } } }, "components": { @@ -1455,4 +1673,4 @@ } } } -} +} \ No newline at end of file diff --git a/tests/test_endpoints_v4.py b/tests/test_endpoints_v4.py index 8bafa9d35908e6f9e3bd0bd45be3695cd8d80510..086108cdb5abf8018677ffc7d191d4ea1fe20971 100644 --- a/tests/test_endpoints_v4.py +++ b/tests/test_endpoints_v4.py @@ -183,3 +183,102 @@ class TestExperimentWrappers: assert isinstance(wrapper["wrapper_name"], str) and wrapper[ "wrapper_name" ].startswith(expid) + + +class TestExperimentFSConfig: + endpoint = "/v4/experiments/{expid}/filesystem-config" + + def test_fs_config(self, fixture_client: FlaskClient): + expid = "a6zj" + + response = fixture_client.get(self.endpoint.format(expid=expid)) + resp_obj: dict = response.get_json() + + assert isinstance(resp_obj, dict) + assert isinstance(resp_obj["config"], dict) + assert ( + isinstance(resp_obj["config"]["contains_nones"], bool) + and not resp_obj["config"]["contains_nones"] + ) + assert isinstance(resp_obj["config"]["JOBS"], dict) + assert isinstance(resp_obj["config"]["WRAPPERS"], dict) + assert isinstance(resp_obj["config"]["WRAPPERS"]["WRAPPER_V"], dict) + + def test_fs_config_v3_retro(self, fixture_client: FlaskClient): + expid = "a3tb" + + response = fixture_client.get(self.endpoint.format(expid=expid)) + resp_obj: dict = response.get_json() + + assert isinstance(resp_obj, dict) + assert isinstance(resp_obj["config"], dict) + + ALLOWED_CONFIG_KEYS = ["conf", "exp", "jobs", "platforms", "proj"] + assert len(resp_obj["config"].keys()) == len(ALLOWED_CONFIG_KEYS) + 1 + assert ( + isinstance(resp_obj["config"]["contains_nones"], bool) + and not resp_obj["config"]["contains_nones"] + ) + for key in ALLOWED_CONFIG_KEYS: + assert key in resp_obj["config"] + assert isinstance(resp_obj["config"][key], dict) + + +class TestExperimentRuns: + endpoint = "/v4/experiments/{expid}/runs" + + @pytest.mark.parametrize("expid, num_runs", [("a6zj", 1), ("a3tb", 51)]) + def test_runs(self, expid: str, num_runs: int, fixture_client: FlaskClient): + response = fixture_client.get(self.endpoint.format(expid=expid)) + resp_obj: dict = response.get_json() + + assert isinstance(resp_obj, dict) + assert isinstance(resp_obj["runs"], list) + assert len(resp_obj["runs"]) == num_runs + + for run in resp_obj["runs"]: + assert isinstance(run, dict) + assert isinstance(run["run_id"], int) + assert isinstance(run["start"], str) or run["start"] is None + assert isinstance(run["finish"], str) or run["finish"] is None + + +class TestExperimentRunConfig: + endpoint = "/v4/experiments/{expid}/runs/{run_id}/config" + + def test_run_config(self, fixture_client: FlaskClient): + expid = "a6zj" + run_id = 1 + + response = fixture_client.get(self.endpoint.format(expid=expid, run_id=run_id)) + resp_obj: dict = response.get_json() + + assert isinstance(resp_obj, dict) + assert isinstance(resp_obj["config"], dict) + assert ( + isinstance(resp_obj["config"]["contains_nones"], bool) + and not resp_obj["config"]["contains_nones"] + ) + assert isinstance(resp_obj["config"]["JOBS"], dict) + assert isinstance(resp_obj["config"]["WRAPPERS"], dict) + assert isinstance(resp_obj["config"]["WRAPPERS"]["WRAPPER_V"], dict) + + @pytest.mark.parametrize("run_id", [51, 48, 31]) + def test_run_config_v3_retro(self, run_id: int, fixture_client: FlaskClient): + expid = "a3tb" + + response = fixture_client.get(self.endpoint.format(expid=expid, run_id=run_id)) + resp_obj: dict = response.get_json() + + assert isinstance(resp_obj, dict) + assert isinstance(resp_obj["config"], dict) + + ALLOWED_CONFIG_KEYS = ["conf", "exp", "jobs", "platforms", "proj"] + assert len(resp_obj["config"].keys()) == len(ALLOWED_CONFIG_KEYS) + 1 + assert ( + isinstance(resp_obj["config"]["contains_nones"], bool) + and not resp_obj["config"]["contains_nones"] + ) + for key in ALLOWED_CONFIG_KEYS: + assert key in resp_obj["config"] + assert isinstance(resp_obj["config"][key], dict)