From 14f20c5ec782b2fa362a7593df799284f4628e8e Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Fri, 3 Nov 2023 15:05:13 +0100 Subject: [PATCH 01/26] update get_processors: handle PLATFORMS #28 --- README.md | 13 +++++++++ autosubmit_api/config/IConfigStrategy.py | 11 +++++--- autosubmit_api/config/__init__.py | 2 +- autosubmit_api/config/ymlConfigStrategy.py | 27 +++++++++++++------ tests/experiments/a003/conf/minimal.yml | 5 +++- .../a003/proj/git_project/as_conf/jobs.yml | 2 +- tests/test_endpoints.py | 5 ++++ 7 files changed, 50 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 6c612d6c..0703de41 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,19 @@ In this image you can see the flow of information in the **Autosubmit environmen * Gunicorn * Unit testing +## Installation +Autosubmit API can be easily installed via pip +```sh +pip install autosubmit-api # >=4.0 (recommended) +# Check installation +autosubmit_api start -h +``` + +Start the server: + +```sh +autosubmit_api start +``` diff --git a/autosubmit_api/config/IConfigStrategy.py b/autosubmit_api/config/IConfigStrategy.py index c1524fe5..56e3f483 100644 --- a/autosubmit_api/config/IConfigStrategy.py +++ b/autosubmit_api/config/IConfigStrategy.py @@ -123,7 +123,12 @@ class IConfigStrategy(ABC): pass @abstractmethod - def get_job_platform(self, section): + def get_job_platform(self, section: str) -> str: + """ + Gets wallclock for the given job type + :param section: job type + :return: wallclock time + """ pass @abstractmethod @@ -162,13 +167,11 @@ class IConfigStrategy(ABC): """ pass - def get_processors(self, section): + def get_processors(self, section: str) -> str: """ Gets processors needed for the given job type :param section: job type - :type section: str :return: wallclock time - :rtype: str """ pass diff --git a/autosubmit_api/config/__init__.py b/autosubmit_api/config/__init__.py index 95542ee6..14935853 100644 --- a/autosubmit_api/config/__init__.py +++ b/autosubmit_api/config/__init__.py @@ -7,7 +7,7 @@ JWT_SECRET = os.environ.get("SECRET_KEY", "M87;Z$,o5?MSC(/@#-LbzgE3PH-5ki.ZvS}N. JWT_ALGORITHM = "HS256" JWT_EXP_DELTA_SECONDS = 84000*5 # 5 days -RUN_BACKGROUND_TASKS_ON_START = True if os.environ.get("RUN_BACKGROUND_TASKS_ON_START") in ["True", "T", "true"] else False # Default dalse +RUN_BACKGROUND_TASKS_ON_START = os.environ.get("RUN_BACKGROUND_TASKS_ON_START") in ["True", "T", "true"] # Default false # CAS Stuff CAS_LOGIN_URL = os.environ.get("CAS_LOGIN_URL") # e.g: 'https://cas.bsc.es/cas/login' diff --git a/autosubmit_api/config/ymlConfigStrategy.py b/autosubmit_api/config/ymlConfigStrategy.py index a0eb76e2..20294d47 100644 --- a/autosubmit_api/config/ymlConfigStrategy.py +++ b/autosubmit_api/config/ymlConfigStrategy.py @@ -131,24 +131,24 @@ class ymlConfigStrategy(IConfigStrategy): def get_queue(self, section): return self._conf_parser.jobs_data[section].get('QUEUE', "") - def get_job_platform(self, section): - pass + def get_job_platform(self, section: str) -> str: + return self._conf_parser.jobs_data.get(section, {}).get("PLATFORM", "") def get_platform_queue(self, platform): logger.info("get_platform_queue") - return self._conf_parser.platforms_data[platform]["QUEUE"] + return self._conf_parser.experiment_data.get("PLATFORMS", {}).get(platform, {}).get("QUEUE") def get_platform_serial_queue(self, platform): logger.info("get_platform_serial_queue") - return self._conf_parser.platforms_data[platform]["SERIAL_QUEUE"] + return self._conf_parser.experiment_data.get("PLATFORMS", {}).get(platform, {}).get("SERIAL_QUEUE") def get_platform_project(self, platform): logger.info("get_platform_project") - return self._conf_parser.platforms_data[platform]["PROJECT"] + return self._conf_parser.experiment_data.get("PLATFORMS", {}).get(platform, {}).get("PROJECT") def get_platform_wallclock(self, platform): logger.info("get_platform_wallclock") - return self._conf_parser.platforms_data[platform].get('MAX_WALLCLOCK', "") + return self._conf_parser.experiment_data.get("PLATFORMS", {}).get(platform, {}).get('MAX_WALLCLOCK', "") def get_wallclock(self, section): return self._conf_parser.jobs_data[section].get('WALLCLOCK', '') @@ -157,8 +157,19 @@ class ymlConfigStrategy(IConfigStrategy): def get_synchronize(self, section): return self._conf_parser.get_synchronize(section) - def get_processors(self, section): - return self._conf_parser.jobs_data.get(section, {}).get("PROCESSORS", "1") + def get_processors(self, section: str) -> str: + # Check processors in job + processors = self._conf_parser.jobs_data.get(section, {}).get("PROCESSORS") + if processors: + return processors + + # Check processors in job platform + processors = self._conf_parser.experiment_data.get("PLATFORMS", {}).get(self.get_job_platform(section), {}).get("PROCESSORS") + if processors: + return processors + else: + return "1" + def get_threads(self, section): return self._conf_parser.get_threads(section) diff --git a/tests/experiments/a003/conf/minimal.yml b/tests/experiments/a003/conf/minimal.yml index ae4731fa..95f40818 100644 --- a/tests/experiments/a003/conf/minimal.yml +++ b/tests/experiments/a003/conf/minimal.yml @@ -23,4 +23,7 @@ GIT: PROJECT_BRANCH: "main" PROJECT_COMMIT: '' PROJECT_SUBMODULES: '' - FETCH_SINGLE_BRANCH: true \ No newline at end of file + FETCH_SINGLE_BRANCH: true +PLATFORMS: + MN4: + PROCESSORS: 16 \ No newline at end of file diff --git a/tests/experiments/a003/proj/git_project/as_conf/jobs.yml b/tests/experiments/a003/proj/git_project/as_conf/jobs.yml index 80078eb1..486e2bc3 100644 --- a/tests/experiments/a003/proj/git_project/as_conf/jobs.yml +++ b/tests/experiments/a003/proj/git_project/as_conf/jobs.yml @@ -18,7 +18,7 @@ JOBS: DEPENDENCIES: INI SIM-1 RUNNING: chunk WALLCLOCK: 00:05 - PROCESSORS: 8 + PLATFORM: MN4 POST: FILE: POST.sh DEPENDENCIES: SIM diff --git a/tests/test_endpoints.py b/tests/test_endpoints.py index d3dc6681..b40a6a25 100644 --- a/tests/test_endpoints.py +++ b/tests/test_endpoints.py @@ -16,6 +16,11 @@ class TestPerformance: result = PerformanceMetrics(expid, JobListHelperDirector(JobListHelperBuilder(expid)).build_job_list_helper()).to_json() assert result["Parallelization"] == 8 + def test_parallelization_platforms(self, fixture_mock_basic_config: fixture_mock_basic_config): + expid = "a003" + result = PerformanceMetrics(expid, JobListHelperDirector(JobListHelperBuilder(expid)).build_job_list_helper()).to_json() + assert result["Parallelization"] == 16 + class TestTree: -- GitLab From a719c736fd88862f375662a91de56f3e8f1bab14 Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Mon, 6 Nov 2023 10:05:48 +0100 Subject: [PATCH 02/26] init v4.0.0b2 --- CHANGELOG.md | 5 +++++ VERSION | 2 +- autosubmit_api/__init__.py | 4 ++-- autosubmit_api/config/config_common.py | 4 ++-- autosubmit_api/config/ymlConfigStrategy.py | 2 +- setup.py | 5 ++++- 6 files changed, 15 insertions(+), 7 deletions(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..59454234 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +# CHANGELOG + +## v4.0.0b2 - (2023-mm-dd) + +* Fix bug where `Parallelization` doesn't report the platform `PROCESSORS`. \ No newline at end of file diff --git a/VERSION b/VERSION index c9d9681b..0c076d6a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -4.0.0b1 +4.0.0b2 diff --git a/autosubmit_api/__init__.py b/autosubmit_api/__init__.py index 38aa811c..764d200b 100644 --- a/autosubmit_api/__init__.py +++ b/autosubmit_api/__init__.py @@ -1,3 +1,3 @@ -__version__ = "4.0.0b1" -__author__ = 'Luiggi Tenorio, Cristian Gutiérrez, Julian Berlin, Wilmer Uruchi' +__version__ = "4.0.0b2" +__author__ = 'Luiggi Tenorio, Bruno P. Kinoshita, Cristian Gutiérrez, Julian Berlin, Wilmer Uruchi' __credits__ = 'Barcelona Supercomputing Center' diff --git a/autosubmit_api/config/config_common.py b/autosubmit_api/config/config_common.py index 0bf282b3..1f2aad5e 100644 --- a/autosubmit_api/config/config_common.py +++ b/autosubmit_api/config/config_common.py @@ -48,10 +48,10 @@ class AutosubmitConfigResolver(object): # check which type of config files (AS3 or AS4) expdef_conf_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "expdef_" + expid + ".conf") if os.path.exists(expdef_conf_file): - logger.info("Setting AS3 Config strategy - conf") + # logger.info("Setting AS3 Config strategy - conf") self._configWrapper = confConfigStrategy(expid, basic_config, parser_factory, ".conf") else: - logger.info("Setting AS4 Config strategy - yml") + # logger.info("Setting AS4 Config strategy - yml") self._configWrapper = ymlConfigStrategy(expid, basic_config, parser_factory, ".yml") diff --git a/autosubmit_api/config/ymlConfigStrategy.py b/autosubmit_api/config/ymlConfigStrategy.py index 20294d47..e304cf30 100644 --- a/autosubmit_api/config/ymlConfigStrategy.py +++ b/autosubmit_api/config/ymlConfigStrategy.py @@ -48,7 +48,7 @@ class ymlConfigStrategy(IConfigStrategy): :type expid: str """ def __init__(self, expid, basic_config = APIBasicConfig, parser_factory = None, extension=".yml"): - logger.info("Creating AS4 Parser !!!!!") + # logger.info("Creating AS4 Parser !!!!!") self._conf_parser = Autosubmit4Config(expid, basic_config) self._conf_parser.reload(True) diff --git a/setup.py b/setup.py index 7c35cf58..cb54956f 100644 --- a/setup.py +++ b/setup.py @@ -10,13 +10,16 @@ current_path = path.abspath(path.dirname(__file__)) def get_version(): return autosubmit_api.__version__ +def get_authors(): + return autosubmit_api.__author__ + setup( name='autosubmit_api', version=get_version(), description='An extension to the Autosubmit package that serves its information as an API', url='https://earth.bsc.es/gitlab/es/autosubmit_api', - author='Luiggi Tenorio, Cristian Gutiérrez, Julian Berlin, Wilmer Uruchi', + author=get_authors(), author_email='support-autosubmit@bsc.es', license='GNU GPL', packages=find_packages(), -- GitLab From 4e41a40bbb86f14451bed2e5d20d99f19d2c1670 Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Mon, 6 Nov 2023 10:24:51 +0100 Subject: [PATCH 03/26] patch worker --- autosubmit_api/experiment/common_db_requests.py | 2 +- autosubmit_api/workers/business/populate_times.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/autosubmit_api/experiment/common_db_requests.py b/autosubmit_api/experiment/common_db_requests.py index 3fd7c13d..bf15bbc8 100644 --- a/autosubmit_api/experiment/common_db_requests.py +++ b/autosubmit_api/experiment/common_db_requests.py @@ -195,7 +195,7 @@ def _create_exp_times(row_content): """ try: conn = create_connection(DB_FILE_AS_TIMES) - sql = ''' INSERT INTO experiment_times(exp_id, name, created, modified, total_jobs, completed_jobs) VALUES(?,?,?,?,?,?) ''' + sql = ''' INSERT OR REPLACE INTO experiment_times(exp_id, name, created, modified, total_jobs, completed_jobs) VALUES(?,?,?,?,?,?) ''' # print(row_content) cur = conn.cursor() cur.execute(sql, row_content) diff --git a/autosubmit_api/workers/business/populate_times.py b/autosubmit_api/workers/business/populate_times.py index 2fd19fb5..0ba4ab60 100644 --- a/autosubmit_api/workers/business/populate_times.py +++ b/autosubmit_api/workers/business/populate_times.py @@ -68,13 +68,13 @@ def process_completed_times(time_condition=60): if current_table.get(exp_str, None) is None: # Pkl exists but is not registered in the table # INSERT - print("Pkl of " + exp_str + " exists but not in the table: INSERT") + # print("Pkl of " + exp_str + " exists but not in the table: INSERT") current_id = _process_pkl_insert_times(exp_str, full_path, timest, APIBasicConfig, DEBUG) _process_details_insert_or_update(exp_str, experiments_table_exp_id, experiments_table_exp_id in details_table_ids_set) else: exp_id, created, modified, total_jobs, completed_jobs = current_table[exp_str] time_diff = int(timest - modified) - print("Pkl of " + exp_str + " exists") + # print("Pkl of " + exp_str + " exists") current_id = _process_pkl_insert_times(exp_str, full_path, timest, APIBasicConfig, DEBUG) if time_diff > time_condition: # Update table -- GitLab From 4e4fd846a1a8bbae50def09827579ce73ec9992a Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Mon, 6 Nov 2023 14:20:59 +0100 Subject: [PATCH 04/26] fix cconfig compatibility #40 --- README.md | 2 +- autosubmit_api/app.py | 79 ++++++-------------- autosubmit_api/auth/__init__.py | 44 +++++++++++ autosubmit_api/config/__init__.py | 8 +- autosubmit_api/config/ymlConfigStrategy.py | 4 +- autosubmit_api/experiment/common_requests.py | 79 ++++++++++++-------- autosubmit_api/logger.py | 40 ++++++++++ 7 files changed, 163 insertions(+), 93 deletions(-) create mode 100644 autosubmit_api/auth/__init__.py create mode 100644 autosubmit_api/logger.py diff --git a/README.md b/README.md index 0703de41..381bf2b7 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ 1. [Overview](#overview) 2. [Autosubmit Big Picture](#autosubmit-big-picture) 3. [General Knowledge Requirements](#general-knowledge-requirements) -4. [Deployment](#deployment) +4. [Installation](#Installation) ## Overview diff --git a/autosubmit_api/app.py b/autosubmit_api/app.py index 9799534b..b4b8a444 100644 --- a/autosubmit_api/app.py +++ b/autosubmit_api/app.py @@ -17,20 +17,21 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . -from functools import wraps import os import sys -import time from datetime import datetime, timedelta +from typing import Optional import requests -import logging from flask_cors import CORS, cross_origin from flask import Flask, request, session, redirect +from autosubmit_api import __version__ as APIVersion +from autosubmit_api.auth import with_auth_token from autosubmit_api.database.extended_db import ExtendedDB from autosubmit_api.database.db_common import get_current_running_exp, update_experiment_description_owner from autosubmit_api.experiment import common_requests as CommonRequests from autosubmit_api.experiment import utils as Utiles +from autosubmit_api.logger import get_app_logger, with_log_run_times from autosubmit_api.performance.performance_metrics import PerformanceMetrics from autosubmit_api.database.db_common import search_experiment_by_id from autosubmit_api.config.basicConfig import APIBasicConfig @@ -42,23 +43,6 @@ from flask_apscheduler import APScheduler from autosubmit_api.workers import populate_details_db, populate_queue_run_times, populate_running_experiments, populate_graph, verify_complete from autosubmit_api.config import JWT_SECRET, JWT_ALGORITHM, JWT_EXP_DELTA_SECONDS, RUN_BACKGROUND_TASKS_ON_START, CAS_LOGIN_URL, CAS_VERIFY_URL -def with_log_run_times(_logger: logging.Logger, _tag: str): - def decorator(func): - @wraps(func) - def inner_wrapper(*args, **kwargs): - start_time = time.time() - path = "" - try: - path = request.path - except: - pass - _logger.info('{}|RECEIVED|{}'.format(_tag, path)) - response = func(*args, **kwargs) - _logger.info('{}|RTIME|{}|{:.3f}'.format(_tag, path,(time.time() - start_time))) - return response - - return inner_wrapper - return decorator def create_app(): """ @@ -73,10 +57,7 @@ def create_app(): D = Manager().dict() CORS(app) - gunicorn_logger = logging.getLogger('gunicorn.error') - app.logger.handlers = gunicorn_logger.handlers - app.logger.setLevel(gunicorn_logger.level) - + app.logger = get_app_logger() # Bind logger app.logger.info("PYTHON VERSION: " + sys.version) requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += 'HIGH:!DH:!aNULL' @@ -134,6 +115,13 @@ def create_app(): worker_verify_complete() worker_populate_graph() + @app.route('/') + def home(): + return { + "name": "Autosubmit API", + "version": APIVersion + } + # CAS Login @app.route('/login') def login(): @@ -175,7 +163,8 @@ def create_app(): @app.route('/updatedesc', methods=['GET', 'POST']) @cross_origin(expose_headers="Authorization") @with_log_run_times(app.logger, "UDESC") - def update_description(): + @with_auth_token() + def update_description(user_id: Optional[str] = None): """ Updates the description of an experiment. Requires authenticated user. """ @@ -185,51 +174,29 @@ def create_app(): body_data = request.json expid = body_data.get("expid", None) new_description = body_data.get("description", None) - current_token = request.headers.get("Authorization") - try: - jwt_token = jwt.decode(current_token, JWT_SECRET, JWT_ALGORITHM) - except jwt.ExpiredSignatureError: - jwt_token = {"user_id": None} - except Exception as exp: - jwt_token = {"user_id": None} - valid_user = jwt_token.get("user_id", None) - return update_experiment_description_owner(expid, new_description, valid_user) + return update_experiment_description_owner(expid, new_description, user_id), 200 if user_id else 401 @app.route('/tokentest', methods=['GET', 'POST']) @cross_origin(expose_headers="Authorization") @with_log_run_times(app.logger, "TTEST") - def test_token(): + @with_auth_token() + def test_token(user_id: Optional[str] = None): """ Tests if a token is still valid """ - current_token = request.headers.get("Authorization") - try: - jwt_token = jwt.decode(current_token, JWT_SECRET, JWT_ALGORITHM) - except jwt.ExpiredSignatureError: - jwt_token = {"user_id": None} - except Exception as exp: - print(exp) - jwt_token = {"user_id": None} - - valid_user = jwt_token.get("user_id", None) return { - "isValid": True if valid_user else False, - "message": "Session expired" if not valid_user else None - } + "isValid": True if user_id else False, + "message": "Unauthorized" if not user_id else None + }, 200 if user_id else 401 @app.route('/cconfig/', methods=['GET']) @cross_origin(expose_headers="Authorization") @with_log_run_times(app.logger, "CCONFIG") - def get_current_configuration(expid): - current_token = request.headers.get("Authorization") - try: - jwt_token = jwt.decode(current_token, JWT_SECRET, JWT_ALGORITHM) - except Exception as exp: - jwt_token = {"user_id": None} - valid_user = jwt_token.get("user_id", None) - result = CommonRequests.get_current_configuration_by_expid(expid, valid_user, app.logger) + @with_auth_token(response_on_fail=True) + def get_current_configuration(expid: str, user_id: Optional[str] = None): + result = CommonRequests.get_current_configuration_by_expid(expid, user_id) return result diff --git a/autosubmit_api/auth/__init__.py b/autosubmit_api/auth/__init__.py new file mode 100644 index 00000000..f4a650f2 --- /dev/null +++ b/autosubmit_api/auth/__init__.py @@ -0,0 +1,44 @@ +from functools import wraps +from flask import request +from jwt.jwt import JWT +from autosubmit_api.logger import logger +from autosubmit_api.config import AUTHORIZATION, JWT_ALGORITHM, JWT_SECRET + + +class AppAuthError(ValueError): + code = 401 + + +def with_auth_token(response_on_fail=False, raise_on_fail=False): + """ + Decorator that validates the Authorization token in a request. + + It adds the `user_id` variable inside the arguments of the wrapped function. + + :param response_on_fail: if `True` will return a Flask response + :param raise_on_fail: if `True` will raise an exception + :raises AppAuthError: if raise_on_fail=True and decoding fails + """ + def decorator(func): + @wraps(func) + def inner_wrapper(*args, **kwargs): + current_token = request.headers.get("Authorization") + + try: + jwt_token = JWT.decode( + current_token, JWT_SECRET, JWT_ALGORITHM) + except Exception as exp: + if AUTHORIZATION and raise_on_fail: + raise AppAuthError("User not authenticated") + if AUTHORIZATION and response_on_fail: + return {"error": True, "message": "Unauthorized"}, 401 + jwt_token = {"user_id": None} + + user_id = jwt_token.get("user_id", None) + logger.debug("decorator user_id: " + str(user_id)) + kwargs["user_id"] = user_id + + return func(*args, **kwargs) + + return inner_wrapper + return decorator diff --git a/autosubmit_api/config/__init__.py b/autosubmit_api/config/__init__.py index 14935853..607a8a58 100644 --- a/autosubmit_api/config/__init__.py +++ b/autosubmit_api/config/__init__.py @@ -3,13 +3,15 @@ from dotenv import load_dotenv load_dotenv() +# Authorization +AUTHORIZATION = os.environ.get("AUTHORIZATION") in ["TRUE", "true", "T", "True"] # Default false JWT_SECRET = os.environ.get("SECRET_KEY", "M87;Z$,o5?MSC(/@#-LbzgE3PH-5ki.ZvS}N.s09v>I#v8I'00THrA-:ykh3HX?") # WARNING: Always provide a SECRET_KEY for production JWT_ALGORITHM = "HS256" JWT_EXP_DELTA_SECONDS = 84000*5 # 5 days -RUN_BACKGROUND_TASKS_ON_START = os.environ.get("RUN_BACKGROUND_TASKS_ON_START") in ["True", "T", "true"] # Default false - -# CAS Stuff +## CAS Stuff CAS_LOGIN_URL = os.environ.get("CAS_LOGIN_URL") # e.g: 'https://cas.bsc.es/cas/login' CAS_VERIFY_URL = os.environ.get("CAS_VERIFY_URL") # e.g: 'https://cas.bsc.es/cas/serviceValidate' +# Startup options +RUN_BACKGROUND_TASKS_ON_START = os.environ.get("RUN_BACKGROUND_TASKS_ON_START") in ["True", "T", "true"] # Default false diff --git a/autosubmit_api/config/ymlConfigStrategy.py b/autosubmit_api/config/ymlConfigStrategy.py index e304cf30..9f16d138 100644 --- a/autosubmit_api/config/ymlConfigStrategy.py +++ b/autosubmit_api/config/ymlConfigStrategy.py @@ -116,10 +116,10 @@ class ymlConfigStrategy(IConfigStrategy): # print(self._conf_parser) #result["conf"] = get_data( self._conf_parser.experiment_data["CONF"]) if self._conf_parser else None #result["exp"] = get_data( self._conf_parser.experiment_data["CONF"]) if self._exp_parser else None - result["platforms"] = self._conf_parser.platforms_data if self._conf_parser.platforms_data else None + # result["platforms"] = self._conf_parser.platforms_data if self._conf_parser.platforms_data else None #result["jobs"] = get_data( self._conf_parser.experiment_data["JOBS"]) if self._conf_parser.experiment_data["JOBS"] else None #result["proj"] = get_data( self._conf_parser.experiment_data["CONF"] ) if self._proj_parser else None - return result + return self._conf_parser.experiment_data def get_full_config_as_json(self): diff --git a/autosubmit_api/experiment/common_requests.py b/autosubmit_api/experiment/common_requests.py index 0cb889a5..724d5714 100644 --- a/autosubmit_api/experiment/common_requests.py +++ b/autosubmit_api/experiment/common_requests.py @@ -28,9 +28,9 @@ import datetime import json import multiprocessing import subprocess -import logging from collections import deque +from autosubmit_api.config.confConfigStrategy import confConfigStrategy from autosubmit_api.database import db_common as db_common from autosubmit_api.experiment import common_db_requests as DbRequests from autosubmit_api.database import db_jobdata as JobData @@ -40,6 +40,7 @@ from autosubmit_api.components.jobs import utils as JUtils from autosubmit_api.autosubmit_legacy.job.job_list import JobList from autosubmit_api.autosubmit_legacy.job.job import Job +from autosubmit_api.logger import logger from autosubmit_api.performance.utils import calculate_SYPD_perjob from autosubmit_api.monitor.monitor import Monitor @@ -57,7 +58,7 @@ from autosubmit_api.builders.experiment_history_builder import ExperimentHistory from autosubmit_api.builders.configuration_facade_builder import ConfigurationFacadeDirector, AutosubmitConfigurationFacadeBuilder from autosubmit_api.builders.joblist_loader_builder import JobListLoaderBuilder, JobListLoaderDirector from autosubmit_api.components.jobs.job_support import JobSupport -from typing import Dict, Any +from typing import Dict, Any, Optional import locale from autosubmitconfigparser.config.configcommon import AutosubmitConfig as Autosubmit4Config @@ -66,8 +67,6 @@ APIBasicConfig.read() SAFE_TIME_LIMIT = 300 SAFE_TIME_LIMIT_STATUS = 180 -# global object for logging -logger = logging.getLogger('gunicorn.error') def get_experiment_stats(expid, filter_period, filter_type): # type: (str, int, str) -> Dict[str, Any] @@ -1245,13 +1244,11 @@ def get_job_history(expid, job_name): return {"error": error, "error_message": error_message, "history": result, "path_to_logs": path_to_job_logs} -def get_current_configuration_by_expid(expid, valid_user, log): +def get_current_configuration_by_expid(expid: str, user_id: Optional[str]): """ Gets the current configuration by expid. The procedure queries the historical database and the filesystem. :param expid: Experiment Identifier - :type expdi: str :return: configuration content formatted as a JSON object - :rtype: Dictionary """ error = False warning = False @@ -1259,20 +1256,24 @@ def get_current_configuration_by_expid(expid, valid_user, log): warning_message = "" currentRunConfig = {} currentFileSystemConfig = {} + ALLOWED_CONFIG_KEYS = ['conf', 'exp', 'jobs', 'platforms', 'proj'] - def removeParameterDuplication(currentDict, keyToRemove, exceptionsKeys=[]): + def removeParameterDuplication(currentDict: dict, keyToRemove: str, exceptionsKeys=[]): if "exp" in currentDict.keys() and isinstance(currentDict["exp"], dict): try: for k, nested_d in list(currentDict["exp"].items()): if k not in exceptionsKeys and isinstance(nested_d, dict): nested_d.pop(keyToRemove, None) - except Exception as exp: - log.info("Error while trying to eliminate duplicated key from config.") - pass + except Exception as exc: + logger.error(f"Error while trying to eliminate duplicated key from config: {exc}") + logger.error(traceback.format_exc()) try: - allowedConfigKeys = ['conf', 'exp', 'jobs', 'platforms', 'proj'] APIBasicConfig.read() + autosubmitConfig = AutosubmitConfigResolver( + expid, APIBasicConfig, ConfigParserFactory()) + is_as3 = isinstance(autosubmitConfig._configWrapper, confConfigStrategy) + historicalDatabase = JobData.JobDataStructure(expid, APIBasicConfig) experimentRun = historicalDatabase.get_max_id_experiment_run() currentMetadata = json.loads( @@ -1283,42 +1284,58 @@ def get_current_configuration_by_expid(expid, valid_user, log): # TODO: Define which keys should be included in the answer if currentMetadata: currentRunConfig = { - key: currentMetadata[key] for key in currentMetadata if key in allowedConfigKeys} - currentRunConfig["contains_nones"] = True if not currentMetadata or None in list(currentMetadata.values( - )) else False + key: currentMetadata[key] + for key in currentMetadata + if not is_as3 or (key.lower() in ALLOWED_CONFIG_KEYS) + } + currentRunConfig["contains_nones"] = ( + not currentMetadata or + None in list(currentMetadata.values()) + ) APIBasicConfig.read() - autosubmitConfig = AutosubmitConfigResolver( - expid, APIBasicConfig, ConfigParserFactory()) try: autosubmitConfig.reload() currentFileSystemConfigContent = autosubmitConfig.get_full_config_as_dict() if currentFileSystemConfigContent: currentFileSystemConfig = { - key: currentFileSystemConfigContent[key] for key in currentFileSystemConfigContent if key in allowedConfigKeys} - currentFileSystemConfig["contains_nones"] = True if not currentFileSystemConfigContent or None in list(currentFileSystemConfigContent.values( - )) else False - - except Exception as exp: + key: currentFileSystemConfigContent[key] + for key in currentFileSystemConfigContent + if not is_as3 or (key.lower() in ALLOWED_CONFIG_KEYS) + } + currentFileSystemConfig["contains_nones"] = ( + not currentFileSystemConfigContent or + ( None in list(currentFileSystemConfigContent.values()) ) + ) + + except Exception as exc: warning = True warning_message = "The filesystem system configuration can't be retrieved because '{}'".format( - exp) - logger.info(traceback.format_exc()) + exc) + logger.warning(warning_message) + logger.warning(traceback.format_exc()) currentFileSystemConfig["contains_nones"] = True - log.info(warning_message) - pass removeParameterDuplication(currentRunConfig, "EXPID", ["experiment"]) removeParameterDuplication(currentFileSystemConfig, "EXPID", ["experiment"]) - except Exception as exp: + except Exception as exc: error = True - error_message = str(exp) + error_message = str(exc) currentRunConfig["contains_nones"] = True currentFileSystemConfig["contains_nones"] = True - log.info("Exception while generating the configuration: " + error_message) - pass - return {"error": error, "error_message": error_message, "warning": warning, "warning_message": warning_message, "configuration_current_run": currentRunConfig, "configuration_filesystem": currentFileSystemConfig, "are_equal": currentRunConfig == currentFileSystemConfig} + logger.error("Exception while generating the configuration: " + error_message) + logger.error(traceback.format_exc()) + + return { + "error": error, + "error_message": error_message, + "warning": warning, + "warning_message": warning_message, + "configuration_current_run": currentRunConfig, + "configuration_filesystem": currentFileSystemConfig, + "are_equal": currentRunConfig == currentFileSystemConfig + } def get_experiment_runs(expid): diff --git a/autosubmit_api/logger.py b/autosubmit_api/logger.py new file mode 100644 index 00000000..16b14e42 --- /dev/null +++ b/autosubmit_api/logger.py @@ -0,0 +1,40 @@ +from functools import wraps +import logging +import time + +from flask import request + + +def with_log_run_times(_logger: logging.Logger, _tag: str = ""): + """ + Function decorator to log runtimes of the endpoints + """ + def decorator(func): + @wraps(func) + def inner_wrapper(*args, **kwargs): + start_time = time.time() + path = "" + try: + path = request.path + except: + pass + _logger.info('{}|RECEIVED|{}'.format(_tag, path)) + response = func(*args, **kwargs) + _logger.info('{}|RTIME|{}|{:.3f}'.format( + _tag, path, (time.time() - start_time))) + return response + + return inner_wrapper + return decorator + + +def get_app_logger() -> logging.Logger: + """ + Returns app logger + """ + _logger = logging.getLogger('gunicorn.error') + return _logger + + +# Logger instance for reutilization +logger = get_app_logger() -- GitLab From a211e9a8ac4f60d99f6a51d8a5535b2ec90a3b79 Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Tue, 7 Nov 2023 11:11:47 +0100 Subject: [PATCH 05/26] added difference calculation in cconfig --- CHANGELOG.md | 5 ++++- autosubmit_api/experiment/common_requests.py | 20 +++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59454234..7169d2fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,4 +2,7 @@ ## v4.0.0b2 - (2023-mm-dd) -* Fix bug where `Parallelization` doesn't report the platform `PROCESSORS`. \ No newline at end of file +* Fix bug where `Parallelization` doesn't report the platform `PROCESSORS` in the `/performance/` endpoint. +* Fixed bug that doesn't shows the full config of AS4 in the `/cconfig/` endpoint. +* Added differences calculation in the `/cconfig/` endpoint. +* Improved logging and code structure. \ No newline at end of file diff --git a/autosubmit_api/experiment/common_requests.py b/autosubmit_api/experiment/common_requests.py index 724d5714..766042bc 100644 --- a/autosubmit_api/experiment/common_requests.py +++ b/autosubmit_api/experiment/common_requests.py @@ -1268,6 +1268,22 @@ def get_current_configuration_by_expid(expid: str, user_id: Optional[str]): logger.error(f"Error while trying to eliminate duplicated key from config: {exc}") logger.error(traceback.format_exc()) + def sideDifferences(base_dict: dict, comparing_dict: dict): + diffs = set() + for key, value in base_dict.items(): + comp_value = comparing_dict.get(key) + if isinstance(value, dict) and isinstance(comp_value, dict): + aux_diffs = sideDifferences(value, comp_value) + for d in aux_diffs: + diffs.add(f"{key}.{d}") + else: + if isinstance(value, str) and isinstance(comp_value, int) or isinstance(value, int) and isinstance(comp_value, str): + if str(value) != str(comp_value): + diffs.add(key) + elif value != comp_value: + diffs.add(key) + return list(diffs) + try: APIBasicConfig.read() autosubmitConfig = AutosubmitConfigResolver( @@ -1327,6 +1343,7 @@ def get_current_configuration_by_expid(expid: str, user_id: Optional[str]): logger.error("Exception while generating the configuration: " + error_message) logger.error(traceback.format_exc()) + diffs = sideDifferences(currentFileSystemConfig, currentRunConfig) return { "error": error, "error_message": error_message, @@ -1334,7 +1351,8 @@ def get_current_configuration_by_expid(expid: str, user_id: Optional[str]): "warning_message": warning_message, "configuration_current_run": currentRunConfig, "configuration_filesystem": currentFileSystemConfig, - "are_equal": currentRunConfig == currentFileSystemConfig + "are_equal": len(diffs) == 0, + "differences": diffs } -- GitLab From 1400be4c1d2895be2bd5f63625dee45789e8d60f Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Tue, 7 Nov 2023 12:43:29 +0100 Subject: [PATCH 06/26] add parent differences #40 --- autosubmit_api/experiment/common_requests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/autosubmit_api/experiment/common_requests.py b/autosubmit_api/experiment/common_requests.py index 766042bc..ce5198b9 100644 --- a/autosubmit_api/experiment/common_requests.py +++ b/autosubmit_api/experiment/common_requests.py @@ -1274,6 +1274,8 @@ def get_current_configuration_by_expid(expid: str, user_id: Optional[str]): comp_value = comparing_dict.get(key) if isinstance(value, dict) and isinstance(comp_value, dict): aux_diffs = sideDifferences(value, comp_value) + if len(aux_diffs) > 0: + diffs.add(key) for d in aux_diffs: diffs.add(f"{key}.{d}") else: -- GitLab From 49761ff66e477ec9179f9835d046c1aed003ed19 Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Tue, 7 Nov 2023 13:41:09 +0100 Subject: [PATCH 07/26] add job status to quick endpoint --- autosubmit_api/experiment/common_requests.py | 23 ++++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/autosubmit_api/experiment/common_requests.py b/autosubmit_api/experiment/common_requests.py index ce5198b9..506bc1b6 100644 --- a/autosubmit_api/experiment/common_requests.py +++ b/autosubmit_api/experiment/common_requests.py @@ -1151,7 +1151,7 @@ def get_quick_view(expid): now_ = time.time() job_to_package, package_to_jobs, package_to_package_id, package_to_symbol = JobList.retrieve_packages( APIBasicConfig, expid) - print(("Retrieving packages {0} seconds.".format( + logger.debug(("Retrieving packages {0} seconds.".format( str(time.time() - now_)))) try: @@ -1202,16 +1202,16 @@ def get_quick_view(expid): 'out': "/" + out, 'err': "/" + err, }) + tree_job = {'title': Job.getTitle(job_name, status_color, status_text) + wrapper_tag, + 'refKey': job_name, + 'data': 'Empty', + 'children': [], + 'status': status_text, + } if status_code in [common_utils.Status.COMPLETED, common_utils.Status.WAITING, common_utils.Status.READY]: - quick_tree_view.append({'title': Job.getTitle(job_name, status_color, status_text) + wrapper_tag, - 'refKey': job_name, - 'data': 'Empty', - 'children': []}) + quick_tree_view.append(tree_job) else: - quick_tree_view.appendleft({'title': Job.getTitle(job_name, status_color, status_text) + wrapper_tag, - 'refKey': job_name, - 'data': 'Empty', - 'children': []}) + quick_tree_view.appendleft(tree_job) # return {} # quick_tree_view = list(quick_tree_view) else: @@ -1219,9 +1219,8 @@ def get_quick_view(expid): except Exception as exp: error_message = "Exception: {0}".format(str(exp)) error = True - print(error_message) - print((traceback.format_exc())) - pass + logger.error(error_message) + logger.error(traceback.format_exc()) return {"error": error, "error_message": error_message, "view_data": view_data, "tree_view": list(quick_tree_view), "total": total_count, "completed": completed_count, "failed": failed_count, "running": running_count, "queuing": queuing_count} -- GitLab From c82f2f1342fd5b8f871eac2f23b5f486d14599fa Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Tue, 7 Nov 2023 15:33:07 +0100 Subject: [PATCH 08/26] improve security level --- autosubmit_api/app.py | 217 ++++++++++-------- autosubmit_api/auth/__init__.py | 30 ++- autosubmit_api/config/__init__.py | 17 +- .../update_launch_autosubmit_API_conda.sh | 23 +- tests/custom_utils.py | 2 +- tests/test_auth.py | 70 ++++++ 6 files changed, 250 insertions(+), 109 deletions(-) create mode 100644 tests/test_auth.py diff --git a/autosubmit_api/app.py b/autosubmit_api/app.py index b4b8a444..fad0c56d 100644 --- a/autosubmit_api/app.py +++ b/autosubmit_api/app.py @@ -25,7 +25,7 @@ import requests from flask_cors import CORS, cross_origin from flask import Flask, request, session, redirect from autosubmit_api import __version__ as APIVersion -from autosubmit_api.auth import with_auth_token +from autosubmit_api.auth import AuthorizationLevels, with_auth_token from autosubmit_api.database.extended_db import ExtendedDB from autosubmit_api.database.db_common import get_current_running_exp, update_experiment_description_owner @@ -54,10 +54,10 @@ def create_app(): app = Flask(__name__) - D = Manager().dict() + D = Manager().dict() CORS(app) - app.logger = get_app_logger() # Bind logger + app.logger = get_app_logger() # Bind logger app.logger.info("PYTHON VERSION: " + sys.version) requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += 'HIGH:!DH:!aNULL' @@ -80,7 +80,7 @@ def create_app(): @with_log_run_times(app.logger, "WRKPOPDET") def worker_populate_details_db(): populate_details_db.main() - + @scheduler.task('interval', id='populate_queue_run_times', minutes=3) @with_log_run_times(app.logger, "WRKPOPQUE") def worker_populate_queue_run_times(): @@ -139,11 +139,14 @@ def create_app(): target_service = "{}{}/login".format(referrer, environment) if not ticket: - route_to_request_ticket = "{}?service={}".format(CAS_LOGIN_URL, target_service) + route_to_request_ticket = "{}?service={}".format( + CAS_LOGIN_URL, target_service) app.logger.info("Redirected to: " + str(route_to_request_ticket)) return redirect(route_to_request_ticket) - environment = environment if environment is not None else "autosubmitapp" # can be used to target the test environment - cas_verify_ticket_route = CAS_VERIFY_URL + '?service=' + target_service + '&ticket=' + ticket + # can be used to target the test environment + environment = environment if environment is not None else "autosubmitapp" + cas_verify_ticket_route = CAS_VERIFY_URL + \ + '?service=' + target_service + '&ticket=' + ticket response = requests.get(cas_verify_ticket_route) user = None if response: @@ -159,11 +162,10 @@ def create_app(): jwt_token = jwt.encode(payload, JWT_SECRET, JWT_ALGORITHM) return {'authenticated': True, 'user': user, 'token': jwt_token, 'message': "Token generated."} - @app.route('/updatedesc', methods=['GET', 'POST']) @cross_origin(expose_headers="Authorization") @with_log_run_times(app.logger, "UDESC") - @with_auth_token() + @with_auth_token(level=AuthorizationLevels.WRITEONLY) def update_description(user_id: Optional[str] = None): """ Updates the description of an experiment. Requires authenticated user. @@ -176,11 +178,10 @@ def create_app(): new_description = body_data.get("description", None) return update_experiment_description_owner(expid, new_description, user_id), 200 if user_id else 401 - @app.route('/tokentest', methods=['GET', 'POST']) @cross_origin(expose_headers="Authorization") @with_log_run_times(app.logger, "TTEST") - @with_auth_token() + @with_auth_token(level=AuthorizationLevels.WRITEONLY, response_on_fail=False) def test_token(user_id: Optional[str] = None): """ Tests if a token is still valid @@ -190,102 +191,113 @@ def create_app(): "message": "Unauthorized" if not user_id else None }, 200 if user_id else 401 - @app.route('/cconfig/', methods=['GET']) @cross_origin(expose_headers="Authorization") @with_log_run_times(app.logger, "CCONFIG") - @with_auth_token(response_on_fail=True) + @with_auth_token() def get_current_configuration(expid: str, user_id: Optional[str] = None): - result = CommonRequests.get_current_configuration_by_expid(expid, user_id) + result = CommonRequests.get_current_configuration_by_expid( + expid, user_id) return result - @app.route('/expinfo/', methods=['GET']) @with_log_run_times(app.logger, "EXPINFO") - def exp_info(expid): + @with_auth_token() + def exp_info(expid: str, user_id: Optional[str] = None): result = CommonRequests.get_experiment_data(expid) return result - @app.route('/expcount/', methods=['GET']) @with_log_run_times(app.logger, "EXPCOUNT") - def exp_counters(expid): + @with_auth_token() + def exp_counters(expid: str, user_id: Optional[str] = None): result = CommonRequests.get_experiment_counters(expid) return result - @app.route('/searchowner///', methods=['GET']) @app.route('/searchowner/', methods=['GET']) @with_log_run_times(app.logger, "SOWNER") - def search_owner(owner, exptype=None, onlyactive=None): + @with_auth_token() + def search_owner(owner, exptype=None, onlyactive=None, user_id: Optional[str] = None): """ Same output format as search_expid """ - result = search_experiment_by_id(searchString=None, owner=owner, typeExp=exptype, onlyActive=onlyactive) + result = search_experiment_by_id( + searchString=None, owner=owner, typeExp=exptype, onlyActive=onlyactive) return result - @app.route('/search///', methods=['GET']) @app.route('/search/', methods=['GET']) @with_log_run_times(app.logger, "SEARCH") - def search_expid(expid, exptype=None, onlyactive=None): - result = search_experiment_by_id(expid, owner=None, typeExp=exptype, onlyActive=onlyactive) + @with_auth_token() + def search_expid(expid, exptype=None, onlyactive=None, user_id: Optional[str] = None): + result = search_experiment_by_id( + expid, owner=None, typeExp=exptype, onlyActive=onlyactive) return result - @app.route('/running/', methods=['GET']) @with_log_run_times(app.logger, "RUN") - def search_running(): + @with_auth_token() + def search_running(user_id: Optional[str] = None): """ Returns the list of all experiments that are currently running. """ if 'username' in session: print(("USER {}".format(session['username']))) app.logger.info("Active proceses: " + str(D)) - #app.logger.info("Received Currently Running query ") + # app.logger.info("Received Currently Running query ") result = get_current_running_exp() return result - @app.route('/runs/', methods=['GET']) @with_log_run_times(app.logger, "ERUNS") - def get_runs(expid): + @with_auth_token() + def get_runs(expid, user_id: Optional[str] = None): """ Get list of runs of the same experiment from the historical db """ result = CommonRequests.get_experiment_runs(expid) return result - @app.route('/ifrun/', methods=['GET']) @with_log_run_times(app.logger, "IFRUN") - def get_if_running(expid): + @with_auth_token() + def get_if_running(expid, user_id: Optional[str] = None): result = CommonRequests.quick_test_run(expid) return result - @app.route('/logrun/', methods=['GET']) @with_log_run_times(app.logger, "LOGRUN") - def get_log_running(expid): + @with_auth_token() + def get_log_running(expid, user_id: Optional[str] = None): result = CommonRequests.get_current_status_log_plus(expid) return result - @app.route('/summary/', methods=['GET']) @with_log_run_times(app.logger, "SUMMARY") - def get_expsummary(expid): + @with_auth_token() + def get_expsummary(expid, user_id: Optional[str] = None): user = request.args.get("loggedUser", default="null", type=str) - if user != "null": lock.acquire(); D[os.getpid()] = [user, "summary", True]; lock.release(); + if user != "null": + lock.acquire() + D[os.getpid()] = [user, "summary", True] + lock.release() result = CommonRequests.get_experiment_summary(expid, app.logger) app.logger.info('Process: ' + str(os.getpid()) + " workers: " + str(D)) - if user != "null": lock.acquire(); D[os.getpid()] = [user, "summary", False]; lock.release(); - if user != "null": lock.acquire(); D.pop(os.getpid(), None); lock.release(); + if user != "null": + lock.acquire() + D[os.getpid()] = [user, "summary", False] + lock.release() + if user != "null": + lock.acquire() + D.pop(os.getpid(), None) + lock.release() return result - @app.route('/shutdown/') @with_log_run_times(app.logger, "SHUTDOWN") - def shutdown(route): + @with_auth_token() + def shutdown(route, user_id: Optional[str] = None): """ This function is invoked from the frontend (AS-GUI) to kill workers that are no longer needed. This call is common in heavy parts of the GUI such as the Tree and Graph generation or Summaries fetching. @@ -297,13 +309,14 @@ def create_app(): app.logger.info("Bad parameters for user and expid in route.") if user != "null": - app.logger.info('SHUTDOWN|DETAILS|route: ' + route + " user: " + user + " expid: " + expid) + app.logger.info('SHUTDOWN|DETAILS|route: ' + route + + " user: " + user + " expid: " + expid) try: # app.logger.info("user: " + user) # app.logger.info("expid: " + expid) app.logger.info("Workers before: " + str(D)) lock.acquire() - for k,v in list(D.items()): + for k, v in list(D.items()): if v[0] == user and v[1] == route and v[-1] == True: if v[2] == expid: D[k] = [user, route, expid, False] @@ -316,79 +329,103 @@ def create_app(): lock.release() app.logger.info("Workers now: " + str(D)) except Exception as exp: - app.logger.info("[CRITICAL] Could not shutdown process " + expid + " by user \"" + user + "\"") + app.logger.info( + "[CRITICAL] Could not shutdown process " + expid + " by user \"" + user + "\"") return "" - @app.route('/performance/', methods=['GET']) @with_log_run_times(app.logger, "PRF") - def get_exp_performance(expid): + @with_auth_token() + def get_exp_performance(expid, user_id: Optional[str] = None): result = {} try: - result = PerformanceMetrics(expid, JobListHelperDirector(JobListHelperBuilder(expid)).build_job_list_helper()).to_json() + result = PerformanceMetrics(expid, JobListHelperDirector( + JobListHelperBuilder(expid)).build_job_list_helper()).to_json() except Exception as exp: result = {"SYPD": None, - "ASYPD": None, - "RSYPD": None, - "CHSY": None, - "JPSY": None, - "Parallelization": None, - "considered": [], - "error": True, - "error_message": str(exp), - "warnings_job_data": [], - } + "ASYPD": None, + "RSYPD": None, + "CHSY": None, + "JPSY": None, + "Parallelization": None, + "considered": [], + "error": True, + "error_message": str(exp), + "warnings_job_data": [], + } return result - @app.route('/graph///', methods=['GET']) @with_log_run_times(app.logger, "GRAPH") - def get_list_format(expid, layout='standard', grouped='none'): + @with_auth_token() + def get_list_format(expid, layout='standard', grouped='none', user_id: Optional[str] = None): user = request.args.get("loggedUser", default="null", type=str) # app.logger.info("user: " + user) # app.logger.info("expid: " + expid) - if user != "null": lock.acquire(); D[os.getpid()] = [user, "graph", expid, True]; lock.release(); - result = CommonRequests.get_experiment_graph(expid, app.logger, layout, grouped) - app.logger.info('Process: ' + str(os.getpid()) + " graph workers: " + str(D)) - if user != "null": lock.acquire(); D[os.getpid()] = [user, "graph", expid, False]; lock.release(); - if user != "null": lock.acquire(); D.pop(os.getpid(), None); lock.release(); + if user != "null": + lock.acquire() + D[os.getpid()] = [user, "graph", expid, True] + lock.release() + result = CommonRequests.get_experiment_graph( + expid, app.logger, layout, grouped) + app.logger.info('Process: ' + str(os.getpid()) + + " graph workers: " + str(D)) + if user != "null": + lock.acquire() + D[os.getpid()] = [user, "graph", expid, False] + lock.release() + if user != "null": + lock.acquire() + D.pop(os.getpid(), None) + lock.release() return result - @app.route('/tree/', methods=['GET']) @with_log_run_times(app.logger, "TREE") - def get_exp_tree(expid): + @with_auth_token() + def get_exp_tree(expid, user_id: Optional[str] = None): user = request.args.get("loggedUser", default="null", type=str) # app.logger.info("user: " + user) # app.logger.info("expid: " + expid) - if user != "null": lock.acquire(); D[os.getpid()] = [user, "tree", expid, True]; lock.release(); - result = CommonRequests.get_experiment_tree_structured(expid, app.logger) - app.logger.info('Process: ' + str(os.getpid()) + " tree workers: " + str(D)) - if user != "null": lock.acquire(); D[os.getpid()] = [user, "tree", expid, False]; lock.release(); - if user != "null": lock.acquire(); D.pop(os.getpid(), None); lock.release(); + if user != "null": + lock.acquire() + D[os.getpid()] = [user, "tree", expid, True] + lock.release() + result = CommonRequests.get_experiment_tree_structured( + expid, app.logger) + app.logger.info('Process: ' + str(os.getpid()) + + " tree workers: " + str(D)) + if user != "null": + lock.acquire() + D[os.getpid()] = [user, "tree", expid, False] + lock.release() + if user != "null": + lock.acquire() + D.pop(os.getpid(), None) + lock.release() return result - @app.route('/quick/', methods=['GET']) @with_log_run_times(app.logger, "QUICK") - def get_quick_view_data(expid): + @with_auth_token(response_on_fail=True) + def get_quick_view_data(expid, user_id=None): result = CommonRequests.get_quick_view(expid) return result - @app.route('/exprun/', methods=['GET']) @with_log_run_times(app.logger, "LOG") - def get_experiment_running(expid): + @with_auth_token() + def get_experiment_running(expid, user_id: Optional[str] = None): """ Finds log and gets the last 150 lines """ result = CommonRequests.get_experiment_log_last_lines(expid) return result - @app.route('/joblog/', methods=['GET']) @with_log_run_times(app.logger, "JOBLOG") - def get_job_log_from_path(logfile): + @with_auth_token() + def get_job_log_from_path(logfile, user_id: Optional[str] = None): """ Get log from path """ @@ -397,49 +434,49 @@ def create_app(): result = CommonRequests.get_job_log(expid, logfile) return result - @app.route('/pklinfo//', methods=['GET']) @with_log_run_times(app.logger, "GPKL") - def get_experiment_pklinfo(expid, timeStamp): + @with_auth_token() + def get_experiment_pklinfo(expid, timeStamp, user_id: Optional[str] = None): result = CommonRequests.get_experiment_pkl(expid) return result - @app.route('/pkltreeinfo//', methods=['GET']) @with_log_run_times(app.logger, "TPKL") - def get_experiment_tree_pklinfo(expid, timeStamp): + @with_auth_token() + def get_experiment_tree_pklinfo(expid, timeStamp, user_id: Optional[str] = None): result = CommonRequests.get_experiment_tree_pkl(expid) return result - @app.route('/stats///') @with_log_run_times(app.logger, "STAT") - def get_experiment_statistics(expid, filter_period, filter_type): - result = CommonRequests.get_experiment_stats(expid, filter_period, filter_type) + @with_auth_token() + def get_experiment_statistics(expid, filter_period, filter_type, user_id: Optional[str] = None): + result = CommonRequests.get_experiment_stats( + expid, filter_period, filter_type) return result - @app.route('/history//') @with_log_run_times(app.logger, "HISTORY") - def get_exp_job_history(expid, jobname): + @with_auth_token() + def get_exp_job_history(expid, jobname, user_id: Optional[str] = None): result = CommonRequests.get_job_history(expid, jobname) return result - @app.route('/rundetail//') @with_log_run_times(app.logger, "RUNDETAIL") - def get_experiment_run_job_detail(expid, runid): + @with_auth_token() + def get_experiment_run_job_detail(expid, runid, user_id: Optional[str] = None): result = CommonRequests.get_experiment_tree_rundetail(expid, runid) return result - @app.route('/filestatus/') @with_log_run_times(app.logger, "FSTATUS") def get_file_status(): result = CommonRequests.get_last_test_archive_status() return result - return app -app = create_app() \ No newline at end of file + +app = create_app() diff --git a/autosubmit_api/auth/__init__.py b/autosubmit_api/auth/__init__.py index f4a650f2..0de47719 100644 --- a/autosubmit_api/auth/__init__.py +++ b/autosubmit_api/auth/__init__.py @@ -2,14 +2,30 @@ from functools import wraps from flask import request from jwt.jwt import JWT from autosubmit_api.logger import logger -from autosubmit_api.config import AUTHORIZATION, JWT_ALGORITHM, JWT_SECRET +from autosubmit_api.config import AUTHORIZATION_LEVEL, JWT_ALGORITHM, JWT_SECRET +from enum import IntEnum + + +class AuthorizationLevels(IntEnum): + ALL = 100 + WRITEONLY = 20 + NONE = 0 class AppAuthError(ValueError): code = 401 -def with_auth_token(response_on_fail=False, raise_on_fail=False): +def _parse_authorization_level_env(_var): + if _var == "NONE": + return AuthorizationLevels.NONE + elif _var == "WRITEONLY": + return AuthorizationLevels.WRITEONLY + + return AuthorizationLevels.ALL + + +def with_auth_token(level=AuthorizationLevels.ALL, response_on_fail=True, raise_on_fail=False): """ Decorator that validates the Authorization token in a request. @@ -22,15 +38,15 @@ def with_auth_token(response_on_fail=False, raise_on_fail=False): def decorator(func): @wraps(func) def inner_wrapper(*args, **kwargs): - current_token = request.headers.get("Authorization") - try: + current_token = request.headers.get("Authorization") jwt_token = JWT.decode( current_token, JWT_SECRET, JWT_ALGORITHM) - except Exception as exp: - if AUTHORIZATION and raise_on_fail: + except Exception as exc: + auth_level = _parse_authorization_level_env(AUTHORIZATION_LEVEL) + if level <= auth_level and raise_on_fail: raise AppAuthError("User not authenticated") - if AUTHORIZATION and response_on_fail: + if level <= auth_level and response_on_fail: return {"error": True, "message": "Unauthorized"}, 401 jwt_token = {"user_id": None} diff --git a/autosubmit_api/config/__init__.py b/autosubmit_api/config/__init__.py index 607a8a58..5ac7c156 100644 --- a/autosubmit_api/config/__init__.py +++ b/autosubmit_api/config/__init__.py @@ -4,14 +4,19 @@ from dotenv import load_dotenv load_dotenv() # Authorization -AUTHORIZATION = os.environ.get("AUTHORIZATION") in ["TRUE", "true", "T", "True"] # Default false -JWT_SECRET = os.environ.get("SECRET_KEY", "M87;Z$,o5?MSC(/@#-LbzgE3PH-5ki.ZvS}N.s09v>I#v8I'00THrA-:ykh3HX?") # WARNING: Always provide a SECRET_KEY for production +AUTHORIZATION_LEVEL = os.environ.get("AUTHORIZATION_LEVEL") +# WARNING: Always provide a SECRET_KEY for production +JWT_SECRET = os.environ.get( + "SECRET_KEY", "M87;Z$,o5?MSC(/@#-LbzgE3PH-5ki.ZvS}N.s09v>I#v8I'00THrA-:ykh3HX?") JWT_ALGORITHM = "HS256" JWT_EXP_DELTA_SECONDS = 84000*5 # 5 days -## CAS Stuff -CAS_LOGIN_URL = os.environ.get("CAS_LOGIN_URL") # e.g: 'https://cas.bsc.es/cas/login' -CAS_VERIFY_URL = os.environ.get("CAS_VERIFY_URL") # e.g: 'https://cas.bsc.es/cas/serviceValidate' +# CAS Stuff +# e.g: 'https://cas.bsc.es/cas/login' +CAS_LOGIN_URL = os.environ.get("CAS_LOGIN_URL") +# e.g: 'https://cas.bsc.es/cas/serviceValidate' +CAS_VERIFY_URL = os.environ.get("CAS_VERIFY_URL") # Startup options -RUN_BACKGROUND_TASKS_ON_START = os.environ.get("RUN_BACKGROUND_TASKS_ON_START") in ["True", "T", "true"] # Default false +RUN_BACKGROUND_TASKS_ON_START = os.environ.get("RUN_BACKGROUND_TASKS_ON_START") in [ + "True", "T", "true"] # Default false diff --git a/deployment/update_launch_autosubmit_API_conda.sh b/deployment/update_launch_autosubmit_API_conda.sh index f36f8f6b..8b770262 100644 --- a/deployment/update_launch_autosubmit_API_conda.sh +++ b/deployment/update_launch_autosubmit_API_conda.sh @@ -1,8 +1,10 @@ #!/bin/bash . ~/.bashrc +#set -xv + LOG_PATH=${PLOG} -UPDATE=false +UPDATE=true while getopts ":e:u" opt; do case "$opt" in @@ -13,19 +15,23 @@ done # Stop current instance of unicorn -pstree -ap | grep gunicorn | awk -F',' '{print $2}' | awk -F' ' '{print $1}' | head -n 1 | xargs -r kill -pstree -ap | grep gunicorn +# pstree -ap | grep gunicorn | awk -F',' '{print $2}' | awk -F' ' '{print $1}' | head -n 1 | xargs -r kill +pstree -ap | grep autosubmit_api | awk -F',' '{print $2}' | awk -F' ' '{print $1}' | head -n 1 | xargs -r kill +pstree -ap | grep autosubmit_api +# pstree -ap | grep gunicorn # activate conda environment conda activate autosubmit_api # if update to a new version we install it from pip if [ "${UPDATE}" = true ]; then - pip install autosubmit_api --upgrade + pip install ./AS_API_4 fi # prepare to launch echo "Set SECRET KEY" +export AUTHORIZATION_LEVEL='NONE' +echo "Set SECRET KEY" export SECRET_KEY='c&X= AuthorizationLevels.WRITEONLY + assert AuthorizationLevels.WRITEONLY > AuthorizationLevels.NONE + + def test_decorator(self, monkeypatch: pytest.MonkeyPatch): + """ + Test different authorization levels. + Setting an AUTHORIZATION_LEVEL=ALL will protect all routes no matter it's protection level. + If a route is set with level = NONE, will be always protected. + """ + + # Test on AuthorizationLevels.ALL + monkeypatch.setattr(auth, "_parse_authorization_level_env", + custom_return_value(AuthorizationLevels.ALL)) + + _, code = with_auth_token( + level=AuthorizationLevels.ALL)(dummy_response)() + assert code == 401 + + _, code = with_auth_token( + level=AuthorizationLevels.WRITEONLY)(dummy_response)() + assert code == 401 + + _, code = with_auth_token( + level=AuthorizationLevels.NONE)(dummy_response)() + assert code == 401 + + # Test on AuthorizationLevels.WRITEONLY + monkeypatch.setattr(auth, "_parse_authorization_level_env", + custom_return_value(AuthorizationLevels.WRITEONLY)) + + _, code = with_auth_token( + level=AuthorizationLevels.ALL)(dummy_response)() + assert code == 200 + + _, code = with_auth_token( + level=AuthorizationLevels.WRITEONLY)(dummy_response)() + assert code == 401 + + _, code = with_auth_token( + level=AuthorizationLevels.NONE)(dummy_response)() + assert code == 401 + + # Test on AuthorizationLevels.NONE + monkeypatch.setattr(auth, "_parse_authorization_level_env", + custom_return_value(AuthorizationLevels.NONE)) + + _, code = with_auth_token( + level=AuthorizationLevels.ALL)(dummy_response)() + assert code == 200 + + _, code = with_auth_token( + level=AuthorizationLevels.WRITEONLY)(dummy_response)() + assert code == 200 + + _, code = with_auth_token( + level=AuthorizationLevels.NONE)(dummy_response)() + assert code == 401 -- GitLab From 0dcc32e02cc0948b9d977e861f091bf621b81a82 Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Wed, 8 Nov 2023 11:07:27 +0100 Subject: [PATCH 09/26] important fix APIBasicConfig read --- autosubmit_api/app.py | 31 ++++++++++++++++--------- autosubmit_api/config/basicConfig.py | 34 ++++++++++++++++++++++++---- 2 files changed, 49 insertions(+), 16 deletions(-) diff --git a/autosubmit_api/app.py b/autosubmit_api/app.py index fad0c56d..829b0b80 100644 --- a/autosubmit_api/app.py +++ b/autosubmit_api/app.py @@ -54,22 +54,34 @@ def create_app(): app = Flask(__name__) + # Multiprocessing setup D = Manager().dict() + lock = Lock() + # CORS setup CORS(app) - app.logger = get_app_logger() # Bind logger + + # Logger binding + app.logger = get_app_logger() app.logger.info("PYTHON VERSION: " + sys.version) + # Enforce Language Locale + CommonRequests.enforceLocal(app.logger) + requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += 'HIGH:!DH:!aNULL' try: requests.packages.urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST += 'HIGH:!DH:!aNULL' except AttributeError: - # no pyopenssl support used / needed / available - pass + app.logger.warning('No pyopenssl support used / needed / available') - lock = Lock() + # Initial read config + APIBasicConfig.read() + app.logger.debug("API Basic config: " + str(APIBasicConfig().props())) - CommonRequests.enforceLocal(app.logger) + # Prepare DB + ext_db = ExtendedDB(APIBasicConfig.DB_DIR, + APIBasicConfig.DB_FILE, APIBasicConfig.AS_TIMES_DB) + ext_db.prepare_db() # Background Scheduler scheduler = APScheduler() @@ -101,12 +113,7 @@ def create_app(): def worker_populate_graph(): populate_graph.main() - # Prepare DB - config = APIBasicConfig() - config.read() - ext_db = ExtendedDB(config.DB_DIR, config.DB_FILE, config.AS_TIMES_DB) - ext_db.prepare_db() - + # Run workers on create_app if RUN_BACKGROUND_TASKS_ON_START: app.logger.info('Starting populate workers on init...') worker_populate_details_db() @@ -115,6 +122,8 @@ def create_app(): worker_verify_complete() worker_populate_graph() + ################################ ROUTES ################################ + @app.route('/') def home(): return { diff --git a/autosubmit_api/config/basicConfig.py b/autosubmit_api/config/basicConfig.py index 616eb826..fdb9e570 100644 --- a/autosubmit_api/config/basicConfig.py +++ b/autosubmit_api/config/basicConfig.py @@ -27,14 +27,14 @@ class APIBasicConfig(BasicConfig): Extended class to manage configuration for Autosubmit path, database and default values for new experiments in the Autosubmit API """ - GRAPHDATA_DIR = os.path.join('/esarchive', 'autosubmit', 'as_metadata', 'graph') - FILE_STATUS_DIR = os.path.join('/esarchive', 'autosubmit', 'as_metadata', 'test') + GRAPHDATA_DIR = os.path.join(os.path.expanduser('~'), 'autosubmit', 'as_metadata', 'graph') + FILE_STATUS_DIR = os.path.join(os.path.expanduser('~'), 'autosubmit', 'as_metadata', 'test') FILE_STATUS_DB = 'status.db' - ALLOWED_CLIENTS = set(['https://earth.bsc.es/']) + ALLOWED_CLIENTS = set([]) @staticmethod def __read_file_config(file_path): - super().__read_file_config(file_path) + # WARNING: Is unsafe to call this method directly. Doing APIBasicConfig.__read_file_config doesn't run BasicConfig.__read_file_config if not os.path.isfile(file_path): return @@ -50,4 +50,28 @@ class APIBasicConfig(BasicConfig): if parser.has_option('statusdb', 'filename'): APIBasicConfig.FILE_STATUS_DB = parser.get('statusdb', 'filename') if parser.has_option('clients', 'authorized'): - APIBasicConfig.ALLOWED_CLIENTS = set(parser.get('clients', 'authorized').split()) \ No newline at end of file + APIBasicConfig.ALLOWED_CLIENTS = set(parser.get('clients', 'authorized').split()) + + + @staticmethod + def read(): + BasicConfig.read() # This is done to run BasicConfig.__read_file_config indirectly + + filename = 'autosubmitrc' + if 'AUTOSUBMIT_CONFIGURATION' in os.environ and os.path.exists(os.environ['AUTOSUBMIT_CONFIGURATION']): + config_file_path = os.environ['AUTOSUBMIT_CONFIGURATION'] + # Call read_file_config with the value of the environment variable + APIBasicConfig.__read_file_config(config_file_path) + else: + if os.path.exists(os.path.join('', '.' + filename)): + APIBasicConfig.__read_file_config(os.path.join('', '.' + filename)) + elif os.path.exists(os.path.join(os.path.expanduser('~'), '.' + filename)): + APIBasicConfig.__read_file_config(os.path.join( + os.path.expanduser('~'), '.' + filename)) + else: + APIBasicConfig.__read_file_config(os.path.join('/etc', filename)) + + # Check if the environment variable is defined + + APIBasicConfig._update_config() + return \ No newline at end of file -- GitLab From 4825d86fc04ad1ed44a0d50315eb6513b35297f6 Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Wed, 8 Nov 2023 11:17:26 +0100 Subject: [PATCH 10/26] fix jwt dependency --- autosubmit_api/auth/__init__.py | 11 +++++++---- setup.py | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/autosubmit_api/auth/__init__.py b/autosubmit_api/auth/__init__.py index 0de47719..85caa59f 100644 --- a/autosubmit_api/auth/__init__.py +++ b/autosubmit_api/auth/__init__.py @@ -1,6 +1,6 @@ from functools import wraps from flask import request -from jwt.jwt import JWT +import jwt from autosubmit_api.logger import logger from autosubmit_api.config import AUTHORIZATION_LEVEL, JWT_ALGORITHM, JWT_SECRET from enum import IntEnum @@ -40,14 +40,17 @@ def with_auth_token(level=AuthorizationLevels.ALL, response_on_fail=True, raise_ def inner_wrapper(*args, **kwargs): try: current_token = request.headers.get("Authorization") - jwt_token = JWT.decode( + jwt_token = jwt.decode( current_token, JWT_SECRET, JWT_ALGORITHM) except Exception as exc: + error_msg = "Unauthorized" + if isinstance(exc, jwt.ExpiredSignatureError): + error_msg = "Expired token" auth_level = _parse_authorization_level_env(AUTHORIZATION_LEVEL) if level <= auth_level and raise_on_fail: - raise AppAuthError("User not authenticated") + raise AppAuthError(error_msg) if level <= auth_level and response_on_fail: - return {"error": True, "message": "Unauthorized"}, 401 + return {"error": True, "message": error_msg }, 401 jwt_token = {"user_id": None} user_id = jwt_token.get("user_id", None) diff --git a/setup.py b/setup.py index cb54956f..c461a915 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ setup( python_requires='>=3.7', install_requires=[ 'Flask~=2.2.5', - 'jwt~=1.3.1', + 'pyjwt~=2.8.0', 'requests~=2.28.1', 'flask_cors~=3.0.10', 'bscearth.utils~=0.5.2', -- GitLab From 1803466ddb45b9546d67e2f1f040e0429458dc3d Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Wed, 8 Nov 2023 11:25:39 +0100 Subject: [PATCH 11/26] refactor auth definition --- autosubmit_api/app.py | 6 +++--- autosubmit_api/auth/__init__.py | 20 ++++++++--------- autosubmit_api/config/__init__.py | 4 ++-- tests/test_auth.py | 36 +++++++++++++++---------------- 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/autosubmit_api/app.py b/autosubmit_api/app.py index 829b0b80..43e1a616 100644 --- a/autosubmit_api/app.py +++ b/autosubmit_api/app.py @@ -25,7 +25,7 @@ import requests from flask_cors import CORS, cross_origin from flask import Flask, request, session, redirect from autosubmit_api import __version__ as APIVersion -from autosubmit_api.auth import AuthorizationLevels, with_auth_token +from autosubmit_api.auth import ProtectionLevels, with_auth_token from autosubmit_api.database.extended_db import ExtendedDB from autosubmit_api.database.db_common import get_current_running_exp, update_experiment_description_owner @@ -174,7 +174,7 @@ def create_app(): @app.route('/updatedesc', methods=['GET', 'POST']) @cross_origin(expose_headers="Authorization") @with_log_run_times(app.logger, "UDESC") - @with_auth_token(level=AuthorizationLevels.WRITEONLY) + @with_auth_token(threshold=ProtectionLevels.WRITEONLY) def update_description(user_id: Optional[str] = None): """ Updates the description of an experiment. Requires authenticated user. @@ -190,7 +190,7 @@ def create_app(): @app.route('/tokentest', methods=['GET', 'POST']) @cross_origin(expose_headers="Authorization") @with_log_run_times(app.logger, "TTEST") - @with_auth_token(level=AuthorizationLevels.WRITEONLY, response_on_fail=False) + @with_auth_token(threshold=ProtectionLevels.WRITEONLY, response_on_fail=False) def test_token(user_id: Optional[str] = None): """ Tests if a token is still valid diff --git a/autosubmit_api/auth/__init__.py b/autosubmit_api/auth/__init__.py index 85caa59f..01510b37 100644 --- a/autosubmit_api/auth/__init__.py +++ b/autosubmit_api/auth/__init__.py @@ -2,11 +2,11 @@ from functools import wraps from flask import request import jwt from autosubmit_api.logger import logger -from autosubmit_api.config import AUTHORIZATION_LEVEL, JWT_ALGORITHM, JWT_SECRET +from autosubmit_api.config import PROTECTION_LEVEL, JWT_ALGORITHM, JWT_SECRET from enum import IntEnum -class AuthorizationLevels(IntEnum): +class ProtectionLevels(IntEnum): ALL = 100 WRITEONLY = 20 NONE = 0 @@ -16,16 +16,16 @@ class AppAuthError(ValueError): code = 401 -def _parse_authorization_level_env(_var): +def _parse_protection_level_env(_var): if _var == "NONE": - return AuthorizationLevels.NONE + return ProtectionLevels.NONE elif _var == "WRITEONLY": - return AuthorizationLevels.WRITEONLY + return ProtectionLevels.WRITEONLY - return AuthorizationLevels.ALL + return ProtectionLevels.ALL -def with_auth_token(level=AuthorizationLevels.ALL, response_on_fail=True, raise_on_fail=False): +def with_auth_token(threshold=ProtectionLevels.ALL, response_on_fail=True, raise_on_fail=False): """ Decorator that validates the Authorization token in a request. @@ -46,10 +46,10 @@ def with_auth_token(level=AuthorizationLevels.ALL, response_on_fail=True, raise_ error_msg = "Unauthorized" if isinstance(exc, jwt.ExpiredSignatureError): error_msg = "Expired token" - auth_level = _parse_authorization_level_env(AUTHORIZATION_LEVEL) - if level <= auth_level and raise_on_fail: + auth_level = _parse_protection_level_env(PROTECTION_LEVEL) + if threshold <= auth_level and raise_on_fail: raise AppAuthError(error_msg) - if level <= auth_level and response_on_fail: + if threshold <= auth_level and response_on_fail: return {"error": True, "message": error_msg }, 401 jwt_token = {"user_id": None} diff --git a/autosubmit_api/config/__init__.py b/autosubmit_api/config/__init__.py index 5ac7c156..c0ad1881 100644 --- a/autosubmit_api/config/__init__.py +++ b/autosubmit_api/config/__init__.py @@ -3,8 +3,8 @@ from dotenv import load_dotenv load_dotenv() -# Authorization -AUTHORIZATION_LEVEL = os.environ.get("AUTHORIZATION_LEVEL") +# Auth +PROTECTION_LEVEL = os.environ.get("PROTECTION_LEVEL") # WARNING: Always provide a SECRET_KEY for production JWT_SECRET = os.environ.get( "SECRET_KEY", "M87;Z$,o5?MSC(/@#-LbzgE3PH-5ki.ZvS}N.s09v>I#v8I'00THrA-:ykh3HX?") diff --git a/tests/test_auth.py b/tests/test_auth.py index b0637892..90a8e0cd 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,5 +1,5 @@ import pytest -from autosubmit_api.auth import AuthorizationLevels, with_auth_token +from autosubmit_api.auth import ProtectionLevels, with_auth_token from autosubmit_api import auth from tests.custom_utils import custom_return_value @@ -11,8 +11,8 @@ def dummy_response(*args, **kwargs): class TestCommonAuth: def test_levels_enum(self): - assert AuthorizationLevels.ALL > AuthorizationLevels.WRITEONLY - assert AuthorizationLevels.WRITEONLY > AuthorizationLevels.NONE + assert ProtectionLevels.ALL > ProtectionLevels.WRITEONLY + assert ProtectionLevels.WRITEONLY > ProtectionLevels.NONE def test_decorator(self, monkeypatch: pytest.MonkeyPatch): """ @@ -22,49 +22,49 @@ class TestCommonAuth: """ # Test on AuthorizationLevels.ALL - monkeypatch.setattr(auth, "_parse_authorization_level_env", - custom_return_value(AuthorizationLevels.ALL)) + monkeypatch.setattr(auth, "_parse_protection_level_env", + custom_return_value(ProtectionLevels.ALL)) _, code = with_auth_token( - level=AuthorizationLevels.ALL)(dummy_response)() + threshold=ProtectionLevels.ALL)(dummy_response)() assert code == 401 _, code = with_auth_token( - level=AuthorizationLevels.WRITEONLY)(dummy_response)() + threshold=ProtectionLevels.WRITEONLY)(dummy_response)() assert code == 401 _, code = with_auth_token( - level=AuthorizationLevels.NONE)(dummy_response)() + threshold=ProtectionLevels.NONE)(dummy_response)() assert code == 401 # Test on AuthorizationLevels.WRITEONLY - monkeypatch.setattr(auth, "_parse_authorization_level_env", - custom_return_value(AuthorizationLevels.WRITEONLY)) + monkeypatch.setattr(auth, "_parse_protection_level_env", + custom_return_value(ProtectionLevels.WRITEONLY)) _, code = with_auth_token( - level=AuthorizationLevels.ALL)(dummy_response)() + threshold=ProtectionLevels.ALL)(dummy_response)() assert code == 200 _, code = with_auth_token( - level=AuthorizationLevels.WRITEONLY)(dummy_response)() + threshold=ProtectionLevels.WRITEONLY)(dummy_response)() assert code == 401 _, code = with_auth_token( - level=AuthorizationLevels.NONE)(dummy_response)() + threshold=ProtectionLevels.NONE)(dummy_response)() assert code == 401 # Test on AuthorizationLevels.NONE - monkeypatch.setattr(auth, "_parse_authorization_level_env", - custom_return_value(AuthorizationLevels.NONE)) + monkeypatch.setattr(auth, "_parse_protection_level_env", + custom_return_value(ProtectionLevels.NONE)) _, code = with_auth_token( - level=AuthorizationLevels.ALL)(dummy_response)() + threshold=ProtectionLevels.ALL)(dummy_response)() assert code == 200 _, code = with_auth_token( - level=AuthorizationLevels.WRITEONLY)(dummy_response)() + threshold=ProtectionLevels.WRITEONLY)(dummy_response)() assert code == 200 _, code = with_auth_token( - level=AuthorizationLevels.NONE)(dummy_response)() + threshold=ProtectionLevels.NONE)(dummy_response)() assert code == 401 -- GitLab From 1df8d41f0c0f18f52783b1c072363d81be9281b7 Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Wed, 8 Nov 2023 15:27:14 +0100 Subject: [PATCH 12/26] fix pickle read in expcount #29 --- autosubmit_api/experiment/common_requests.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/autosubmit_api/experiment/common_requests.py b/autosubmit_api/experiment/common_requests.py index 506bc1b6..f1f89750 100644 --- a/autosubmit_api/experiment/common_requests.py +++ b/autosubmit_api/experiment/common_requests.py @@ -1107,20 +1107,19 @@ def get_experiment_counters(expid): experiment_counters = {name: 0 for name in common_utils.Status.STRING_TO_CODE} try: if os.path.exists(path_pkl): - fd = open(path_pkl, 'r') + fd = open(path_pkl, 'rb') for item in pickle.load(fd, encoding="latin1"): status_code = int(item[2]) total += 1 experiment_counters[common_utils.Status.VALUE_TO_KEY.get(status_code, "UNKNOWN")] = experiment_counters.get( common_utils.Status.VALUE_TO_KEY.get(status_code, "UNKNOWN"), 0) + 1 - else: raise Exception("PKL file not found.") - except Exception as exp: + except Exception as exc: error = True - error_message = str(exp) - # print(traceback.format_exc()) - # print(exp) + error_message = str(exc) + logger.error(traceback.format_exc()) + logger.error(exc) return {"error": error, "error_message": error_message, "expid": expid, "total": total, "counters": experiment_counters} -- GitLab From 79d4a1ab563f0f56b8056ea9b415581d02257294 Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Fri, 10 Nov 2023 10:30:51 +0100 Subject: [PATCH 13/26] update endpoint protection --- CHANGELOG.md | 12 ++++++++++-- autosubmit_api/app.py | 16 +++++++++++----- autosubmit_api/auth/__init__.py | 16 +++++++++------- autosubmit_api/common/utils.py | 2 -- autosubmit_api/components/jobs/job_factory.py | 11 +++++------ .../performance/performance_metrics.py | 18 ++++++++---------- 6 files changed, 43 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7169d2fc..07ebd78f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,16 @@ # CHANGELOG -## v4.0.0b2 - (2023-mm-dd) +### Pre-release v4.0.0b2 - Release date: TBD +* Fix bug where API allowed clients aren't read from the autosubmitrc. * Fix bug where `Parallelization` doesn't report the platform `PROCESSORS` in the `/performance/` endpoint. * Fixed bug that doesn't shows the full config of AS4 in the `/cconfig/` endpoint. * Added differences calculation in the `/cconfig/` endpoint. -* Improved logging and code structure. \ No newline at end of file +* Added job status in the `/quick/` endpoint. +* Improved security by using protection levels. +* Improved logging and code structure. + +### Pre-release v4.0.0b1 - Release date: 2023-11-02 + +* Introduced `autosubmit_api` CLI +* Majorly solved compatibility with autosubmit >= 4.0.0 \ No newline at end of file diff --git a/autosubmit_api/app.py b/autosubmit_api/app.py index 43e1a616..05fd6b80 100644 --- a/autosubmit_api/app.py +++ b/autosubmit_api/app.py @@ -41,7 +41,7 @@ import jwt import sys from flask_apscheduler import APScheduler from autosubmit_api.workers import populate_details_db, populate_queue_run_times, populate_running_experiments, populate_graph, verify_complete -from autosubmit_api.config import JWT_SECRET, JWT_ALGORITHM, JWT_EXP_DELTA_SECONDS, RUN_BACKGROUND_TASKS_ON_START, CAS_LOGIN_URL, CAS_VERIFY_URL +from autosubmit_api.config import JWT_SECRET, JWT_ALGORITHM, JWT_EXP_DELTA_SECONDS, PROTECTION_LEVEL, RUN_BACKGROUND_TASKS_ON_START, CAS_LOGIN_URL, CAS_VERIFY_URL def create_app(): @@ -77,6 +77,12 @@ def create_app(): # Initial read config APIBasicConfig.read() app.logger.debug("API Basic config: " + str(APIBasicConfig().props())) + app.logger.debug("Env Config: "+ str({ + "PROTECTION_LEVEL": PROTECTION_LEVEL, + "CAS_LOGIN_URL": CAS_LOGIN_URL, + "CAS_VERIFY_URL": CAS_VERIFY_URL, + "RUN_BACKGROUND_TASKS_ON_START": RUN_BACKGROUND_TASKS_ON_START + })) # Prepare DB ext_db = ExtendedDB(APIBasicConfig.DB_DIR, @@ -153,7 +159,7 @@ def create_app(): app.logger.info("Redirected to: " + str(route_to_request_ticket)) return redirect(route_to_request_ticket) # can be used to target the test environment - environment = environment if environment is not None else "autosubmitapp" + # environment = environment if environment is not None else "autosubmitapp" cas_verify_ticket_route = CAS_VERIFY_URL + \ '?service=' + target_service + '&ticket=' + ticket response = requests.get(cas_verify_ticket_route) @@ -190,7 +196,7 @@ def create_app(): @app.route('/tokentest', methods=['GET', 'POST']) @cross_origin(expose_headers="Authorization") @with_log_run_times(app.logger, "TTEST") - @with_auth_token(threshold=ProtectionLevels.WRITEONLY, response_on_fail=False) + @with_auth_token(threshold=ProtectionLevels.NONE, response_on_fail=False) def test_token(user_id: Optional[str] = None): """ Tests if a token is still valid @@ -252,8 +258,8 @@ def create_app(): Returns the list of all experiments that are currently running. """ if 'username' in session: - print(("USER {}".format(session['username']))) - app.logger.info("Active proceses: " + str(D)) + app.logger.debug(("USER {}".format(session['username']))) + app.logger.debug("Active proceses: " + str(D)) # app.logger.info("Received Currently Running query ") result = get_current_running_exp() return result diff --git a/autosubmit_api/auth/__init__.py b/autosubmit_api/auth/__init__.py index 01510b37..dc9af150 100644 --- a/autosubmit_api/auth/__init__.py +++ b/autosubmit_api/auth/__init__.py @@ -31,8 +31,9 @@ def with_auth_token(threshold=ProtectionLevels.ALL, response_on_fail=True, raise It adds the `user_id` variable inside the arguments of the wrapped function. - :param response_on_fail: if `True` will return a Flask response - :param raise_on_fail: if `True` will raise an exception + :param threshold: The minimum PROTECTION_LEVEL that needs to be set to trigger a *_on_fail + :param response_on_fail: if `True` will return a Flask response on fail + :param raise_on_fail: if `True` will raise an exception on fail :raises AppAuthError: if raise_on_fail=True and decoding fails """ def decorator(func): @@ -45,12 +46,13 @@ def with_auth_token(threshold=ProtectionLevels.ALL, response_on_fail=True, raise except Exception as exc: error_msg = "Unauthorized" if isinstance(exc, jwt.ExpiredSignatureError): - error_msg = "Expired token" + error_msg = "Expired token" auth_level = _parse_protection_level_env(PROTECTION_LEVEL) - if threshold <= auth_level and raise_on_fail: - raise AppAuthError(error_msg) - if threshold <= auth_level and response_on_fail: - return {"error": True, "message": error_msg }, 401 + if threshold <= auth_level: # If True, will trigger *_on_fail + if raise_on_fail: + raise AppAuthError(error_msg) + if response_on_fail: + return {"error": True, "message": error_msg}, 401 jwt_token = {"user_id": None} user_id = jwt_token.get("user_id", None) diff --git a/autosubmit_api/common/utils.py b/autosubmit_api/common/utils.py index fe491d21..22ebd6ba 100644 --- a/autosubmit_api/common/utils.py +++ b/autosubmit_api/common/utils.py @@ -1,6 +1,4 @@ #!/usr/bin/env python -import os -import pickle import subprocess import time import datetime diff --git a/autosubmit_api/components/jobs/job_factory.py b/autosubmit_api/components/jobs/job_factory.py index db242e18..d4c4d593 100644 --- a/autosubmit_api/components/jobs/job_factory.py +++ b/autosubmit_api/components/jobs/job_factory.py @@ -1,11 +1,10 @@ #!/usr/bin/env python import collections -import time -from ...common import utils as util -from . import utils as JUtils -from ...common.utils import Status -from ...monitor.monitor import Monitor -from ...history.data_classes.job_data import JobData +from autosubmit_api.common import utils as util +from autosubmit_api.components.jobs import utils as JUtils +from autosubmit_api.common.utils import Status +from autosubmit_api.monitor.monitor import Monitor +from autosubmit_api.history.data_classes.job_data import JobData from typing import Tuple, List, Dict, Set # from autosubmitAPIwu.database.db_jobdata import JobData from abc import ABCMeta, abstractmethod diff --git a/autosubmit_api/performance/performance_metrics.py b/autosubmit_api/performance/performance_metrics.py index 0610aea0..0c3bc7eb 100644 --- a/autosubmit_api/performance/performance_metrics.py +++ b/autosubmit_api/performance/performance_metrics.py @@ -1,10 +1,9 @@ #!/usr/bin/env python import traceback -from ..common import utils as utils -from ..components.experiment.pkl_organizer import PklOrganizer -from ..components.experiment.configuration_facade import AutosubmitConfigurationFacade -from ..components.jobs.joblist_helper import JobListHelper -from ..components.jobs.job_factory import Job +from autosubmit_api.logger import logger +from autosubmit_api.common import utils as utils +from autosubmit_api.components.jobs.joblist_helper import JobListHelper +from autosubmit_api.components.jobs.job_factory import Job, SimJob from typing import List, Dict class PerformanceMetrics(object): @@ -35,13 +34,13 @@ class PerformanceMetrics(object): except Exception as exp: self.error = True self.error_message = "Error while preparing data sources: {0}".format(str(exp)) - print((traceback.format_exc())) - print((str(exp))) + logger.error((traceback.format_exc())) + logger.error((str(exp))) if self.error == False: self.configuration_facade.update_sim_jobs(self.pkl_organizer.sim_jobs) self._update_jobs_with_time_data() self._calculate_post_jobs_total_time_average() - self.sim_jobs_valid = utils.get_jobs_with_no_outliers(self.pkl_organizer.get_completed_section_jobs(utils.JobSection.SIM)) # type: List[Job] + self.sim_jobs_valid: List[SimJob] = utils.get_jobs_with_no_outliers(self.pkl_organizer.get_completed_section_jobs(utils.JobSection.SIM)) self._identify_outlied_jobs() self._update_valid_sim_jobs_with_post_data() self._add_valid_sim_jobs_to_considered() @@ -152,8 +151,7 @@ class PerformanceMetrics(object): return divisor - def _add_to_considered(self, simjob): - # type: (Job) -> None + def _add_to_considered(self, simjob: SimJob): self._considered.append({ "name": simjob.name, "queue": simjob.queue_time, -- GitLab From f6f6b1cf018f19abcfa70f49965b20b2471df784 Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Mon, 13 Nov 2023 17:18:08 +0100 Subject: [PATCH 14/26] added PE calculation in performance #42 --- autosubmit_api/app.py | 5 +- .../experiment/configuration_facade.py | 46 ++++- autosubmit_api/components/jobs/utils.py | 8 +- autosubmit_api/config/IConfigStrategy.py | 3 +- autosubmit_api/config/ymlConfigStrategy.py | 171 ++++++++---------- .../performance/performance_metrics.py | 32 ++-- 6 files changed, 150 insertions(+), 115 deletions(-) diff --git a/autosubmit_api/app.py b/autosubmit_api/app.py index 05fd6b80..03b693f1 100644 --- a/autosubmit_api/app.py +++ b/autosubmit_api/app.py @@ -356,16 +356,17 @@ def create_app(): try: result = PerformanceMetrics(expid, JobListHelperDirector( JobListHelperBuilder(expid)).build_job_list_helper()).to_json() - except Exception as exp: + except Exception as exc: result = {"SYPD": None, "ASYPD": None, "RSYPD": None, "CHSY": None, "JPSY": None, "Parallelization": None, + "PE": None, "considered": [], "error": True, - "error_message": str(exp), + "error_message": str(exc), "warnings_job_data": [], } return result diff --git a/autosubmit_api/components/experiment/configuration_facade.py b/autosubmit_api/components/experiment/configuration_facade.py index 4a725c25..8eba4677 100644 --- a/autosubmit_api/components/experiment/configuration_facade.py +++ b/autosubmit_api/components/experiment/configuration_facade.py @@ -1,11 +1,14 @@ #!/usr/bin/env python +import math import os -from ...config.basicConfig import APIBasicConfig -from ..jobs.job_factory import SimJob, Job -from ...config.config_common import AutosubmitConfigResolver -from bscearth.utils.config_parser import ConfigParserFactory +from autosubmit_api.logger import logger +from autosubmit_api.components.jobs.utils import convert_int_default +from autosubmit_api.config.ymlConfigStrategy import ymlConfigStrategy +from autosubmit_api.config.basicConfig import APIBasicConfig +from autosubmit_api.components.jobs.job_factory import SimJob +from autosubmit_api.config.config_common import AutosubmitConfigResolver from abc import ABCMeta, abstractmethod -from ...common.utils import JobSection, parse_number_processors, timestamp_to_datetime_format, datechunk_to_year +from autosubmit_api.common.utils import JobSection, parse_number_processors, timestamp_to_datetime_format, datechunk_to_year from typing import List class ProjectType: @@ -123,6 +126,19 @@ class AutosubmitConfigurationFacade(ConfigurationFacade): self.chunk_size = self.autosubmit_conf.get_chunk_size() self.current_years_per_sim = datechunk_to_year(self.chunk_unit, self.chunk_size) self.sim_processors = self._get_processors_number(self.autosubmit_conf.get_processors(JobSection.SIM)) + + # Process for yml + if isinstance(self.autosubmit_conf._configWrapper, ymlConfigStrategy): + self.sim_tasks = convert_int_default(self.autosubmit_conf._configWrapper.get_tasks(JobSection.SIM)) + self.sim_nodes = convert_int_default(self.autosubmit_conf._configWrapper.get_nodes(JobSection.SIM)) + self.sim_processors_per_node = convert_int_default(self.autosubmit_conf._configWrapper.get_processors_per_node(JobSection.SIM)) + else: + self.sim_tasks = None + self.sim_nodes = None + self.sim_processors_per_node = None + + self.sim_processing_elements = self._calculate_processing_elements() + self.experiment_stat_data = os.stat(self.experiment_path) def get_pkl_last_modified_timestamp(self): @@ -242,7 +258,7 @@ class AutosubmitConfigurationFacade(ConfigurationFacade): # type: (List[SimJob]) -> None """ Update the jobs with the latest configuration values: Processors, years per sim """ for job in sim_jobs: - job.set_ncpus(self.sim_processors) + job.set_ncpus(self.sim_processing_elements) job.set_years_per_sim(self.current_years_per_sim) def _get_processors_number(self, conf_job_processors): @@ -264,3 +280,21 @@ class AutosubmitConfigurationFacade(ConfigurationFacade): def _add_warning(self, message): # type: (str) -> None self.warnings.append(message) + + def _estimate_requested_nodes(self) -> int: + if self.sim_nodes: + return self.sim_nodes + elif self.sim_tasks: + return math.ceil(self.sim_processors / self.sim_tasks) + elif self.sim_processors_per_node and self.sim_processors > self.sim_processors_per_node: + return math.ceil(self.sim_processors / self.sim_processors_per_node) + else: + return 1 + + def _calculate_processing_elements(self) -> int: + if self.sim_processors_per_node: + estimated_nodes = self._estimate_requested_nodes() + return estimated_nodes * self.sim_processors_per_node + elif self.sim_tasks or self.sim_nodes: + logger.warning('Missing PROCESSORS_PER_NODE. Should be set if TASKS or NODES are defined. The SIM PROCESSORS will used instead.') + return self.sim_processors \ No newline at end of file diff --git a/autosubmit_api/components/jobs/utils.py b/autosubmit_api/components/jobs/utils.py index b73eaf7b..5faaba01 100644 --- a/autosubmit_api/components/jobs/utils.py +++ b/autosubmit_api/components/jobs/utils.py @@ -132,4 +132,10 @@ def get_folder_package_title(package_name, jobs_count, counters): get_folder_queuing_tag(counters[Status.QUEUING]), get_folder_held_tag(counters[Status.HELD]), get_folder_checkmark(counters[Status.COMPLETED], jobs_count) - ) \ No newline at end of file + ) + +def convert_int_default(value, default_value=None): + try: + return int(value) + except: + return default_value \ No newline at end of file diff --git a/autosubmit_api/config/IConfigStrategy.py b/autosubmit_api/config/IConfigStrategy.py index 56e3f483..cf552bdf 100644 --- a/autosubmit_api/config/IConfigStrategy.py +++ b/autosubmit_api/config/IConfigStrategy.py @@ -797,12 +797,11 @@ class IConfigStrategy(ABC): return True else: return False - @classmethod + def is_valid_communications_library(self): library = self.get_communications_library() return library in ['paramiko', 'saga'] - @classmethod def is_valid_storage_type(self): storage_type = self.get_storage_type() return storage_type in ['pkl', 'db'] diff --git a/autosubmit_api/config/ymlConfigStrategy.py b/autosubmit_api/config/ymlConfigStrategy.py index 9f16d138..bb68afdc 100644 --- a/autosubmit_api/config/ymlConfigStrategy.py +++ b/autosubmit_api/config/ymlConfigStrategy.py @@ -17,28 +17,12 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . -try: - # noinspection PyCompatibility - from configparser import SafeConfigParser - from autosubmitconfigparser.config.configcommon import AutosubmitConfig as Autosubmit4Config -except ImportError: - # noinspection PyCompatibility - from configparser import SafeConfigParser - -import os -import re -import subprocess -import json -import logging - -from pyparsing import nestedExpr -from bscearth.utils.config_parser import ConfigParserFactory, ConfigParser -from bscearth.utils.date import parse_date -from bscearth.utils.log import Log -from ..config.basicConfig import APIBasicConfig -from ..config.IConfigStrategy import IConfigStrategy - -logger = logging.getLogger('gunicorn.error') +from typing import Any +from autosubmitconfigparser.config.configcommon import AutosubmitConfig as Autosubmit4Config +from autosubmit_api.logger import logger +from autosubmit_api.config.basicConfig import APIBasicConfig +from autosubmit_api.config.IConfigStrategy import IConfigStrategy + class ymlConfigStrategy(IConfigStrategy): """ @@ -53,7 +37,7 @@ class ymlConfigStrategy(IConfigStrategy): self._conf_parser.reload(True) def jobs_parser(self): - logger.info("Not yet implemented") + logger.error("Not yet implemented") pass #TODO: at the end of the implementation, check which methods can be moved to the top class for avoid code duplication @@ -65,7 +49,7 @@ class ymlConfigStrategy(IConfigStrategy): return self._exp_parser_file def platforms_parser(self): - logger.info("OBSOLOTED - Not yet implemented") + logger.error("OBSOLOTED - Not yet implemented") pass @property @@ -92,35 +76,32 @@ class ymlConfigStrategy(IConfigStrategy): """ return self._jobs_parser_file - def get_full_config_as_dict(self): + def get_full_config_as_dict(self) -> dict: """ Returns full configuration as json object """ - _conf = _exp = _platforms = _jobs = _proj = None - result = {} - - def get_data( parser): - """ - dictionary comprehension to get data from parser - """ - logger.info(parser) - #res = {sec: {option: parser[sec][option] for option in parser[sec].keys()} for sec in [ - # section for section in parser.keys()]} - #return res - return parser - - # res = {sec: {option: parser.get(sec, option) for option in parser.options(sec)} for sec in [ - # section for section in parser.sections()]} - - - # print(self._conf_parser) - #result["conf"] = get_data( self._conf_parser.experiment_data["CONF"]) if self._conf_parser else None - #result["exp"] = get_data( self._conf_parser.experiment_data["CONF"]) if self._exp_parser else None - # result["platforms"] = self._conf_parser.platforms_data if self._conf_parser.platforms_data else None - #result["jobs"] = get_data( self._conf_parser.experiment_data["JOBS"]) if self._conf_parser.experiment_data["JOBS"] else None - #result["proj"] = get_data( self._conf_parser.experiment_data["CONF"] ) if self._proj_parser else None return self._conf_parser.experiment_data + + def _get_platform_config(self, platform: str) -> dict: + return self._conf_parser.experiment_data.get("PLATFORMS", {}).get(platform, {}) + def _get_from_job_or_platform(self, section: str, config_key: str, default_value = None) -> Any: + """ + Helper function to get a value from a JOB section of its related PLATFORM + """ + # Check in job + logger.debug("Checking " + config_key + " in JOBS") + value = self._conf_parser.jobs_data.get(section, {}).get(config_key) + if value: + return value + + # Check in job platform + logger.debug("Checking " + config_key + " in PLATFORMS " + self.get_job_platform(section)) + value = self._get_platform_config(self.get_job_platform(section)).get(config_key) + if value: + return value + else: + return default_value def get_full_config_as_json(self): return self._conf_parser.get_full_config_as_json() @@ -128,63 +109,65 @@ class ymlConfigStrategy(IConfigStrategy): def get_project_dir(self): return self._conf_parser.get_project_dir() - def get_queue(self, section): + def get_queue(self, section: str) -> str: return self._conf_parser.jobs_data[section].get('QUEUE', "") def get_job_platform(self, section: str) -> str: - return self._conf_parser.jobs_data.get(section, {}).get("PLATFORM", "") + # return the JOBS.
.PLATFORM or DEFAULT.HPCARCH + return self._conf_parser.jobs_data.get(section, {}).get("PLATFORM", self.get_platform()) - def get_platform_queue(self, platform): - logger.info("get_platform_queue") - return self._conf_parser.experiment_data.get("PLATFORMS", {}).get(platform, {}).get("QUEUE") + def get_platform_queue(self, platform: str) -> str: + logger.debug("get_platform_queue") + return self._get_platform_config(platform).get("QUEUE") - def get_platform_serial_queue(self, platform): - logger.info("get_platform_serial_queue") - return self._conf_parser.experiment_data.get("PLATFORMS", {}).get(platform, {}).get("SERIAL_QUEUE") + def get_platform_serial_queue(self, platform: str) -> str: + logger.debug("get_platform_serial_queue") + return self._get_platform_config(platform).get("SERIAL_QUEUE") - def get_platform_project(self, platform): - logger.info("get_platform_project") - return self._conf_parser.experiment_data.get("PLATFORMS", {}).get(platform, {}).get("PROJECT") + def get_platform_project(self, platform: str) -> str: + logger.debug("get_platform_project") + return self._get_platform_config(platform).get("PROJECT") - def get_platform_wallclock(self, platform): - logger.info("get_platform_wallclock") - return self._conf_parser.experiment_data.get("PLATFORMS", {}).get(platform, {}).get('MAX_WALLCLOCK', "") + def get_platform_wallclock(self, platform: str) -> str: + logger.debug("get_platform_wallclock") + return self._get_platform_config(platform).get('MAX_WALLCLOCK', "") - def get_wallclock(self, section): + def get_wallclock(self, section: str) -> str: return self._conf_parser.jobs_data[section].get('WALLCLOCK', '') - - def get_synchronize(self, section): - return self._conf_parser.get_synchronize(section) + def get_synchronize(self, section: str) -> str: + # return self._conf_parser.get_synchronize(section) + return self._get_from_job_or_platform(section, "SYNCHRONIZE", "") def get_processors(self, section: str) -> str: - # Check processors in job - processors = self._conf_parser.jobs_data.get(section, {}).get("PROCESSORS") - if processors: - return processors - - # Check processors in job platform - processors = self._conf_parser.experiment_data.get("PLATFORMS", {}).get(self.get_job_platform(section), {}).get("PROCESSORS") - if processors: - return processors - else: - return "1" + # return self._conf_parser.get_processors() + return self._get_from_job_or_platform(section, "PROCESSORS", "1") + def get_threads(self, section: str) -> str: + # return self._conf_parser.get_threads(section) + return self._get_from_job_or_platform(section, "THREADS", "1") - def get_threads(self, section): - return self._conf_parser.get_threads(section) + def get_tasks(self, section: str) -> str: + # return self._conf_parser.get_tasks(section) + return self._get_from_job_or_platform(section, "TASKS", "") + + def get_nodes(self, section: str) -> str: + return self._get_from_job_or_platform(section, "NODES", "") - def get_tasks(self, section): - return self._conf_parser.get_tasks(section) + def get_processors_per_node(self, section: str) -> str: + return self._get_from_job_or_platform(section, "PROCESSORS_PER_NODE", "") - def get_scratch_free_space(self, section): - return self._conf_parser.get_scratch_free_space(section) + def get_scratch_free_space(self, section: str) -> str: + # return self._conf_parser.get_scratch_free_space(section) + return self._get_from_job_or_platform(section, "SCRATCH_FREE_SPACE", "") - def get_memory(self, section): - return self._conf_parser.get_memory(section) + def get_memory(self, section: str) -> str: + # return self._conf_parser.get_memory(section) + return self._get_from_job_or_platform(section, "MEMORY", "") - def get_memory_per_task(self, section): - return self._conf_parser.get_memory_per_task(section) + def get_memory_per_task(self, section: str) -> str: + # return self._conf_parser.get_memory_per_task(section) + return self._get_from_job_or_platform(section, "MEMORY_PER_TASK", "") def get_migrate_user_to(self, section): """ @@ -335,8 +318,12 @@ class ymlConfigStrategy(IConfigStrategy): def get_chunk_list(self): return self._conf_parser.get_chunk_list() - def get_platform(self): - return self._conf_parser.get_platform() + def get_platform(self) -> str: + try: + # return DEFAULT.HPCARCH + return self._conf_parser.get_platform() + except: + return "" def set_platform(self, hpc): self._conf_parser.set_platform(hpc) @@ -411,14 +398,12 @@ class ymlConfigStrategy(IConfigStrategy): return self._conf_parser.get_storage_type() @staticmethod - def is_valid_mail_address(mail_address): - return self._conf_parser.is_valid_mail_address(mail_address) + def is_valid_mail_address(mail_address: str) -> bool: + return Autosubmit4Config.is_valid_mail_address(mail_address) - @classmethod def is_valid_communications_library(self): return self._conf_parser.is_valid_communications_library() - @classmethod def is_valid_storage_type(self): return self._conf_parser.is_valid_storage_type() diff --git a/autosubmit_api/performance/performance_metrics.py b/autosubmit_api/performance/performance_metrics.py index 0c3bc7eb..be2abf1e 100644 --- a/autosubmit_api/performance/performance_metrics.py +++ b/autosubmit_api/performance/performance_metrics.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +import math import traceback from autosubmit_api.logger import logger from autosubmit_api.common import utils as utils @@ -21,8 +22,9 @@ class PerformanceMetrics(object): self.CHSY = 0 # type: float self.JPSY = 0 # type: float self.RSYPD = 0 # type: float + self.processing_elements = 1 self._considered = [] # type : List - self._sim_processors = 0 # type : int + self._sim_processors = 1 # type : int self.warnings = [] # type : List self.post_jobs_total_time_average = 0 # type : int try: @@ -31,13 +33,14 @@ class PerformanceMetrics(object): self.pkl_organizer = self.joblist_helper.pkl_organizer # type : PklOrganizer self.pkl_organizer.prepare_jobs_for_performance_metrics() self._sim_processors = self.configuration_facade.sim_processors + self.processing_elements = self.configuration_facade.sim_processing_elements except Exception as exp: self.error = True self.error_message = "Error while preparing data sources: {0}".format(str(exp)) logger.error((traceback.format_exc())) logger.error((str(exp))) if self.error == False: - self.configuration_facade.update_sim_jobs(self.pkl_organizer.sim_jobs) + self.configuration_facade.update_sim_jobs(self.pkl_organizer.sim_jobs) # This will assign self.configuration_facade.sim_processors to all the SIM jobs self._update_jobs_with_time_data() self._calculate_post_jobs_total_time_average() self.sim_jobs_valid: List[SimJob] = utils.get_jobs_with_no_outliers(self.pkl_organizer.get_completed_section_jobs(utils.JobSection.SIM)) @@ -58,11 +61,11 @@ class PerformanceMetrics(object): self.joblist_helper.update_with_yps_per_run(self.pkl_organizer.sim_jobs) def _calculate_global_metrics(self): - self._calculate_SYPD() - self._calculate_ASYPD() - self._calculate_RSYPD() - self._calculate_JPSY() - self._calculate_CHSY() + self.SYPD = self._calculate_SYPD() + self.ASYPD = self._calculate_ASYPD() + self.RSYPD = self._calculate_RSYPD() + self.JPSY = self._calculate_JPSY() + self.CHSY = self._calculate_CHSY() def _identify_outlied_jobs(self): """ Generates warnings """ @@ -101,34 +104,40 @@ class PerformanceMetrics(object): def _calculate_total_sim_queue_time(self): self.total_sim_queue_time = sum(job.queue_time for job in self.sim_jobs_valid) + def _calculate_SYPD(self): if self.total_sim_run_time > 0: SYPD = ((self.configuration_facade.current_years_per_sim * len(self._considered) * utils.SECONDS_IN_A_DAY) / (self.total_sim_run_time)) - self.SYPD = round(SYPD, 4) + return round(SYPD, 4) + return 0 def _calculate_ASYPD(self): if len(self.sim_jobs_valid) > 0: ASYPD = (self.configuration_facade.current_years_per_sim * len(self.sim_jobs_valid) * utils.SECONDS_IN_A_DAY) / (self.total_sim_run_time + self.total_sim_queue_time + self.post_jobs_total_time_average) - self.ASYPD = round(ASYPD, 4) + return round(ASYPD, 4) + return 0 def _calculate_RSYPD(self): divisor = self._get_RSYPD_divisor() if len(self.sim_jobs_valid) > 0 and divisor > 0: RSYPD = (self.configuration_facade.current_years_per_sim * len(self.sim_jobs_valid) * utils.SECONDS_IN_A_DAY) / divisor - self.RSYPD = round(RSYPD, 4) + return round(RSYPD, 4) + return 0 def _calculate_JPSY(self): """ Joules per Simulated Year """ sims_with_energy_count = self._get_sims_with_energy_count() if len(self.sim_jobs_valid) > 0 and sims_with_energy_count > 0: JPSY = sum(job.JPSY for job in self.sim_jobs_valid)/sims_with_energy_count - self.JPSY = round(JPSY, 4) + return round(JPSY, 4) + return 0 def _calculate_CHSY(self): if len(self.sim_jobs_valid) > 0: CHSY = sum(job.CHSY for job in self.sim_jobs_valid)/len(self.sim_jobs_valid) self.CHSY = round(CHSY, 4) + return 0 def _get_RSYPD_support_list(self): # type: () -> List[Job] @@ -173,6 +182,7 @@ class PerformanceMetrics(object): "CHSY": self.CHSY, "JPSY": self.JPSY, "Parallelization": self._sim_processors, + "PE": self.processing_elements, "considered": self._considered, "error": self.error, "error_message": self.error_message, -- GitLab From c6c503d4fe817ce36d18439c9b8dead7b8201a4e Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Tue, 14 Nov 2023 09:39:49 +0100 Subject: [PATCH 15/26] quickfix CHSY assignation --- CHANGELOG.md | 1 + autosubmit_api/performance/performance_metrics.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07ebd78f..1e7ab270 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ * Added job status in the `/quick/` endpoint. * Improved security by using protection levels. * Improved logging and code structure. +* Added Processing Elements (PE) estimation in the `/performance/` endpoint which improves the CHSY metric accuracy. ### Pre-release v4.0.0b1 - Release date: 2023-11-02 diff --git a/autosubmit_api/performance/performance_metrics.py b/autosubmit_api/performance/performance_metrics.py index be2abf1e..34f9710d 100644 --- a/autosubmit_api/performance/performance_metrics.py +++ b/autosubmit_api/performance/performance_metrics.py @@ -136,7 +136,7 @@ class PerformanceMetrics(object): def _calculate_CHSY(self): if len(self.sim_jobs_valid) > 0: CHSY = sum(job.CHSY for job in self.sim_jobs_valid)/len(self.sim_jobs_valid) - self.CHSY = round(CHSY, 4) + return round(CHSY, 4) return 0 def _get_RSYPD_support_list(self): -- GitLab From b52b1657a4c0f4e01e7372e68c72fff62a909a54 Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Tue, 14 Nov 2023 15:30:06 +0100 Subject: [PATCH 16/26] remove numpy setup dependency --- CHANGELOG.md | 1 + .../autosubmit_legacy/job/job_list.py | 43 ++++++++----------- autosubmit_api/common/utils.py | 8 ++-- autosubmit_api/database/db_jobdata.py | 22 ++++------ setup.py | 1 - 5 files changed, 31 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e7ab270..6cb2c2d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ * Improved security by using protection levels. * Improved logging and code structure. * Added Processing Elements (PE) estimation in the `/performance/` endpoint which improves the CHSY metric accuracy. +* Fixed error while populating the `experiment_times` table which affected the `/running/` endpoint. ### Pre-release v4.0.0b1 - Release date: 2023-11-02 diff --git a/autosubmit_api/autosubmit_legacy/job/job_list.py b/autosubmit_api/autosubmit_legacy/job/job_list.py index 02e23242..ca746747 100644 --- a/autosubmit_api/autosubmit_legacy/job/job_list.py +++ b/autosubmit_api/autosubmit_legacy/job/job_list.py @@ -29,50 +29,43 @@ from bscearth.utils.config_parser import ConfigParserFactory import os import re import pickle -import hashlib import traceback import datetime import math -import random # Spectral imports -import numpy as np import networkx as nx from scipy import sparse from fnmatch import fnmatch from collections import deque, OrderedDict -from threading import Thread # End Spectral imports from time import localtime, strftime, time, mktime from shutil import move -from random import shuffle from dateutil.relativedelta import * -from .job import Job -from ...config.config_common import AutosubmitConfigResolver +from autosubmit_api.config.config_common import AutosubmitConfigResolver from bscearth.utils.log import Log -from .job_dict import DicJobs -from .job_utils import Dependency -from .job_utils import SubJob -from .job_utils import SubJobManager, job_times_to_text, datechunk_to_year -from ...performance.utils import calculate_ASYPD_perjob, calculate_SYPD_perjob -from ...components.jobs import utils as JUtils -from ...monitor.monitor import Monitor -from .job_common import Status, Type -from bscearth.utils.date import date2str, parse_date, sum_str_hours -from ...experiment import common_db_requests as DbRequests -from .job_packages import JobPackageSimple, JobPackageArray, JobPackageThread -from .job_package_persistence import JobPackagePersistence +from autosubmit_api.autosubmit_legacy.job.job_dict import DicJobs +from autosubmit_api.autosubmit_legacy.job.job_utils import Dependency +from autosubmit_api.autosubmit_legacy.job.job_utils import SubJob +from autosubmit_api.autosubmit_legacy.job.job_utils import SubJobManager, job_times_to_text, datechunk_to_year +from autosubmit_api.performance.utils import calculate_ASYPD_perjob, calculate_SYPD_perjob +from autosubmit_api.components.jobs import utils as JUtils +from autosubmit_api.monitor.monitor import Monitor +from autosubmit_api.autosubmit_legacy.job.job_common import Status +from bscearth.utils.date import date2str, parse_date +from autosubmit_api.experiment import common_db_requests as DbRequests +from autosubmit_api.autosubmit_legacy.job.job_package_persistence import JobPackagePersistence # from autosubmit_legacy.job.tree import Tree -from ...database import db_structure as DbStructure -from ...database.db_jobdata import JobDataStructure, JobRow, ExperimentGraphDrawing -from ...builders.experiment_history_builder import ExperimentHistoryDirector, ExperimentHistoryBuilder -from ...history.data_classes.job_data import JobData +from autosubmit_api.database import db_structure as DbStructure +from autosubmit_api.database.db_jobdata import JobDataStructure, JobRow, ExperimentGraphDrawing +from autosubmit_api.builders.experiment_history_builder import ExperimentHistoryDirector, ExperimentHistoryBuilder +from autosubmit_api.history.data_classes.job_data import JobData from networkx import DiGraph -from .job_utils import transitive_reduction -from ...common.utils import timestamp_to_datetime_format +from autosubmit_api.autosubmit_legacy.job.job_utils import transitive_reduction +from autosubmit_api.common.utils import timestamp_to_datetime_format from typing import List, Dict, Tuple diff --git a/autosubmit_api/common/utils.py b/autosubmit_api/common/utils.py index 22ebd6ba..7056076f 100644 --- a/autosubmit_api/common/utils.py +++ b/autosubmit_api/common/utils.py @@ -1,9 +1,9 @@ #!/usr/bin/env python +import statistics import subprocess import time import datetime import math -import numpy as np from collections import namedtuple from bscearth.utils.date import date2str from dateutil.relativedelta import * @@ -79,8 +79,8 @@ def get_jobs_with_no_outliers(jobs): if len(data_run_times) == 0: return jobs - mean = np.mean(data_run_times) - std = np.std(data_run_times) + mean = statistics.mean(data_run_times) + std = statistics.stdev(data_run_times) # print("mean {0} std {1}".format(mean, std)) if std == 0: @@ -89,7 +89,7 @@ def get_jobs_with_no_outliers(jobs): for job in jobs: z_score = (job.run_time - mean) / std # print("{0} {1} {2}".format(job.name, np.abs(z_score), job.run_time)) - if np.abs(z_score) <= THRESHOLD_OUTLIER and job.run_time > 0: + if math.fabs(z_score) <= THRESHOLD_OUTLIER and job.run_time > 0: new_list.append(job) # else: # print(" OUTLIED {0} {1} {2}".format(job.name, np.abs(z_score), job.run_time)) diff --git a/autosubmit_api/database/db_jobdata.py b/autosubmit_api/database/db_jobdata.py index 6ff01ec6..be1c6608 100644 --- a/autosubmit_api/database/db_jobdata.py +++ b/autosubmit_api/database/db_jobdata.py @@ -18,32 +18,26 @@ # along with Autosubmit. If not, see . import os -import sys -import string import time -import pickle import textwrap import traceback import pysqlite3 as sqlite3 -import copy import collections import portalocker -import numpy as np from datetime import datetime, timedelta -from json import dumps, loads +from json import loads from time import mktime # from networkx import DiGraph -from ..config.basicConfig import APIBasicConfig -from ..monitor.monitor import Monitor -from ..autosubmit_legacy.job.job_utils import job_times_to_text, getTitle -from ..performance.utils import calculate_ASYPD_perjob, calculate_SYPD_perjob -from ..components.jobs.job_factory import SimJob -from ..common.utils import get_jobs_with_no_outliers, Status, bcolors, datechunk_to_year +from autosubmit_api.config.basicConfig import APIBasicConfig +from autosubmit_api.monitor.monitor import Monitor +from autosubmit_api.autosubmit_legacy.job.job_utils import job_times_to_text, getTitle +from autosubmit_api.performance.utils import calculate_ASYPD_perjob +from autosubmit_api.components.jobs.job_factory import SimJob +from autosubmit_api.common.utils import get_jobs_with_no_outliers, Status, datechunk_to_year # from autosubmitAPIwu.job.job_list # import autosubmitAPIwu.experiment.common_db_requests as DbRequests -from bscearth.utils.date import date2str, parse_date, previous_day, chunk_end_date, chunk_start_date, Log, subs_dates +from bscearth.utils.date import Log from typing import List -import locale # Version 15 includes out err MaxRSS AveRSS and rowstatus diff --git a/setup.py b/setup.py index c461a915..aa282317 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,6 @@ setup( 'flask_cors~=3.0.10', 'bscearth.utils~=0.5.2', 'pysqlite-binary', - 'numpy~=1.21.6', 'pydotplus~=2.0.2', 'portalocker~=2.6.0', 'networkx~=2.6.3', -- GitLab From 6783b2f656b64d45e92bf7bb190ffc5090d89a3c Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Wed, 15 Nov 2023 14:36:54 +0100 Subject: [PATCH 17/26] remove common job utils legacy dependencies #38 --- CHANGELOG.md | 3 ++- .../components/jobs/joblist_helper.py | 14 +++++------ .../components/jobs/joblist_loader.py | 17 +++++-------- autosubmit_api/components/jobs/utils.py | 4 +-- autosubmit_api/database/autosubmit.py | 5 ++-- .../history/data_classes/experiment_run.py | 13 +++++----- autosubmit_api/monitor/monitor.py | 7 +++--- autosubmit_api/performance/utils.py | 25 ++----------------- 8 files changed, 29 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cb2c2d9..f62de4ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,13 @@ * Fix bug where API allowed clients aren't read from the autosubmitrc. * Fix bug where `Parallelization` doesn't report the platform `PROCESSORS` in the `/performance/` endpoint. -* Fixed bug that doesn't shows the full config of AS4 in the `/cconfig/` endpoint. +* Fixed bug that doesn't show the full config of AS4 in the `/cconfig/` endpoint. * Added differences calculation in the `/cconfig/` endpoint. * Added job status in the `/quick/` endpoint. * Improved security by using protection levels. * Improved logging and code structure. * Added Processing Elements (PE) estimation in the `/performance/` endpoint which improves the CHSY metric accuracy. +* Fixed bug where jobs don't show the correct platform. * Fixed error while populating the `experiment_times` table which affected the `/running/` endpoint. ### Pre-release v4.0.0b1 - Release date: 2023-11-02 diff --git a/autosubmit_api/components/jobs/joblist_helper.py b/autosubmit_api/components/jobs/joblist_helper.py index 7c6820f6..997b272e 100644 --- a/autosubmit_api/components/jobs/joblist_helper.py +++ b/autosubmit_api/components/jobs/joblist_helper.py @@ -1,12 +1,12 @@ #!/usr/bin/env python -from ...autosubmit_legacy.job.job_list import JobList -from ...database.db_jobdata import JobDataStructure, JobRow -from ..experiment.configuration_facade import AutosubmitConfigurationFacade -from ..experiment.pkl_organizer import PklOrganizer -from ...config.basicConfig import APIBasicConfig -from ...autosubmit_legacy.job.job_utils import datechunk_to_year +from autosubmit_api.autosubmit_legacy.job.job_list import JobList +from autosubmit_api.common.utils import datechunk_to_year +from autosubmit_api.database.db_jobdata import JobDataStructure, JobRow +from autosubmit_api.components.experiment.configuration_facade import AutosubmitConfigurationFacade +from autosubmit_api.components.experiment.pkl_organizer import PklOrganizer +from autosubmit_api.config.basicConfig import APIBasicConfig from typing import List, Dict -from .job_factory import Job +from autosubmit_api.components.jobs.job_factory import Job class JobListHelper(object): """ Loads time (queuing runnning) and packages. Applies the fix for queue time of jobs in wrappers. """ diff --git a/autosubmit_api/components/jobs/joblist_loader.py b/autosubmit_api/components/jobs/joblist_loader.py index 6d66c309..1edc563f 100644 --- a/autosubmit_api/components/jobs/joblist_loader.py +++ b/autosubmit_api/components/jobs/joblist_loader.py @@ -2,18 +2,13 @@ import os from fnmatch import fnmatch -from .joblist_helper import JobListHelper -from ..experiment.pkl_organizer import PklOrganizer -from ..experiment.configuration_facade import AutosubmitConfigurationFacade -from .job_factory import StandardJob, Job -from ...database.db_structure import get_structure -from ...autosubmit_legacy.job.job_common import Status -from bscearth.utils.date import date2str, parse_date -from typing import Dict, List, Set, Tuple +from autosubmit_api.components.jobs.joblist_helper import JobListHelper +from autosubmit_api.components.jobs.job_factory import StandardJob, Job +from autosubmit_api.database.db_structure import get_structure +from autosubmit_api.common.utils import Status +from bscearth.utils.date import date2str +from typing import Dict, List, Set # Builder Imports -from ...config.config_common import AutosubmitConfigResolver -from ...config.basicConfig import APIBasicConfig -import json import logging diff --git a/autosubmit_api/components/jobs/utils.py b/autosubmit_api/components/jobs/utils.py index 5faaba01..0a27e948 100644 --- a/autosubmit_api/components/jobs/utils.py +++ b/autosubmit_api/components/jobs/utils.py @@ -1,8 +1,6 @@ #!/usr/bin/env python -from datetime import date -from pickle import NONE -from ...autosubmit_legacy.job.job_common import Status +from autosubmit_api.common.utils import Status from typing import List, Dict wrapped_title_format = " Wrapped {0} " diff --git a/autosubmit_api/database/autosubmit.py b/autosubmit_api/database/autosubmit.py index 86a94261..4b75fa68 100644 --- a/autosubmit_api/database/autosubmit.py +++ b/autosubmit_api/database/autosubmit.py @@ -23,7 +23,7 @@ import traceback from pyparsing import nestedExpr from collections import defaultdict from distutils.util import strtobool -from pkg_resources import require, resource_listdir, resource_exists, resource_string +from pkg_resources import resource_listdir, resource_exists, resource_string import portalocker import datetime import signal @@ -44,7 +44,7 @@ sys.path.insert(0, os.path.abspath('.')) from autosubmit_api.config.basicConfig import APIBasicConfig from autosubmit_api.config.config_common import AutosubmitConfigResolver from bscearth.utils.config_parser import ConfigParserFactory -from autosubmit_api.autosubmit_legacy.job.job_common import Status +from autosubmit_api.common.utils import Status from autosubmit_api.git.autosubmit_git import AutosubmitGit from autosubmit_api.autosubmit_legacy.job.job_list import JobList from autosubmit_api.autosubmit_legacy.job.job_packages import JobPackageThread @@ -66,7 +66,6 @@ from autosubmit_api.autosubmit_legacy.platforms.saga_submitter import SagaSubmit from autosubmit_api.autosubmit_legacy.platforms.paramiko_submitter import ParamikoSubmitter from autosubmit_api.autosubmit_legacy.job.job_exceptions import WrongTemplateException from autosubmit_api.autosubmit_legacy.job.job_packager import JobPackager -from autosubmit_api.autosubmit_legacy.platforms.paramiko_platform import ParamikoTimeout """ Main module for autosubmit. Only contains an interface class to all functionality implemented on autosubmit """ diff --git a/autosubmit_api/history/data_classes/experiment_run.py b/autosubmit_api/history/data_classes/experiment_run.py index dec6b42c..ef3459f6 100644 --- a/autosubmit_api/history/data_classes/experiment_run.py +++ b/autosubmit_api/history/data_classes/experiment_run.py @@ -17,11 +17,10 @@ # along with Autosubmit. If not, see . import json -from ...common import utils as common_utils -from ...performance import utils as PUtils -from ..utils import get_current_datetime_if_none -from .job_data import JobData -from ...components.jobs.job_factory import SimJob +from autosubmit_api.common import utils as common_utils +from autosubmit_api.history.utils import get_current_datetime_if_none +from autosubmit_api.history.data_classes.job_data import JobData +from autosubmit_api.components.jobs.job_factory import SimJob from typing import List, Dict, Tuple @@ -93,7 +92,7 @@ class ExperimentRun(object): outlier_free_list = common_utils.get_jobs_with_no_outliers(performance_jobs) # print("{} -> {}".format(self.run_id, len(outlier_free_list))) if len(outlier_free_list) > 0: - years_per_sim = PUtils.datechunk_to_year(self.chunk_unit, self.chunk_size) + years_per_sim = common_utils.datechunk_to_year(self.chunk_unit, self.chunk_size) # print(self.run_id) # print(years_per_sim) seconds_per_day = 86400 @@ -118,7 +117,7 @@ class ExperimentRun(object): job_sim_list = [job for job in job_sim_list if job.job_name in valid_names] if job_sim_list and len(job_sim_list) > 0 and job_post_list and len(job_post_list) > 0: - years_per_sim = PUtils.datechunk_to_year(self.chunk_unit, self.chunk_size) + years_per_sim = common_utils.datechunk_to_year(self.chunk_unit, self.chunk_size) seconds_per_day = 86400 number_SIM = len(job_sim_list) number_POST = len(job_post_list) diff --git a/autosubmit_api/monitor/monitor.py b/autosubmit_api/monitor/monitor.py index 58e141af..834910fd 100644 --- a/autosubmit_api/monitor/monitor.py +++ b/autosubmit_api/monitor/monitor.py @@ -16,7 +16,6 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . -import datetime import os import time @@ -30,9 +29,9 @@ import copy import subprocess -from ..autosubmit_legacy.job.job_common import Status -from ..config.basicConfig import APIBasicConfig -from ..config.config_common import AutosubmitConfigResolver +from autosubmit_api.common.utils import Status +from autosubmit_api.config.basicConfig import APIBasicConfig +from autosubmit_api.config.config_common import AutosubmitConfigResolver from bscearth.utils.log import Log from bscearth.utils.config_parser import ConfigParserFactory diff --git a/autosubmit_api/performance/utils.py b/autosubmit_api/performance/utils.py index 42e19fe6..0364d52b 100644 --- a/autosubmit_api/performance/utils.py +++ b/autosubmit_api/performance/utils.py @@ -1,5 +1,5 @@ #!/usr/bin/env pytthon -from ..common.utils import Status +from autosubmit_api.common.utils import Status, datechunk_to_year def calculate_SYPD_perjob(chunk_unit, chunk_size, job_chunk, run_time, status): # type: (str, int, int, int, int) -> float @@ -24,25 +24,4 @@ def calculate_ASYPD_perjob(chunk_unit, chunk_size, job_chunk, queue_run_time, av divisor = queue_run_time + average_post if divisor > 0.0: return round((years_per_sim * 86400) / divisor, 2) - return None - -def datechunk_to_year(chunk_unit, chunk_size): - """ - Gets chunk unit and size and returns the value in years - - :return: years - :rtype: float - """ - # type : (int, int) -> float - chunk_size = chunk_size * 1.0 - options = ["year", "month", "day", "hour"] - if (chunk_unit == "year"): - return chunk_size - elif (chunk_unit == "month"): - return chunk_size / 12 - elif (chunk_unit == "day"): - return chunk_size / 365 - elif (chunk_unit == "hour"): - return chunk_size / 8760 - else: - return 0.0 \ No newline at end of file + return None \ No newline at end of file -- GitLab From 23fde607bdd35a4b470b9729bdb99d0c00187433 Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Wed, 15 Nov 2023 15:26:14 +0100 Subject: [PATCH 18/26] remove autosubmit legacy class #38 --- autosubmit_api/database/autosubmit.py | 3500 ------------------------- 1 file changed, 3500 deletions(-) delete mode 100644 autosubmit_api/database/autosubmit.py diff --git a/autosubmit_api/database/autosubmit.py b/autosubmit_api/database/autosubmit.py deleted file mode 100644 index 4b75fa68..00000000 --- a/autosubmit_api/database/autosubmit.py +++ /dev/null @@ -1,3500 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2017 Earth Sciences Department, BSC-CNS - -# This file is part of Autosubmit. - -# Autosubmit is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# Autosubmit is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with Autosubmit. If not, see . -# pipeline_test - - -import traceback -from pyparsing import nestedExpr -from collections import defaultdict -from distutils.util import strtobool -from pkg_resources import resource_listdir, resource_exists, resource_string -import portalocker -import datetime -import signal -import random -import re -import shutil -import sys -import pwd -import os -import copy -import time -import tarfile -import json -import subprocess -import argparse - -sys.path.insert(0, os.path.abspath('.')) -from autosubmit_api.config.basicConfig import APIBasicConfig -from autosubmit_api.config.config_common import AutosubmitConfigResolver -from bscearth.utils.config_parser import ConfigParserFactory -from autosubmit_api.common.utils import Status -from autosubmit_api.git.autosubmit_git import AutosubmitGit -from autosubmit_api.autosubmit_legacy.job.job_list import JobList -from autosubmit_api.autosubmit_legacy.job.job_packages import JobPackageThread -from autosubmit_api.autosubmit_legacy.job.job_package_persistence import JobPackagePersistence -from autosubmit_api.autosubmit_legacy.job.job_list_persistence import JobListPersistenceDb -from autosubmit_api.autosubmit_legacy.job.job_list_persistence import JobListPersistencePkl -from autosubmit_api.autosubmit_legacy.job.job_grouping import JobGrouping -from bscearth.utils.log import Log -from autosubmit_api.database.db_common import create_db -from autosubmit_api.experiment.experiment_common import new_experiment -from autosubmit_api.experiment.experiment_common import copy_experiment -from autosubmit_api.database.db_common import delete_experiment -from autosubmit_api.database.db_common import get_autosubmit_version -from autosubmit_api.monitor.monitor import Monitor -from bscearth.utils.date import date2str -from autosubmit_api.notifications.mail_notifier import MailNotifier -from autosubmit_api.notifications.notifier import Notifier -from autosubmit_api.autosubmit_legacy.platforms.saga_submitter import SagaSubmitter -from autosubmit_api.autosubmit_legacy.platforms.paramiko_submitter import ParamikoSubmitter -from autosubmit_api.autosubmit_legacy.job.job_exceptions import WrongTemplateException -from autosubmit_api.autosubmit_legacy.job.job_packager import JobPackager -""" -Main module for autosubmit. Only contains an interface class to all functionality implemented on autosubmit -""" - -try: - # noinspection PyCompatibility - from configparser import SafeConfigParser -except ImportError: - # noinspection PyCompatibility - from configparser import SafeConfigParser - -# It is Python dialog available? (optional dependency) -try: - import dialog -except Exception: - dialog = None - - -# noinspection PyPackageRequirements -# noinspection PyPackageRequirements -# noinspection PyPackageRequirements - -# noinspection PyUnusedLocal - - -def signal_handler(signal_received, frame): - """ - Used to handle interrupt signals, allowing autosubmit to clean before exit - - :param signal_received: - :param frame: - """ - Log.info('Autosubmit will interrupt at the next safe occasion') - Autosubmit.exit = True - - -class Autosubmit: - """ - Interface class for autosubmit. - """ - # sys.setrecursionlimit(500000) - # # Get the version number from the relevant file. If not, from autosubmit package - # scriptdir = os.path.abspath(os.path.dirname(__file__)) - - # if not os.path.exists(os.path.join(scriptdir, 'VERSION')): - # scriptdir = os.path.join(scriptdir, os.path.pardir) - - # version_path = os.path.join(scriptdir, 'VERSION') - # readme_path = os.path.join(scriptdir, 'README') - # changes_path = os.path.join(scriptdir, 'CHANGELOG') - # if os.path.isfile(version_path): - # with open(version_path) as f: - # autosubmit_version = f.read().strip() - # else: - # autosubmit_version = require("autosubmitAPIwu")[0].version - - exit = False - - @staticmethod - def parse_args(): - """ - Parse arguments given to an executable and start execution of command given - """ - try: - APIBasicConfig.read() - - parser = argparse.ArgumentParser( - description='Main executable for autosubmit. ') - parser.add_argument('-v', '--version', action='version', version=Autosubmit.autosubmit_version, - help="returns autosubmit's version number and exit") - parser.add_argument('-lf', '--logfile', choices=('EVERYTHING', 'DEBUG', 'INFO', 'RESULT', 'USER_WARNING', - 'WARNING', 'ERROR', 'CRITICAL', 'NO_LOG'), - default='DEBUG', type=str, - help="sets file's log level.") - parser.add_argument('-lc', '--logconsole', choices=('EVERYTHING', 'DEBUG', 'INFO', 'RESULT', 'USER_WARNING', - 'WARNING', 'ERROR', 'CRITICAL', 'NO_LOG'), - default='INFO', type=str, - help="sets console's log level") - - subparsers = parser.add_subparsers(dest='command') - - # Run - subparser = subparsers.add_parser( - 'run', description="runs specified experiment") - subparser.add_argument('expid', help='experiment identifier') - subparser.add_argument('-nt', '--notransitive', action='store_true', - default=False, help='Disable transitive reduction') - - # Expid - subparser = subparsers.add_parser( - 'expid', description="Creates a new experiment") - group = subparser.add_mutually_exclusive_group() - group.add_argument( - '-y', '--copy', help='makes a copy of the specified experiment') - group.add_argument('-dm', '--dummy', action='store_true', - help='creates a new experiment with default values, usually for testing') - group.add_argument('-op', '--operational', action='store_true', - help='creates a new experiment with operational experiment id') - subparser.add_argument('-H', '--HPC', required=True, - help='specifies the HPC to use for the experiment') - subparser.add_argument('-d', '--description', type=str, required=True, - help='sets a description for the experiment to store in the database.') - - # Delete - subparser = subparsers.add_parser( - 'delete', description="delete specified experiment") - subparser.add_argument('expid', help='experiment identifier') - subparser.add_argument( - '-f', '--force', action='store_true', help='deletes experiment without confirmation') - - # Monitor - subparser = subparsers.add_parser( - 'monitor', description="plots specified experiment") - subparser.add_argument('expid', help='experiment identifier') - subparser.add_argument('-o', '--output', choices=('pdf', 'png', 'ps', 'svg'), default='pdf', - help='chooses type of output for generated plot') - subparser.add_argument('-group_by', choices=('date', 'member', 'chunk', 'split', 'automatic'), default=None, - help='Groups the jobs automatically or by date, member, chunk or split') - subparser.add_argument('-expand', type=str, - help='Supply the list of dates/members/chunks to filter the list of jobs. Default = "Any". ' - 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"') - subparser.add_argument( - '-expand_status', type=str, help='Select the statuses to be expanded') - subparser.add_argument('--hide_groups', action='store_true', - default=False, help='Hides the groups from the plot') - subparser.add_argument('-cw', '--check_wrapper', action='store_true', - default=False, help='Generate possible wrapper in the current workflow') - - group2 = subparser.add_mutually_exclusive_group(required=False) - - group.add_argument('-fs', '--filter_status', type=str, - choices=('Any', 'READY', 'COMPLETED', - 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN'), - help='Select the original status to filter the list of jobs') - group = subparser.add_mutually_exclusive_group(required=False) - group.add_argument('-fl', '--list', type=str, - help='Supply the list of job names to be filtered. Default = "Any". ' - 'LIST = "b037_20101101_fc3_21_sim b037_20111101_fc4_26_sim"') - group.add_argument('-fc', '--filter_chunks', type=str, - help='Supply the list of chunks to filter the list of jobs. Default = "Any". ' - 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"') - group.add_argument('-fs', '--filter_status', type=str, - choices=('Any', 'READY', 'COMPLETED', - 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN'), - help='Select the original status to filter the list of jobs') - group.add_argument('-ft', '--filter_type', type=str, - help='Select the job type to filter the list of jobs') - subparser.add_argument('--hide', action='store_true', default=False, - help='hides plot window') - group2.add_argument('--txt', action='store_true', default=False, - help='Generates only txt status file') - - group2.add_argument('-txtlog', '--txt_logfiles', action='store_true', default=False, - help='Generates only txt status file(AS < 3.12b behaviour)') - - subparser.add_argument('-nt', '--notransitive', action='store_true', - default=False, help='Disable transitive reduction') - - # Stats - subparser = subparsers.add_parser( - 'stats', description="plots statistics for specified experiment") - subparser.add_argument('expid', help='experiment identifier') - subparser.add_argument('-ft', '--filter_type', type=str, help='Select the job type to filter ' - 'the list of jobs') - subparser.add_argument('-fp', '--filter_period', type=int, help='Select the period to filter jobs ' - 'from current time to the past ' - 'in number of hours back') - subparser.add_argument('-o', '--output', choices=('pdf', 'png', 'ps', 'svg'), default='pdf', - help='type of output for generated plot') - subparser.add_argument('--hide', action='store_true', default=False, - help='hides plot window') - subparser.add_argument('-nt', '--notransitive', action='store_true', - default=False, help='Disable transitive reduction') - - # Clean - subparser = subparsers.add_parser( - 'clean', description="clean specified experiment") - subparser.add_argument('expid', help='experiment identifier') - subparser.add_argument( - '-pr', '--project', action="store_true", help='clean project') - subparser.add_argument('-p', '--plot', action="store_true", - help='clean plot, only 2 last will remain') - subparser.add_argument('-s', '--stats', action="store_true", - help='clean stats, only last will remain') - - # Recovery - subparser = subparsers.add_parser( - 'recovery', description="recover specified experiment") - subparser.add_argument( - 'expid', type=str, help='experiment identifier') - subparser.add_argument( - '-np', '--noplot', action='store_true', default=False, help='omit plot') - subparser.add_argument('--all', action="store_true", default=False, - help='Get completed files to synchronize pkl') - subparser.add_argument( - '-s', '--save', action="store_true", default=False, help='Save changes to disk') - subparser.add_argument('--hide', action='store_true', default=False, - help='hides plot window') - subparser.add_argument('-group_by', choices=('date', 'member', 'chunk', 'split', 'automatic'), default=None, - help='Groups the jobs automatically or by date, member, chunk or split') - subparser.add_argument('-expand', type=str, - help='Supply the list of dates/members/chunks to filter the list of jobs. Default = "Any". ' - 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"') - subparser.add_argument( - '-expand_status', type=str, help='Select the statuses to be expanded') - subparser.add_argument('-nt', '--notransitive', action='store_true', - default=False, help='Disable transitive reduction') - subparser.add_argument('-nl', '--no_recover_logs', action='store_true', default=False, - help='Disable logs recovery') - # Migrate - subparser = subparsers.add_parser( - 'migrate', description="Migrate experiments from current user to another") - subparser.add_argument('expid', help='experiment identifier') - group = subparser.add_mutually_exclusive_group(required=True) - group.add_argument('-o', '--offer', action="store_true", - default=False, help='Offer experiment') - group.add_argument('-p', '--pickup', action="store_true", - default=False, help='Pick-up released experiment') - - # Inspect - subparser = subparsers.add_parser( - 'inspect', description="Generate all .cmd files") - subparser.add_argument('expid', help='experiment identifier') - subparser.add_argument('-nt', '--notransitive', action='store_true', - default=False, help='Disable transitive reduction') - subparser.add_argument( - '-f', '--force', action="store_true", help='Overwrite all cmd') - subparser.add_argument('-cw', '--check_wrapper', action='store_true', - default=False, help='Generate possible wrapper in the current workflow') - - group.add_argument('-fs', '--filter_status', type=str, - choices=('Any', 'READY', 'COMPLETED', - 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN'), - help='Select the original status to filter the list of jobs') - group = subparser.add_mutually_exclusive_group(required=False) - group.add_argument('-fl', '--list', type=str, - help='Supply the list of job names to be filtered. Default = "Any". ' - 'LIST = "b037_20101101_fc3_21_sim b037_20111101_fc4_26_sim"') - group.add_argument('-fc', '--filter_chunks', type=str, - help='Supply the list of chunks to filter the list of jobs. Default = "Any". ' - 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"') - group.add_argument('-fs', '--filter_status', type=str, - choices=('Any', 'READY', 'COMPLETED', - 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN'), - help='Select the original status to filter the list of jobs') - group.add_argument('-ft', '--filter_type', type=str, - help='Select the job type to filter the list of jobs') - - # Check - subparser = subparsers.add_parser( - 'check', description="check configuration for specified experiment") - subparser.add_argument('expid', help='experiment identifier') - subparser.add_argument('-nt', '--notransitive', action='store_true', - default=False, help='Disable transitive reduction') - # Describe - subparser = subparsers.add_parser( - 'describe', description="Show details for specified experiment") - subparser.add_argument('expid', help='experiment identifier') - - # Create - subparser = subparsers.add_parser( - 'create', description="create specified experiment joblist") - subparser.add_argument('expid', help='experiment identifier') - subparser.add_argument( - '-np', '--noplot', action='store_true', default=False, help='omit plot') - subparser.add_argument('--hide', action='store_true', default=False, - help='hides plot window') - subparser.add_argument('-o', '--output', choices=('pdf', 'png', 'ps', 'svg'), default='pdf', - help='chooses type of output for generated plot') - subparser.add_argument('-group_by', choices=('date', 'member', 'chunk', 'split', 'automatic'), default=None, - help='Groups the jobs automatically or by date, member, chunk or split') - subparser.add_argument('-expand', type=str, - help='Supply the list of dates/members/chunks to filter the list of jobs. Default = "Any". ' - 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"') - subparser.add_argument( - '-expand_status', type=str, help='Select the statuses to be expanded') - subparser.add_argument('-nt', '--notransitive', action='store_true', - default=False, help='Disable transitive reduction') - subparser.add_argument('-cw', '--check_wrapper', action='store_true', - default=False, help='Generate possible wrapper in the current workflow') - - # Configure - subparser = subparsers.add_parser('configure', description="configure database and path for autosubmit. It " - "can be done at machine, user or local level." - "If no arguments specified configure will " - "display dialog boxes (if installed)") - subparser.add_argument( - '--advanced', action="store_true", help="Open advanced configuration of autosubmit") - subparser.add_argument('-db', '--databasepath', default=None, help='path to database. If not supplied, ' - 'it will prompt for it') - subparser.add_argument( - '-dbf', '--databasefilename', default=None, help='database filename') - subparser.add_argument('-lr', '--localrootpath', default=None, help='path to store experiments. If not ' - 'supplied, it will prompt for it') - subparser.add_argument('-pc', '--platformsconfpath', default=None, help='path to platforms.conf file to ' - 'use by default. Optional') - subparser.add_argument('-jc', '--jobsconfpath', default=None, help='path to jobs.conf file to use by ' - 'default. Optional') - subparser.add_argument( - '-sm', '--smtphostname', default=None, help='STMP server hostname. Optional') - subparser.add_argument( - '-mf', '--mailfrom', default=None, help='Notifications sender address. Optional') - group = subparser.add_mutually_exclusive_group() - group.add_argument('--all', action="store_true", - help='configure for all users') - group.add_argument('--local', action="store_true", help='configure only for using Autosubmit from this ' - 'path') - - # Install - subparsers.add_parser( - 'install', description='install database for autosubmit on the configured folder') - - # Set status - subparser = subparsers.add_parser( - 'setstatus', description="sets job status for an experiment") - subparser.add_argument('expid', help='experiment identifier') - subparser.add_argument( - '-np', '--noplot', action='store_true', default=False, help='omit plot') - subparser.add_argument( - '-s', '--save', action="store_true", default=False, help='Save changes to disk') - - subparser.add_argument('-t', '--status_final', - choices=('READY', 'COMPLETED', 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN', - 'QUEUING', 'RUNNING'), - required=True, - help='Supply the target status') - group = subparser.add_mutually_exclusive_group(required=True) - group.add_argument('-fl', '--list', type=str, - help='Supply the list of job names to be changed. Default = "Any". ' - 'LIST = "b037_20101101_fc3_21_sim b037_20111101_fc4_26_sim"') - group.add_argument('-fc', '--filter_chunks', type=str, - help='Supply the list of chunks to change the status. Default = "Any". ' - 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"') - group.add_argument('-fs', '--filter_status', type=str, - help='Select the status (one or more) to filter the list of jobs.' - "Valid values = ['Any', 'READY', 'COMPLETED', 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN']") - group.add_argument('-ft', '--filter_type', type=str, - help='Select the job type to filter the list of jobs') - - subparser.add_argument('--hide', action='store_true', default=False, - help='hides plot window') - subparser.add_argument('-group_by', choices=('date', 'member', 'chunk', 'split', 'automatic'), default=None, - help='Groups the jobs automatically or by date, member, chunk or split') - subparser.add_argument('-expand', type=str, - help='Supply the list of dates/members/chunks to filter the list of jobs. Default = "Any". ' - 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"') - subparser.add_argument( - '-expand_status', type=str, help='Select the statuses to be expanded') - subparser.add_argument('-nt', '--notransitive', action='store_true', - default=False, help='Disable transitive reduction') - subparser.add_argument('-cw', '--check_wrapper', action='store_true', - default=False, help='Generate possible wrapper in the current workflow') - - # Test Case - subparser = subparsers.add_parser( - 'testcase', description='create test case experiment') - subparser.add_argument( - '-y', '--copy', help='makes a copy of the specified experiment') - subparser.add_argument( - '-d', '--description', required=True, help='description of the test case') - subparser.add_argument('-c', '--chunks', help='chunks to run') - subparser.add_argument('-m', '--member', help='member to run') - subparser.add_argument('-s', '--stardate', help='stardate to run') - subparser.add_argument( - '-H', '--HPC', required=True, help='HPC to run experiment on it') - subparser.add_argument( - '-b', '--branch', help='branch of git to run (or revision from subversion)') - - # Test - subparser = subparsers.add_parser( - 'test', description='test experiment') - subparser.add_argument('expid', help='experiment identifier') - subparser.add_argument( - '-c', '--chunks', required=True, help='chunks to run') - subparser.add_argument('-m', '--member', help='member to run') - subparser.add_argument('-s', '--stardate', help='stardate to run') - subparser.add_argument( - '-H', '--HPC', help='HPC to run experiment on it') - subparser.add_argument( - '-b', '--branch', help='branch of git to run (or revision from subversion)') - - # Refresh - subparser = subparsers.add_parser( - 'refresh', description='refresh project directory for an experiment') - subparser.add_argument('expid', help='experiment identifier') - subparser.add_argument('-mc', '--model_conf', default=False, action='store_true', - help='overwrite model conf file') - subparser.add_argument('-jc', '--jobs_conf', default=False, action='store_true', - help='overwrite jobs conf file') - - # Archive - subparser = subparsers.add_parser( - 'archive', description='archives an experiment') - subparser.add_argument('expid', help='experiment identifier') - - # Unarchive - subparser = subparsers.add_parser( - 'unarchive', description='unarchives an experiment') - subparser.add_argument('expid', help='experiment identifier') - - # Readme - subparsers.add_parser('readme', description='show readme') - - # Changelog - subparsers.add_parser('changelog', description='show changelog') - - args = parser.parse_args() - - Log.set_console_level(args.logconsole) - Log.set_file_level(args.logfile) - - if args.command == 'run': - return Autosubmit.run_experiment(args.expid, args.notransitive) - elif args.command == 'expid': - return Autosubmit.expid(args.HPC, args.description, args.copy, args.dummy, False, - args.operational) != '' - elif args.command == 'delete': - return Autosubmit.delete(args.expid, args.force) - elif args.command == 'monitor': - return Autosubmit.monitor(args.expid, args.output, args.list, args.filter_chunks, args.filter_status, - args.filter_type, args.hide, args.txt, args.group_by, args.expand, - args.expand_status, args.hide_groups, args.notransitive, args.check_wrapper, args.txt_logfiles) - elif args.command == 'stats': - return Autosubmit.statistics(args.expid, args.filter_type, args.filter_period, args.output, args.hide, - args.notransitive) - elif args.command == 'clean': - return Autosubmit.clean(args.expid, args.project, args.plot, args.stats) - elif args.command == 'recovery': - return Autosubmit.recovery(args.expid, args.noplot, args.save, args.all, args.hide, args.group_by, - args.expand, args.expand_status, args.notransitive, args.no_recover_logs) - elif args.command == 'check': - return Autosubmit.check(args.expid, args.notransitive) - elif args.command == 'inspect': - return Autosubmit.inspect(args.expid, args.list, args.filter_chunks, args.filter_status, - args.filter_type, args.notransitive, args.force, args.check_wrapper) - elif args.command == 'describe': - return Autosubmit.describe(args.expid) - elif args.command == 'migrate': - return Autosubmit.migrate(args.expid, args.offer, args.pickup) - elif args.command == 'create': - return Autosubmit.create(args.expid, args.noplot, args.hide, args.output, args.group_by, args.expand, - args.expand_status, args.notransitive, args.check_wrapper) - elif args.command == 'configure': - if not args.advanced or (args.advanced and dialog is None): - return Autosubmit.configure(args.advanced, args.databasepath, args.databasefilename, - args.localrootpath, args.platformsconfpath, args.jobsconfpath, - args.smtphostname, args.mailfrom, args.all, args.local) - else: - return Autosubmit.configure_dialog() - elif args.command == 'install': - return Autosubmit.install() - elif args.command == 'setstatus': - return Autosubmit.set_status(args.expid, args.noplot, args.save, args.status_final, args.list, - args.filter_chunks, args.filter_status, args.filter_type, args.hide, - args.group_by, args.expand, args.expand_status, args.notransitive, args.check_wrapper) - elif args.command == 'testcase': - return Autosubmit.testcase(args.copy, args.description, args.chunks, args.member, args.stardate, - args.HPC, args.branch) - elif args.command == 'test': - return Autosubmit.test(args.expid, args.chunks, args.member, args.stardate, args.HPC, args.branch) - elif args.command == 'refresh': - return Autosubmit.refresh(args.expid, args.model_conf, args.jobs_conf) - elif args.command == 'archive': - return Autosubmit.archive(args.expid) - elif args.command == 'unarchive': - return Autosubmit.unarchive(args.expid) - - elif args.command == 'readme': - if os.path.isfile(Autosubmit.readme_path): - with open(Autosubmit.readme_path) as f: - print(f.read()) - return True - return False - elif args.command == 'changelog': - if os.path.isfile(Autosubmit.changes_path): - with open(Autosubmit.changes_path) as f: - print(f.read()) - return True - return False - except Exception as e: - from traceback import format_exc - Log.critical( - 'Unhandled exception on Autosubmit: {0}\n{1}', e, format_exc(10)) - - return False - - @staticmethod - def _delete_expid(expid_delete): - """ - Removes an experiment from path and database - - :type expid_delete: str - :param expid_delete: identifier of the experiment to delete - """ - if expid_delete == '' or expid_delete is None and not os.path.exists(os.path.join(APIBasicConfig.LOCAL_ROOT_DIR, - expid_delete)): - Log.info("Experiment directory does not exist.") - else: - Log.info("Removing experiment directory...") - ret = False - if pwd.getpwuid(os.stat(os.path.join(APIBasicConfig.LOCAL_ROOT_DIR, expid_delete)).st_uid).pw_name == os.getlogin(): - try: - - shutil.rmtree(os.path.join( - APIBasicConfig.LOCAL_ROOT_DIR, expid_delete)) - except OSError as e: - Log.warning('Can not delete experiment folder: {0}', e) - return ret - Log.info("Deleting experiment from database...") - ret = delete_experiment(expid_delete) - if ret: - Log.result("Experiment {0} deleted".format(expid_delete)) - else: - Log.warning( - "Current User is not the Owner {0} can not be deleted!", expid_delete) - return ret - - @staticmethod - def expid(hpc, description, copy_id='', dummy=False, test=False, operational=False): - """ - Creates a new experiment for given HPC - - :param operational: if true, creates an operational experiment - :type operational: bool - :type hpc: str - :type description: str - :type copy_id: str - :type dummy: bool - :param hpc: name of the main HPC for the experiment - :param description: short experiment's description. - :param copy_id: experiment identifier of experiment to copy - :param dummy: if true, writes a default dummy configuration for testing - :param test: if true, creates an experiment for testing - :return: experiment identifier. If method fails, returns ''. - :rtype: str - """ - APIBasicConfig.read() - - log_path = os.path.join( - APIBasicConfig.LOCAL_ROOT_DIR, 'ASlogs', 'expid.log'.format(os.getuid())) - try: - Log.set_file(log_path) - except IOError as e: - Log.error("Can not create log file in path {0}: {1}".format( - log_path, e.message)) - exp_id = None - if description is None: - Log.error("Missing experiment description.") - return '' - if hpc is None: - Log.error("Missing HPC.") - return '' - if not copy_id: - exp_id = new_experiment( - description, Autosubmit.autosubmit_version, test, operational) - if exp_id == '': - return '' - try: - os.mkdir(os.path.join(APIBasicConfig.LOCAL_ROOT_DIR, exp_id)) - - os.mkdir(os.path.join( - APIBasicConfig.LOCAL_ROOT_DIR, exp_id, 'conf')) - Log.info("Copying config files...") - - # autosubmit config and experiment copied from AS. - files = resource_listdir('autosubmit.config', 'files') - for filename in files: - if resource_exists('autosubmit.config', 'files/' + filename): - index = filename.index('.') - new_filename = filename[:index] + \ - "_" + exp_id + filename[index:] - - if filename == 'platforms.conf' and APIBasicConfig.DEFAULT_PLATFORMS_CONF != '': - content = open(os.path.join( - APIBasicConfig.DEFAULT_PLATFORMS_CONF, filename)).read() - elif filename == 'jobs.conf' and APIBasicConfig.DEFAULT_JOBS_CONF != '': - content = open(os.path.join( - APIBasicConfig.DEFAULT_JOBS_CONF, filename)).read() - else: - content = resource_string( - 'autosubmit.config', 'files/' + filename) - - conf_new_filename = os.path.join( - APIBasicConfig.LOCAL_ROOT_DIR, exp_id, "conf", new_filename) - Log.debug(conf_new_filename) - open(conf_new_filename, 'w').write(content) - Autosubmit._prepare_conf_files( - exp_id, hpc, Autosubmit.autosubmit_version, dummy) - except (OSError, IOError) as e: - Log.error( - "Can not create experiment: {0}\nCleaning...".format(e)) - Autosubmit._delete_expid(exp_id) - return '' - else: - try: - if os.path.exists(os.path.join(APIBasicConfig.LOCAL_ROOT_DIR, copy_id)): - exp_id = copy_experiment( - copy_id, description, Autosubmit.autosubmit_version, test, operational) - if exp_id == '': - return '' - dir_exp_id = os.path.join( - APIBasicConfig.LOCAL_ROOT_DIR, exp_id) - os.mkdir(dir_exp_id) - os.mkdir(dir_exp_id + '/conf') - Log.info("Copying previous experiment config directories") - conf_copy_id = os.path.join( - APIBasicConfig.LOCAL_ROOT_DIR, copy_id, "conf") - files = os.listdir(conf_copy_id) - for filename in files: - if os.path.isfile(os.path.join(conf_copy_id, filename)): - new_filename = filename.replace(copy_id, exp_id) - content = open(os.path.join( - conf_copy_id, filename), 'r').read() - open(os.path.join(dir_exp_id, "conf", - new_filename), 'w').write(content) - Autosubmit._prepare_conf_files( - exp_id, hpc, Autosubmit.autosubmit_version, dummy) - ##### - autosubmit_config = AutosubmitConfigResolver( - copy_id, APIBasicConfig, ConfigParserFactory()) - if autosubmit_config.check_conf_files(): - project_type = autosubmit_config.get_project_type() - if project_type == "git": - autosubmit_config.check_proj() - autosubmit_git = AutosubmitGit(copy_id[0]) - Log.info("checking model version...") - if not autosubmit_git.check_commit(autosubmit_config): - return False - ##### - else: - Log.critical( - "The previous experiment directory does not exist") - return '' - except (OSError, IOError) as e: - Log.error( - "Can not create experiment: {0}\nCleaning...".format(e)) - Autosubmit._delete_expid(exp_id) - return '' - - Log.debug("Creating temporal directory...") - exp_id_path = os.path.join(APIBasicConfig.LOCAL_ROOT_DIR, exp_id) - tmp_path = os.path.join(exp_id_path, "tmp") - os.mkdir(tmp_path) - os.chmod(tmp_path, 0o775) - os.mkdir(os.path.join(tmp_path, APIBasicConfig.LOCAL_ASLOG_DIR)) - os.chmod(os.path.join(tmp_path, APIBasicConfig.LOCAL_ASLOG_DIR), 0o775) - Log.debug("Creating temporal remote directory...") - remote_tmp_path = os.path.join(tmp_path, "LOG_" + exp_id) - os.mkdir(remote_tmp_path) - os.chmod(remote_tmp_path, 0o775) - - Log.debug("Creating pkl directory...") - os.mkdir(os.path.join(exp_id_path, "pkl")) - - Log.debug("Creating plot directory...") - os.mkdir(os.path.join(exp_id_path, "plot")) - os.chmod(os.path.join(exp_id_path, "plot"), 0o775) - Log.result("Experiment registered successfully") - Log.user_warning("Remember to MODIFY the config files!") - return exp_id - - @staticmethod - def delete(expid, force): - """ - Deletes and experiment from database and experiment's folder - - :type force: bool - :type expid: str - :param expid: identifier of the experiment to delete - :param force: if True, does not ask for confirmation - - :returns: True if succesful, False if not - :rtype: bool - """ - log_path = os.path.join( - APIBasicConfig.LOCAL_ROOT_DIR, "ASlogs", 'delete.log'.format(os.getuid())) - try: - Log.set_file(log_path) - except IOError as e: - Log.error("Can not create log file in path {0}: {1}".format( - log_path, e.message)) - - if os.path.exists(os.path.join(APIBasicConfig.LOCAL_ROOT_DIR, expid)): - if force or Autosubmit._user_yes_no_query("Do you want to delete " + expid + " ?"): - return Autosubmit._delete_expid(expid) - else: - Log.info("Quitting...") - return False - else: - Log.error("The experiment does not exist") - return True - - @staticmethod - def _load_parameters(as_conf, job_list, platforms): - # Load parameters - Log.debug("Loading parameters...") - parameters = as_conf.load_parameters() - for platform_name in platforms: - platform = platforms[platform_name] - platform.add_parameters(parameters) - - platform = platforms[as_conf.get_platform().lower()] - platform.add_parameters(parameters, True) - - job_list.parameters = parameters - - @staticmethod - def inspect(expid, lst, filter_chunks, filter_status, filter_section, notransitive=False, force=False, check_wrapper=False): - """ - Generates cmd files experiment. - - :type expid: str - :param expid: identifier of experiment to be run - :return: True if run to the end, False otherwise - :rtype: bool - """ - - if expid is None: - Log.critical("Missing experiment id") - - APIBasicConfig.read() - exp_path = os.path.join(APIBasicConfig.LOCAL_ROOT_DIR, expid) - tmp_path = os.path.join(exp_path, APIBasicConfig.LOCAL_TMP_DIR) - if os.path.exists(os.path.join(tmp_path, 'autosubmit.lock')): - locked = True - else: - locked = False - - if not os.path.exists(exp_path): - Log.critical( - "The directory %s is needed and does not exist" % exp_path) - Log.warning("Does an experiment with the given id exist?") - return 1 - Log.info("Starting inspect command") - Log.set_file(os.path.join( - tmp_path, APIBasicConfig.LOCAL_ASLOG_DIR, 'generate.log')) - os.system('clear') - signal.signal(signal.SIGINT, signal_handler) - as_conf = AutosubmitConfigResolver(expid, APIBasicConfig, ConfigParserFactory()) - if not as_conf.check_conf_files(): - Log.critical('Can not generate scripts with invalid configuration') - return False - project_type = as_conf.get_project_type() - if project_type != "none": - # Check proj configuration - as_conf.check_proj() - safetysleeptime = as_conf.get_safetysleeptime() - Log.debug("The Experiment name is: {0}", expid) - Log.debug("Sleep: {0}", safetysleeptime) - packages_persistence = JobPackagePersistence(os.path.join(APIBasicConfig.LOCAL_ROOT_DIR, expid, "pkl"), - "job_packages_" + expid) - os.chmod(os.path.join(APIBasicConfig.LOCAL_ROOT_DIR, expid, - "pkl", "job_packages_" + expid + ".db"), 0o664) - - packages_persistence.reset_table(True) - job_list_original = Autosubmit.load_job_list( - expid, as_conf, notransitive=notransitive) - job_list = copy.deepcopy(job_list_original) - job_list.packages_dict = {} - - Log.debug("Length of the jobs list: {0}", len(job_list)) - - # variables to be updated on the fly - safetysleeptime = as_conf.get_safetysleeptime() - Log.debug("Sleep: {0}", safetysleeptime) - # Generate - Log.info("Starting to generate cmd scripts") - - if not isinstance(job_list, type([])): - jobs = [] - jobs_cw = [] - if check_wrapper and (not locked or (force and locked)): - Log.info("Generating all cmd script adapted for wrappers") - jobs = job_list.get_uncompleted() - - jobs_cw = job_list.get_completed() - else: - if (force and not locked) or (force and locked): - Log.info("Overwritting all cmd scripts") - jobs = job_list.get_job_list() - elif locked: - Log.warning( - "There is a .lock file and not -f, generating only all unsubmitted cmd scripts") - jobs = job_list.get_unsubmitted() - else: - Log.info("Generating cmd scripts only for selected jobs") - if filter_chunks: - fc = filter_chunks - Log.debug(fc) - if fc == 'Any': - jobs = job_list.get_job_list() - else: - # noinspection PyTypeChecker - data = json.loads(Autosubmit._create_json(fc)) - for date_json in data['sds']: - date = date_json['sd'] - jobs_date = [j for j in job_list.get_job_list() if date2str( - j.date) == date] - - for member_json in date_json['ms']: - member = member_json['m'] - jobs_member = [j for j in jobs_date if j.member == member] - - for chunk_json in member_json['cs']: - chunk = int(chunk_json) - jobs = jobs + \ - [job for job in [j for j in jobs_member if j.chunk == chunk]] - - elif filter_status: - Log.debug( - "Filtering jobs with status {0}", filter_status) - if filter_status == 'Any': - jobs = job_list.get_job_list() - else: - fs = Autosubmit._get_status(filter_status) - jobs = [job for job in [j for j in job_list.get_job_list() if j.status == fs]] - - elif filter_section: - ft = filter_section - Log.debug(ft) - - if ft == 'Any': - jobs = job_list.get_job_list() - else: - for job in job_list.get_job_list(): - if job.section == ft: - jobs.append(job) - elif lst: - jobs_lst = lst.split() - - if jobs == 'Any': - jobs = job_list.get_job_list() - else: - for job in job_list.get_job_list(): - if job.name in jobs_lst: - jobs.append(job) - else: - jobs = job_list.get_job_list() - if isinstance(jobs, type([])): - referenced_jobs_to_remove = set() - for job in jobs: - for child in job.children: - if child not in jobs: - referenced_jobs_to_remove.add(child) - for parent in job.parents: - if parent not in jobs: - referenced_jobs_to_remove.add(parent) - - for job in jobs: - job.status = Status.WAITING - - Autosubmit.generate_scripts_andor_wrappers( - as_conf, job_list, jobs, packages_persistence, False) - if len(jobs_cw) > 0: - referenced_jobs_to_remove = set() - for job in jobs_cw: - for child in job.children: - if child not in jobs_cw: - referenced_jobs_to_remove.add(child) - for parent in job.parents: - if parent not in jobs_cw: - referenced_jobs_to_remove.add(parent) - - for job in jobs_cw: - job.status = Status.WAITING - Autosubmit.generate_scripts_andor_wrappers( - as_conf, job_list, jobs_cw, packages_persistence, False) - - Log.info("no more scripts to generate, now proceed to check them manually") - time.sleep(safetysleeptime) - return True - - @staticmethod - def generate_scripts_andor_wrappers(as_conf, job_list, jobs_filtered, packages_persistence, only_wrappers=False): - """ - as_conf: AutosubmitConfig object - job_list: JobList object, contains a list of jobs - jobs_filtered: list of jobs - packages_persistence: Database handler - only_wrappers: True - """ - job_list._job_list = jobs_filtered - job_list.update_list(as_conf, False) - # Identifying the submitter and loading it - submitter = Autosubmit._get_submitter(as_conf) - # Function depending on the submitter - submitter.load_platforms(as_conf) - # Identifying HPC from config files - hpcarch = as_conf.get_platform() - # - Autosubmit._load_parameters(as_conf, job_list, submitter.platforms) - platforms_to_test = set() - for job in job_list.get_job_list(): - if job.platform_name is None: - job.platform_name = hpcarch - # noinspection PyTypeChecker - job.platform = submitter.platforms[job.platform_name.lower()] - # noinspection PyTypeChecker - platforms_to_test.add(job.platform) - # case setstatus - job_list.check_scripts(as_conf) - job_list.update_list(as_conf, False) - Autosubmit._load_parameters(as_conf, job_list, submitter.platforms) - while job_list.get_active(): - Autosubmit.submit_ready_jobs( - as_conf, job_list, platforms_to_test, packages_persistence, True, only_wrappers) - - job_list.update_list(as_conf, False) - - @staticmethod - def run_experiment(expid, notransitive=False): - """ - Runs and experiment (submitting all the jobs properly and repeating its execution in case of failure). - - :type expid: str - :param expid: identifier of experiment to be run - :return: True if run to the end, False otherwise - :rtype: bool - """ - if expid is None: - Log.critical("Missing experiment id") - - APIBasicConfig.read() - exp_path = os.path.join(APIBasicConfig.LOCAL_ROOT_DIR, expid) - tmp_path = os.path.join(exp_path, APIBasicConfig.LOCAL_TMP_DIR) - aslogs_path = os.path.join(tmp_path, APIBasicConfig.LOCAL_ASLOG_DIR) - if not os.path.exists(aslogs_path): - os.mkdir(aslogs_path) - os.chmod(aslogs_path, 0o775) - if not os.path.exists(exp_path): - Log.critical( - "The directory %s is needed and does not exist" % exp_path) - Log.warning("Does an experiment with the given id exist?") - return 1 - - # checking host whitelist - import platform - host = platform.node() - print(host) - if APIBasicConfig.ALLOWED_HOSTS and host not in APIBasicConfig.ALLOWED_HOSTS: - Log.info("\n Autosubmit run command is not allowed on this host") - return False - - # checking if there is a lock file to avoid multiple running on the same expid - try: - with portalocker.Lock(os.path.join(tmp_path, 'autosubmit.lock'), timeout=1): - Log.info( - "Preparing .lock file to avoid multiple instances with same experiment id") - - Log.set_file(os.path.join(aslogs_path, 'run.log')) - os.system('clear') - - signal.signal(signal.SIGINT, signal_handler) - - as_conf = AutosubmitConfigResolver( - expid, APIBasicConfig, ConfigParserFactory()) - if not as_conf.check_conf_files(): - Log.critical('Can not run with invalid configuration') - return False - - project_type = as_conf.get_project_type() - if project_type != "none": - # Check proj configuration - as_conf.check_proj() - - hpcarch = as_conf.get_platform() - - safetysleeptime = as_conf.get_safetysleeptime() - retrials = as_conf.get_retrials() - - submitter = Autosubmit._get_submitter(as_conf) - submitter.load_platforms(as_conf) - - Log.debug("The Experiment name is: {0}", expid) - Log.debug("Sleep: {0}", safetysleeptime) - Log.debug("Default retrials: {0}", retrials) - - Log.info("Starting job submission...") - - pkl_dir = os.path.join( - APIBasicConfig.LOCAL_ROOT_DIR, expid, 'pkl') - job_list = Autosubmit.load_job_list( - expid, as_conf, notransitive=notransitive) - - Log.debug( - "Starting from job list restored from {0} files", pkl_dir) - - Log.debug("Length of the jobs list: {0}", len(job_list)) - - Autosubmit._load_parameters( - as_conf, job_list, submitter.platforms) - - # check the job list script creation - Log.debug("Checking experiment templates...") - - platforms_to_test = set() - for job in job_list.get_job_list(): - if job.platform_name is None: - job.platform_name = hpcarch - # noinspection PyTypeChecker - job.platform = submitter.platforms[job.platform_name.lower( - )] - # noinspection PyTypeChecker - platforms_to_test.add(job.platform) - - job_list.check_scripts(as_conf) - - packages_persistence = JobPackagePersistence(os.path.join(APIBasicConfig.LOCAL_ROOT_DIR, expid, "pkl"), - "job_packages_" + expid) - - if as_conf.get_wrapper_type() != 'none': - os.chmod(os.path.join(APIBasicConfig.LOCAL_ROOT_DIR, - expid, "pkl", "job_packages_" + expid + ".db"), 0o664) - packages = packages_persistence.load() - for (exp_id, package_name, job_name) in packages: - if package_name not in job_list.packages_dict: - job_list.packages_dict[package_name] = [] - job_list.packages_dict[package_name].append( - job_list.get_job_by_name(job_name)) - - for package_name, jobs in list(job_list.packages_dict.items()): - from job.job import WrapperJob - wrapper_job = WrapperJob(package_name, jobs[0].id, Status.SUBMITTED, 0, jobs, - None, - None, jobs[0].platform, as_conf) - job_list.job_package_map[jobs[0].id] = wrapper_job - job_list.update_list(as_conf) - job_list.save() - ######################### - # AUTOSUBMIT - MAIN LOOP - ######################### - # Main loop. Finishing when all jobs have been submitted - while job_list.get_active(): - if Autosubmit.exit: - return 2 - # reload parameters changes - Log.debug("Reloading parameters...") - as_conf.reload() - Autosubmit._load_parameters( - as_conf, job_list, submitter.platforms) - # variables to be updated on the fly - total_jobs = len(job_list.get_job_list()) - Log.info( - "\n\n{0} of {1} jobs remaining ({2})".format(total_jobs - len(job_list.get_completed()), - total_jobs, - time.strftime("%H:%M"))) - safetysleeptime = as_conf.get_safetysleeptime() - Log.debug("Sleep: {0}", safetysleeptime) - default_retrials = as_conf.get_retrials() - Log.debug("Number of retrials: {0}", default_retrials) - - check_wrapper_jobs_sleeptime = as_conf.get_wrapper_check_time() - Log.debug('WRAPPER CHECK TIME = {0}'.format( - check_wrapper_jobs_sleeptime)) - - save = False - - slurm = [] - for platform in platforms_to_test: - list_jobid = "" - completed_joblist = [] - list_prevStatus = [] - queuing_jobs = job_list.get_in_queue_grouped_id( - platform) - for job_id, job in list(queuing_jobs.items()): - if job_list.job_package_map and job_id in job_list.job_package_map: - Log.debug( - 'Checking wrapper job with id ' + str(job_id)) - wrapper_job = job_list.job_package_map[job_id] - check_wrapper = True - if wrapper_job.status == Status.RUNNING: - check_wrapper = True if datetime.timedelta.total_seconds(datetime.datetime.now( - ) - wrapper_job.checked_time) >= check_wrapper_jobs_sleeptime else False - if check_wrapper: - wrapper_job.checked_time = datetime.datetime.now() - platform.check_job(wrapper_job) - Log.info( - 'Wrapper job ' + wrapper_job.name + ' is ' + str(Status.VALUE_TO_KEY[wrapper_job.new_status])) - wrapper_job.check_status( - wrapper_job.new_status) - save = True - else: - Log.info( - "Waiting for wrapper check time: {0}\n", check_wrapper_jobs_sleeptime) - else: - job = job[0] - prev_status = job.status - if job.status == Status.FAILED: - continue - - if platform.type == "slurm": - list_jobid += str(job_id) + ',' - list_prevStatus.append(prev_status) - completed_joblist.append(job) - else: - platform.check_job(job) - if prev_status != job.update_status(as_conf.get_copy_remote_logs() == 'true'): - if as_conf.get_notifications() == 'true': - if Status.VALUE_TO_KEY[job.status] in job.notify_on: - Notifier.notify_status_change(MailNotifier(APIBasicConfig), expid, job.name, - Status.VALUE_TO_KEY[prev_status], - Status.VALUE_TO_KEY[job.status], - as_conf.get_mails_to()) - save = True - - if platform.type == "slurm" and list_jobid != "": - slurm.append( - [platform, list_jobid, list_prevStatus, completed_joblist]) - # END LOOP - for platform_jobs in slurm: - platform = platform_jobs[0] - jobs_to_check = platform_jobs[1] - platform.check_Alljobs( - platform_jobs[3], jobs_to_check, as_conf.get_copy_remote_logs()) - - for j_Indx in range(0, len(platform_jobs[3])): - prev_status = platform_jobs[2][j_Indx] - job = platform_jobs[3][j_Indx] - - if prev_status != job.update_status(as_conf.get_copy_remote_logs() == 'true'): - if as_conf.get_notifications() == 'true': - if Status.VALUE_TO_KEY[job.status] in job.notify_on: - Notifier.notify_status_change(MailNotifier(APIBasicConfig), expid, job.name, - Status.VALUE_TO_KEY[prev_status], - Status.VALUE_TO_KEY[job.status], - as_conf.get_mails_to()) - save = True - - if job_list.update_list(as_conf) or save: - job_list.save() - - if Autosubmit.submit_ready_jobs(as_conf, job_list, platforms_to_test, packages_persistence): - job_list.save() - - if Autosubmit.exit: - return 2 - time.sleep(safetysleeptime) - - Log.info("No more jobs to run.") - if len(job_list.get_failed()) > 0: - Log.info("Some jobs have failed and reached maximum retrials") - return False - else: - Log.result("Run successful") - return True - - except portalocker.AlreadyLocked: - Autosubmit.show_lock_warning(expid) - - except WrongTemplateException: - return False - - @staticmethod - def submit_ready_jobs(as_conf, job_list, platforms_to_test, packages_persistence, inspect=False, - only_wrappers=False): - """ - Gets READY jobs and send them to the platforms if there is available space on the queues - - :param as_conf: autosubmit config object. \n - :type as_conf: AutosubmitConfig Object. \n - :param job_list: JobList as a single entity. \n - :type job_list: JobList() Object. \n - :param platforms_to_test: List of platforms that will be used in the experiment. \n - :type platforms_to_test: Set() of Platform() Object. e.g. EcPlatform(), LsfPlatform(), etc. \n - :return: True if at least one job was submitted, False otherwise - :rtype: bool - """ - save = False - - for platform in platforms_to_test: - Log.debug("\nJobs ready for {1}: {0}", len( - job_list.get_ready(platform)), platform.name) - packages_to_submit, remote_dependencies_dict = JobPackager( - as_conf, platform, job_list).build_packages() - if not inspect: - platform.open_submit_script() - valid_packages_to_submit = [] - for package in packages_to_submit: - try: - if hasattr(package, "name"): - if remote_dependencies_dict and package.name in remote_dependencies_dict['dependencies']: - remote_dependency = remote_dependencies_dict['dependencies'][package.name] - remote_dependency_id = remote_dependencies_dict['name_to_id'][remote_dependency] - package.set_job_dependency(remote_dependency_id) - if not only_wrappers: - try: - package.submit( - as_conf, job_list.parameters, inspect) - valid_packages_to_submit.append(package) - except (IOError, OSError): - # write error file - continue - if only_wrappers or inspect: - for innerJob in package._jobs: - innerJob.status = Status.COMPLETED - - if hasattr(package, "name"): - job_list.packages_dict[package.name] = package.jobs - from job.job import WrapperJob - wrapper_job = WrapperJob(package.name, package.jobs[0].id, Status.READY, 0, - package.jobs, - package._wallclock, package._num_processors, - package.platform, as_conf) - job_list.job_package_map[package.jobs[0].id] = wrapper_job - if remote_dependencies_dict and package.name in remote_dependencies_dict['name_to_id']: - remote_dependencies_dict['name_to_id'][package.name] = package.jobs[0].id - if isinstance(package, JobPackageThread): - packages_persistence.save( - package.name, package.jobs, package._expid, inspect) - save = True - except WrongTemplateException as e: - Log.error( - "Invalid parameter substitution in {0} template", e.job_name) - raise - except Exception: - Log.error( - "{0} submission failed due to Unknown error", platform.name) - raise - - if platform.type == "slurm" and not inspect and not only_wrappers: - try: - save = True - if len(valid_packages_to_submit) > 0: - jobs_id = platform.submit_Script() - if jobs_id is None: - raise BaseException( - "Exiting AS being unable to get jobID") - i = 0 - for package in valid_packages_to_submit: - for job in package.jobs: - job.id = str(jobs_id[i]) - Log.info("{0} submitted", job.name) - job.status = Status.SUBMITTED - job.write_submit_time() - if hasattr(package, "name"): - job_list.packages_dict[package.name] = package.jobs - from job.job import WrapperJob - wrapper_job = WrapperJob(package.name, package.jobs[0].id, Status.SUBMITTED, 0, - package.jobs, - package._wallclock, package._num_processors, - package.platform, as_conf) - job_list.job_package_map[package.jobs[0].id] = wrapper_job - if remote_dependencies_dict and package.name in remote_dependencies_dict[ - 'name_to_id']: - remote_dependencies_dict['name_to_id'][package.name] = package.jobs[0].id - if isinstance(package, JobPackageThread): - packages_persistence.save( - package.name, package.jobs, package._expid, inspect) - i += 1 - - except WrongTemplateException as e: - Log.error( - "Invalid parameter substitution in {0} template", e.job_name) - raise - except Exception: - Log.error("{0} submission failed", platform.name) - raise - - return save - - @staticmethod - def monitor(expid, file_format, lst, filter_chunks, filter_status, filter_section, hide, txt_only=False, - group_by=None, expand=list(), expand_status=list(), hide_groups=False, notransitive=False, check_wrapper=False, txt_logfiles=False): - """ - Plots workflow graph for a given experiment with status of each job coded by node color. - Plot is created in experiment's plot folder with name __