diff --git a/CHANGELOG.md b/CHANGELOG.md
index 978cdfd9d8444d459ad2758a7bde099be58adce2..e491adba0dfcc65c6d4f7d07485dde3038cd7de2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,16 @@
# CHANGELOG
-### Pre-release v4.0.0b3 - Release date: 2023-02-09
+### Pre-release v4.0.0b4 - Release date: 2024-02-23
+
+* The background task that updates the experiment status has been refactored. Now, it keeps the records of all the experiments
+* **Major change:** Removed `experiment_times` and `job_times` tables and background tasks related to them
+* Fixed bug when performance metrics are not calculated when there is only one SIM job
+* Multiple tests have been added
+* Testing module configuration fixtures have been fixed
+* A lot of dead code has been removed
+* Fix the decoding issue on graph coordinates generation on the background task
+
+### Pre-release v4.0.0b3 - Release date: 2024-02-09
* Fix HPC value in the running endpoint
* **Major change:** Updated all route names. Versioning path prefix is included:
diff --git a/README.md b/README.md
index dcc7389e88f7002e72e3c7eca833d7a7ad85c0c9..c33ed7c8b575e7401ed505c4ce3d7c4e7b395d34 100644
--- a/README.md
+++ b/README.md
@@ -70,19 +70,19 @@ The Autosubmit API have some configuration options that can be modified by setti
### Install pytest
```bash
-pip install -U pytest pytest-cov
+pip install -e .[test]
```
### Run tests:
```bash
-pytest tests/*
+pytest
```
### Run tests with coverage HTML report:
```bash
-pytest --cov=autosubmit_api --cov-config=.coveragerc --cov-report=html tests/*
+pytest --cov=autosubmit_api --cov-config=.coveragerc --cov-report=html tests/
```
You will find the report in `htmlcov/index.html`
diff --git a/autosubmit_api/__init__.py b/autosubmit_api/__init__.py
index 8226b2e173ba6816bd3a4ee2f31809d89c138746..31671cf47a0b214d028dfdb2bb1c04f2f7e50be1 100644
--- a/autosubmit_api/__init__.py
+++ b/autosubmit_api/__init__.py
@@ -17,6 +17,6 @@
# You should have received a copy of the GNU General Public License
# along with Autosubmit. If not, see .
-__version__ = "4.0.0b3"
+__version__ = "4.0.0b4"
__author__ = "Luiggi Tenorio, Bruno P. Kinoshita, Cristian Gutiérrez, Julian Berlin, Wilmer Uruchi"
__credits__ = "Barcelona Supercomputing Center"
\ No newline at end of file
diff --git a/autosubmit_api/app.py b/autosubmit_api/app.py
index 5c7acd71a9137ef5e6da9225375633b5d6f6b652..b191135239c77afa832a98abc81dc67fb9b0cbbd 100644
--- a/autosubmit_api/app.py
+++ b/autosubmit_api/app.py
@@ -6,7 +6,7 @@ from flask import Flask
from autosubmit_api.bgtasks.scheduler import create_bind_scheduler
from autosubmit_api.blueprints.v3 import create_v3_blueprint
from autosubmit_api.blueprints.v4 import create_v4_blueprint
-from autosubmit_api.database.extended_db import ExtendedDB
+from autosubmit_api.database import prepare_db
from autosubmit_api.experiment import common_requests as CommonRequests
from autosubmit_api.logger import get_app_logger
from autosubmit_api.config.basicConfig import APIBasicConfig
@@ -66,11 +66,8 @@ def create_app():
)
# Prepare DB
- ext_db = ExtendedDB(
- APIBasicConfig.DB_DIR, APIBasicConfig.DB_FILE, APIBasicConfig.AS_TIMES_DB
- )
- ext_db.prepare_db()
-
+ prepare_db()
+
# Background Scheduler
create_bind_scheduler(app)
diff --git a/autosubmit_api/auth/__init__.py b/autosubmit_api/auth/__init__.py
index b15562c61716c5f2a1c8d5dc3e010db8ff4c5e39..92ee07ab2b97200ac1eec43b0aae89fb6f85fe73 100644
--- a/autosubmit_api/auth/__init__.py
+++ b/autosubmit_api/auth/__init__.py
@@ -3,7 +3,7 @@ from http import HTTPStatus
from flask import request
import jwt
from autosubmit_api.logger import logger
-from autosubmit_api.config import PROTECTION_LEVEL, JWT_ALGORITHM, JWT_SECRET
+from autosubmit_api import config
from enum import IntEnum
@@ -45,12 +45,12 @@ def with_auth_token(
def inner_wrapper(*args, **kwargs):
try:
current_token = request.headers.get("Authorization")
- jwt_token = jwt.decode(current_token, JWT_SECRET, JWT_ALGORITHM)
+ jwt_token = jwt.decode(current_token, config.JWT_SECRET, config.JWT_ALGORITHM)
except Exception as exc:
error_msg = "Unauthorized"
if isinstance(exc, jwt.ExpiredSignatureError):
error_msg = "Expired token"
- auth_level = _parse_protection_level_env(PROTECTION_LEVEL)
+ auth_level = _parse_protection_level_env(config.PROTECTION_LEVEL)
if threshold <= auth_level: # If True, will trigger *_on_fail
if raise_on_fail:
raise AppAuthError(error_msg)
diff --git a/autosubmit_api/autosubmit_legacy/job/job_list.py b/autosubmit_api/autosubmit_legacy/job/job_list.py
index 40cff2ac604c1b9a4eacd85e44910183604d2bfe..c9ad0cba243cd707061de96e6ea092d122f1f8a6 100644
--- a/autosubmit_api/autosubmit_legacy/job/job_list.py
+++ b/autosubmit_api/autosubmit_legacy/job/job_list.py
@@ -31,6 +31,7 @@ from dateutil.relativedelta import *
from bscearth.utils.log import Log
from autosubmit_api.autosubmit_legacy.job.job_utils import SubJob
from autosubmit_api.autosubmit_legacy.job.job_utils import SubJobManager, job_times_to_text
+from autosubmit_api.config.basicConfig import APIBasicConfig
from autosubmit_api.performance.utils import calculate_ASYPD_perjob, calculate_SYPD_perjob
from autosubmit_api.components.jobs import utils as JUtils
from autosubmit_api.monitor.monitor import Monitor
@@ -46,6 +47,8 @@ from autosubmit_api.history.data_classes.job_data import JobData
from typing import List, Dict, Tuple
+from autosubmit_api.persistance.experiment import ExperimentPaths
+
class JobList:
"""
@@ -153,8 +156,8 @@ class JobList:
date_member_repetition = {}
job_name_to_job_title = {}
job_name_to_job = {job.job_name: job for job in job_list}
- path_to_logs = os.path.join(
- BasicConfig.LOCAL_ROOT_DIR, expid, "tmp", "LOG_" + expid)
+ exp_paths = ExperimentPaths(expid)
+ path_to_logs = exp_paths.tmp_log_dir
packages = {job.rowtype for job in job_list if job.rowtype > 2}
package_to_jobs = {package: [
@@ -577,7 +580,7 @@ class JobList:
return ""
@staticmethod
- def get_job_times_collection(basic_config, allJobs, expid, job_to_package=None, package_to_jobs=None, timeseconds=True):
+ def get_job_times_collection(basic_config: APIBasicConfig, allJobs, expid, job_to_package=None, package_to_jobs=None, timeseconds=True):
"""
Gets queuing and running time for the collection of jobs
@@ -590,7 +593,7 @@ class JobList:
conn = DbRequests.create_connection(db_file)
# job_data = None
# Job information from worker database
- job_times = DbRequests.get_times_detail_by_expid(conn, expid)
+ # job_times = dict() # REMOVED: DbRequests.get_times_detail_by_expid(conn, expid)
conn.close()
# Job information from job historic data
# print("Get current job data structure...")
@@ -612,7 +615,7 @@ class JobList:
# print("Start main loop")
for job in allJobs:
job_info = JobList.retrieve_times(
- job.status, job.name, job._tmp_path, make_exception=False, job_times=job_times, seconds=timeseconds, job_data_collection=job_data)
+ job.status, job.name, job._tmp_path, make_exception=False, job_times=None, seconds=timeseconds, job_data_collection=job_data)
# if job_info:
job_name_to_job_info[job.name] = job_info
time_total = (job_info.queue_time +
@@ -860,23 +863,21 @@ class JobList:
"""
monitor = Monitor()
packages = None
+ exp_paths = ExperimentPaths(expid)
try:
- packages = JobPackagePersistence(os.path.join(basic_config.LOCAL_ROOT_DIR, expid, "pkl"),
- "job_packages_" + expid).load(wrapper=False)
+ packages = JobPackagePersistence(exp_paths.job_packages_db).load(wrapper=False)
# if the main table exist but is empty, we try the other one
if not (any(packages.keys()) or any(packages.values())):
Log.info("Wrapper table empty, trying packages.")
- packages = JobPackagePersistence(os.path.join(basic_config.LOCAL_ROOT_DIR, expid, "pkl"),
- "job_packages_" + expid).load(wrapper=True)
+ packages = JobPackagePersistence(exp_paths.job_packages_db).load(wrapper=True)
except Exception as ex:
print("Wrapper table not found, trying packages.")
packages = None
try:
- packages = JobPackagePersistence(os.path.join(basic_config.LOCAL_ROOT_DIR, expid, "pkl"),
- "job_packages_" + expid).load(wrapper=True)
+ packages = JobPackagePersistence(exp_paths.job_packages_db).load(wrapper=True)
except Exception as exp2:
packages = None
pass
diff --git a/autosubmit_api/bgtasks/bgtask.py b/autosubmit_api/bgtasks/bgtask.py
index 30698b62d6db6ec7dc36f4ec10da18a9a9ca31b7..a23aa27ec76b64f0928ba408a96005f7fda34dcd 100644
--- a/autosubmit_api/bgtasks/bgtask.py
+++ b/autosubmit_api/bgtasks/bgtask.py
@@ -1,91 +1,63 @@
from abc import ABC, abstractmethod
-from autosubmit_api.experiment import common_requests
-from autosubmit_api.history.experiment_status_manager import ExperimentStatusManager
+import traceback
+from autosubmit_api.logger import logger
from autosubmit_api.config.basicConfig import APIBasicConfig
-from autosubmit_api.workers.business import populate_times, process_graph_drawings
+from autosubmit_api.workers.business import process_graph_drawings
from autosubmit_api.workers.populate_details.populate import DetailsProcessor
-class BackgroundTask(ABC):
- @staticmethod
+class BackgroundTaskTemplate(ABC):
+ """
+ Interface to define Background Tasks.
+ Do not override the run method.
+ """
+
+ logger = logger
+
+ @classmethod
+ def run(cls):
+ """
+ Not blocking exceptions
+ """
+ try:
+ cls.procedure()
+ except Exception as exc:
+ cls.logger.error(f"Exception on Background Task {cls.id}: {exc}")
+ cls.logger.error(traceback.print_exc())
+
+ @classmethod
@abstractmethod
- def run():
+ def procedure(cls):
raise NotImplementedError
-
+
@property
@abstractmethod
def id(self) -> dict:
raise NotImplementedError
-
+
@property
@abstractmethod
def trigger_options(self) -> dict:
raise NotImplementedError
-
-class PopulateDetailsDB(BackgroundTask):
+
+class PopulateDetailsDB(BackgroundTaskTemplate):
id = "TASK_POPDET"
- trigger_options = {
- "trigger": "interval",
- "hours": 4
- }
+ trigger_options = {"trigger": "interval", "hours": 4}
- @staticmethod
- def run():
+ @classmethod
+ def procedure(cls):
APIBasicConfig.read()
return DetailsProcessor(APIBasicConfig).process()
-
-
-class PopulateQueueRuntimes(BackgroundTask):
- id = "TASK_POPQUE"
- trigger_options = {
- "trigger": "interval",
- "minutes": 3
- }
-
- @staticmethod
- def run():
- """ Process and updates queuing and running times. """
- populate_times.process_completed_times()
-
-
-class PopulateRunningExperiments(BackgroundTask):
- id = "TASK_POPRUNEX"
- trigger_options = {
- "trigger": "interval",
- "minutes": 5
- }
-
- @staticmethod
- def run():
- """
- Updates STATUS of experiments.
- """
- ExperimentStatusManager().update_running_experiments()
-
-
-class VerifyComplete(BackgroundTask):
- id = "TASK_VRFCMPT"
- trigger_options = {
- "trigger": "interval",
- "minutes": 10
- }
- @staticmethod
- def run():
- common_requests.verify_last_completed(1800)
-
-class PopulateGraph(BackgroundTask):
+class PopulateGraph(BackgroundTaskTemplate):
id = "TASK_POPGRPH"
- trigger_options = {
- "trigger": "interval",
- "hours": 24
- }
+ trigger_options = {"trigger": "interval", "hours": 24}
- @staticmethod
- def run():
+ @classmethod
+ def procedure(cls):
"""
Process coordinates of nodes in a graph drawing and saves them.
"""
- process_graph_drawings.process_active_graphs()
\ No newline at end of file
+ process_graph_drawings.process_active_graphs()
diff --git a/autosubmit_api/bgtasks/scheduler.py b/autosubmit_api/bgtasks/scheduler.py
index 2c667c1cbb5f78cc4edb4d6fd04ab8cf8ef0d323..6e890a6453211040c0f78a48b0ddb6c86d3b1ca1 100644
--- a/autosubmit_api/bgtasks/scheduler.py
+++ b/autosubmit_api/bgtasks/scheduler.py
@@ -1,13 +1,11 @@
from typing import List
from flask_apscheduler import APScheduler
from autosubmit_api.bgtasks.bgtask import (
- BackgroundTask,
+ BackgroundTaskTemplate,
PopulateDetailsDB,
- PopulateQueueRuntimes,
- PopulateRunningExperiments,
- VerifyComplete,
PopulateGraph,
)
+from autosubmit_api.bgtasks.tasks.status_updater import StatusUpdater
from autosubmit_api.config import (
DISABLE_BACKGROUND_TASKS,
RUN_BACKGROUND_TASKS_ON_START,
@@ -15,11 +13,9 @@ from autosubmit_api.config import (
from autosubmit_api.logger import logger, with_log_run_times
-REGISTERED_TASKS: List[BackgroundTask] = [
+REGISTERED_TASKS: List[BackgroundTaskTemplate] = [
PopulateDetailsDB,
- PopulateQueueRuntimes,
- PopulateRunningExperiments,
- VerifyComplete,
+ StatusUpdater,
PopulateGraph,
]
diff --git a/autosubmit_api/history/platform_monitor/__init__.py b/autosubmit_api/bgtasks/tasks/__init__.py
similarity index 100%
rename from autosubmit_api/history/platform_monitor/__init__.py
rename to autosubmit_api/bgtasks/tasks/__init__.py
diff --git a/autosubmit_api/bgtasks/tasks/status_updater.py b/autosubmit_api/bgtasks/tasks/status_updater.py
new file mode 100644
index 0000000000000000000000000000000000000000..186be6fccb18c783a2c3c9079d9389a66b3b23c8
--- /dev/null
+++ b/autosubmit_api/bgtasks/tasks/status_updater.py
@@ -0,0 +1,119 @@
+from datetime import datetime
+import os
+import time
+from typing import List
+
+from sqlalchemy import select
+from autosubmit_api.bgtasks.bgtask import BackgroundTaskTemplate
+from autosubmit_api.database import tables
+from autosubmit_api.database.common import (
+ create_autosubmit_db_engine,
+ create_as_times_db_engine,
+ create_main_db_conn,
+)
+from autosubmit_api.database.models import ExperimentModel
+from autosubmit_api.experiment.common_requests import _is_exp_running
+from autosubmit_api.history.database_managers.database_models import RunningStatus
+from autosubmit_api.persistance.experiment import ExperimentPaths
+
+
+class StatusUpdater(BackgroundTaskTemplate):
+ id = "TASK_STTSUPDTR"
+ trigger_options = {"trigger": "interval", "minutes": 5}
+
+ @classmethod
+ def _clear_missing_experiments(cls):
+ """
+ Clears the experiments that are not in the experiments table
+ """
+ with create_main_db_conn() as conn:
+ try:
+ del_stmnt = tables.experiment_status_table.delete().where(
+ tables.experiment_status_table.c.exp_id.not_in(
+ select(tables.experiment_table.c.id)
+ )
+ )
+ conn.execute(del_stmnt)
+ conn.commit()
+ except Exception as exc:
+ conn.rollback()
+ cls.logger.error(
+ f"[{cls.id}] Error while clearing missing experiments status: {exc}"
+ )
+
+ @classmethod
+ def _get_experiments(cls) -> List[ExperimentModel]:
+ """
+ Get the experiments list
+ """
+ with create_autosubmit_db_engine().connect() as conn:
+ query_result = conn.execute(tables.experiment_table.select()).all()
+ return [ExperimentModel.model_validate(row._mapping) for row in query_result]
+
+ @classmethod
+ def _check_exp_running(cls, expid: str) -> bool:
+ """
+ Decide if the experiment is running
+ """
+ MAX_PKL_AGE = 600 # 10 minutes
+ MAX_PKL_AGE_EXHAUSTIVE = 3600 # 1 hour
+
+ is_running = False
+ try:
+ pkl_path = ExperimentPaths(expid).job_list_pkl
+ pkl_age = int(time.time()) - int(os.stat(pkl_path).st_mtime)
+
+ if pkl_age < MAX_PKL_AGE: # First running check
+ is_running = True
+ elif pkl_age < MAX_PKL_AGE_EXHAUSTIVE: # Exhaustive check
+ _, _, _flag, _, _ = _is_exp_running(expid) # Exhaustive validation
+ if _flag:
+ is_running = True
+ except Exception as exc:
+ cls.logger.error(
+ f"[{cls.id}] Error while checking experiment {expid}: {exc}"
+ )
+
+ return is_running
+
+ @classmethod
+ def _update_experiment_status(cls, experiment: ExperimentModel, is_running: bool):
+ with create_as_times_db_engine().connect() as conn:
+ try:
+ del_stmnt = tables.experiment_status_table.delete().where(
+ tables.experiment_status_table.c.exp_id == experiment.id
+ )
+ ins_stmnt = tables.experiment_status_table.insert().values(
+ exp_id=experiment.id,
+ name=experiment.name,
+ status=(
+ RunningStatus.RUNNING
+ if is_running
+ else RunningStatus.NOT_RUNNING
+ ),
+ seconds_diff=0,
+ modified=datetime.now().isoformat(sep="-", timespec="seconds"),
+ )
+ conn.execute(del_stmnt)
+ conn.execute(ins_stmnt)
+ conn.commit()
+ except Exception as exc:
+ conn.rollback()
+ cls.logger.error(
+ f"[{cls.id}] Error while doing database operations on experiment {experiment.name}: {exc}"
+ )
+
+ @classmethod
+ def procedure(cls):
+ """
+ Updates STATUS of experiments.
+ """
+ cls._clear_missing_experiments()
+
+ # Read experiments table
+ exp_list = cls._get_experiments()
+
+ # Check every experiment status & update
+ for experiment in exp_list:
+ is_running = cls._check_exp_running(experiment.name)
+ cls._update_experiment_status(experiment, is_running)
diff --git a/autosubmit_api/builders/experiment_builder.py b/autosubmit_api/builders/experiment_builder.py
index 5a3b5a60afab4de3c8302778356b22e1ede99c8b..7bf5c48d17810eb61f78ed6285ea3c8147d86690 100644
--- a/autosubmit_api/builders/experiment_builder.py
+++ b/autosubmit_api/builders/experiment_builder.py
@@ -4,16 +4,16 @@ from autosubmit_api.builders.configuration_facade_builder import (
ConfigurationFacadeDirector,
)
from autosubmit_api.database import tables
-from autosubmit_api.database.common import create_main_db_conn
-from autosubmit_api.database.models import Experiment
+from autosubmit_api.database.common import create_autosubmit_db_engine, create_main_db_conn
+from autosubmit_api.database.models import ExperimentModel
class ExperimentBuilder(BaseBuilder):
def produce_base_from_dict(self, obj: dict):
- self._product = Experiment.model_validate(obj)
+ self._product = ExperimentModel.model_validate(obj)
def produce_base(self, expid):
- with create_main_db_conn() as conn:
+ with create_autosubmit_db_engine().connect() as conn:
result = conn.execute(
tables.experiment_table.select().where(
tables.experiment_table.c.name == expid
@@ -21,7 +21,7 @@ class ExperimentBuilder(BaseBuilder):
).one()
# Set new product
- self._product = Experiment(
+ self._product = ExperimentModel(
id=result.id,
name=result.name,
description=result.description,
@@ -30,12 +30,12 @@ class ExperimentBuilder(BaseBuilder):
def produce_details(self):
exp_id = self._product.id
- with create_main_db_conn() as conn:
+ with create_autosubmit_db_engine().connect()() as conn:
result = conn.execute(
tables.details_table.select().where(
tables.details_table.c.exp_id == exp_id
)
- ).one_or_none
+ ).one_or_none()
# Set details props
if result:
@@ -63,5 +63,5 @@ class ExperimentBuilder(BaseBuilder):
)
@property
- def product(self) -> Experiment:
+ def product(self) -> ExperimentModel:
return super().product
\ No newline at end of file
diff --git a/autosubmit_api/common/utils.py b/autosubmit_api/common/utils.py
index 7056076fa7b6727dcd3c0d2d1045c57e0b18de4f..e0ce68f253f0f482ecd138d4daa85d49dca49b71 100644
--- a/autosubmit_api/common/utils.py
+++ b/autosubmit_api/common/utils.py
@@ -71,12 +71,12 @@ def parse_number_processors(processors_str):
except:
return 1
-def get_jobs_with_no_outliers(jobs):
+def get_jobs_with_no_outliers(jobs: List):
""" Detects outliers and removes them from the returned list """
new_list = []
data_run_times = [job.run_time for job in jobs]
# print(data_run_times)
- if len(data_run_times) == 0:
+ if len(data_run_times) <= 1:
return jobs
mean = statistics.mean(data_run_times)
diff --git a/autosubmit_api/components/experiment/configuration_facade.py b/autosubmit_api/components/experiment/configuration_facade.py
index 47762a2432313781d542a2244fb7aac884141cea..8ff5fade857778cc1473b46c5c1f03ef70d57587 100644
--- a/autosubmit_api/components/experiment/configuration_facade.py
+++ b/autosubmit_api/components/experiment/configuration_facade.py
@@ -11,6 +11,8 @@ from abc import ABCMeta, abstractmethod
from autosubmit_api.common.utils import JobSection, parse_number_processors, timestamp_to_datetime_format, datechunk_to_year
from typing import List
+from autosubmit_api.persistance.experiment import ExperimentPaths
+
class ProjectType:
GIT = "git"
SVN = "svn"
@@ -40,11 +42,12 @@ class ConfigurationFacade(metaclass=ABCMeta):
def _process_basic_config(self):
# type: () -> None
- self.pkl_filename = "job_list_{0}.pkl".format(self.expid)
- self.experiment_path = os.path.join(self.basic_configuration.LOCAL_ROOT_DIR, self.expid)
- self.pkl_path = os.path.join(self.basic_configuration.LOCAL_ROOT_DIR, self.expid, "pkl", self.pkl_filename)
- self.tmp_path = os.path.join(self.basic_configuration.LOCAL_ROOT_DIR, self.expid, self.basic_configuration.LOCAL_TMP_DIR)
- self.log_path = os.path.join(self.basic_configuration.LOCAL_ROOT_DIR, self.expid, "tmp", "LOG_{0}".format(self.expid))
+ exp_paths = ExperimentPaths(self.expid)
+ self.pkl_filename = os.path.basename(exp_paths.job_list_pkl)
+ self.experiment_path = exp_paths.exp_dir
+ self.pkl_path = exp_paths.job_list_pkl
+ self.tmp_path = exp_paths.tmp_dir
+ self.log_path = exp_paths.tmp_log_dir
self.structures_path = self.basic_configuration.STRUCTURES_DIR
if not os.path.exists(self.experiment_path): raise IOError("Experiment folder {0} not found".format(self.experiment_path))
if not os.path.exists(self.pkl_path): raise IOError("Required file {0} not found.".format(self.pkl_path))
diff --git a/autosubmit_api/components/jobs/test.py b/autosubmit_api/components/jobs/test.py
deleted file mode 100644
index dc64a7531ff11c6e2103d15abf4a7694af09badc..0000000000000000000000000000000000000000
--- a/autosubmit_api/components/jobs/test.py
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env python
-
-import unittest
-from components.jobs.joblist_loader import JobListLoader
-
-
-class TestJobListHelper(unittest.TestCase):
- def setUp(self):
- pass
-
- def get_joblist_helper(self):
- pass
-
-class TestJobListLoader(unittest.TestCase):
- def setUp(self):
- pass
-
- def tearDown(self):
- pass
-
- def test_load(self):
- pass
- # loader = JobListLoader("a29z")
- # loader.load_jobs()
- # self.assertTrue(len(loader.jobs) > 0)
- # for job in loader.jobs:
- # job.do_print()
-
- # def test_loader(self):
- # tree =
-
- # def test_load_out_err_files(self):
-
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/autosubmit_api/config/basicConfig.py b/autosubmit_api/config/basicConfig.py
index fdb9e57013a7bc2281effaed95112a221afd9a07..95acbff22e737fec77104603dd8bf939fa80bbdf 100644
--- a/autosubmit_api/config/basicConfig.py
+++ b/autosubmit_api/config/basicConfig.py
@@ -27,8 +27,8 @@ class APIBasicConfig(BasicConfig):
Extended class to manage configuration for Autosubmit path, database and default values for new experiments in the Autosubmit API
"""
- GRAPHDATA_DIR = os.path.join(os.path.expanduser('~'), 'autosubmit', 'as_metadata', 'graph')
- FILE_STATUS_DIR = os.path.join(os.path.expanduser('~'), 'autosubmit', 'as_metadata', 'test')
+ GRAPHDATA_DIR = os.path.join(os.path.expanduser('~'), 'autosubmit', 'metadata', 'graph')
+ FILE_STATUS_DIR = os.path.join(os.path.expanduser('~'), 'autosubmit', 'metadata', 'test')
FILE_STATUS_DB = 'status.db'
ALLOWED_CLIENTS = set([])
@@ -45,6 +45,8 @@ class APIBasicConfig(BasicConfig):
if parser.has_option('graph', 'path'):
APIBasicConfig.GRAPHDATA_DIR = parser.get('graph', 'path')
+ else:
+ APIBasicConfig.GRAPHDATA_DIR = os.path.join(APIBasicConfig.LOCAL_ROOT_DIR, 'metadata', 'graph')
if parser.has_option('statusdb', 'path'):
APIBasicConfig.FILE_STATUS_DIR = parser.get('statusdb', 'path')
if parser.has_option('statusdb', 'filename'):
diff --git a/autosubmit_api/config/confConfigStrategy.py b/autosubmit_api/config/confConfigStrategy.py
index 9ccd8ee22b786fde5627ac5de3e8f5ebaa786e30..2bbadb2e438f81b17188e2dc353f030023dc039c 100644
--- a/autosubmit_api/config/confConfigStrategy.py
+++ b/autosubmit_api/config/confConfigStrategy.py
@@ -647,7 +647,7 @@ class confConfigStrategy(IConfigStrategy):
if self._proj_parser_file == '':
self._proj_parser = None
else:
- self._proj_parser = AutosubmitConfig.get_parser(
+ self._proj_parser = confConfigStrategy.get_parser(
self.parser_factory, self._proj_parser_file)
return True
except Exception as e:
diff --git a/autosubmit_api/config/files/autosubmit.conf b/autosubmit_api/config/files/autosubmit.conf
deleted file mode 100644
index 90770b8d0a9611157495ea14d9dfa8518faae208..0000000000000000000000000000000000000000
--- a/autosubmit_api/config/files/autosubmit.conf
+++ /dev/null
@@ -1,42 +0,0 @@
-[config]
-# Experiment identifier
-# No need to change
-EXPID =
-# No need to change.
-# Autosubmit version identifier
-AUTOSUBMIT_VERSION =
-# Default maximum number of jobs to be waiting in any platform
-# Default = 3
-MAXWAITINGJOBS = 3
-# Default maximum number of jobs to be running at the same time at any platform
-# Default = 6
-TOTALJOBS = 6
-# Time (seconds) between connections to the HPC queue scheduler to poll already submitted jobs status
-# Default = 10
-SAFETYSLEEPTIME = 10
-# Number of retrials if a job fails. Can ve override at job level
-# Default = 0
-RETRIALS = 0
-
-[mail]
-# Enable mail notifications
-# Default = False
-NOTIFICATIONS = False
-# Mail address where notifications will be received
-TO =
-
-[communications]
-# Communications library used to connect with platforms: paramiko or saga.
-# Default = paramiko
-API = paramiko
-
-[storage]
-# Defines the way of storing the progress of the experiment. The available options are:
-# A PICKLE file (pkl) or an SQLite database (db). Default = pkl
-TYPE = pkl
-# Defines if the remote logs will be copied to the local platform. Default = True.
-COPY_REMOTE_LOGS = True
-
-[migrate]
-# Changes experiment files owner.
-TO_USER =
diff --git a/autosubmit_api/config/files/expdef.conf b/autosubmit_api/config/files/expdef.conf
deleted file mode 100644
index 96e2b6fd32c5c2e5153fba6c82367f2677f7210f..0000000000000000000000000000000000000000
--- a/autosubmit_api/config/files/expdef.conf
+++ /dev/null
@@ -1,77 +0,0 @@
-[DEFAULT]
-# Experiment identifier
-# No need to change
-EXPID =
-# HPC name.
-# No need to change
-HPCARCH =
-
-[experiment]
-# Supply the list of start dates. Available formats: YYYYMMDD YYYYMMDDhh YYYYMMDDhhmm
-# You can also use an abbreviated syntax for multiple dates with common parts: 200001[01 15] <=> 20000101 20000115
-# 200001[01-04] <=> 20000101 20000102 20000103 20000104
-# DATELIST = 19600101 19650101 19700101
-# DATELIST = 1960[0101 0201 0301]
-# DATELIST = 19[60-65]
-DATELIST =
-# Supply the list of members. Format fcX
-# You can also use an abreviated syntax for multiple members: fc[0 1 2] <=> fc0 fc1 fc2
-# fc[0-2] <=> fc0 fc1 fc2
-# MEMBERS = fc0 fc1 fc2 fc3 fc4
-# MEMBERS = fc[0-4]
-MEMBERS =
-# Chunk size unit. STRING = hour, day, month, year
-CHUNKSIZEUNIT = month
-# Chunk size. NUMERIC = 4, 6, 12
-CHUNKSIZE =
-# Total number of chunks in experiment. NUMERIC = 30, 15, 10
-NUMCHUNKS =
-# Initial chunk of the experiment. Optional. DEFAULT = 1
-CHUNKINI =
-# Calendar used. LIST: standard, noleap
-CALENDAR = standard
-
-[project]
-# Select project type. STRING = git, svn, local, none
-# If PROJECT_TYPE is set to none, Autosubmit self-contained dummy templates will be used
-PROJECT_TYPE =
-# Destination folder name for project. type = STRING, default = leave empty,
-PROJECT_DESTINATION =
-
-# If PROJECT_TYPE is not git, no need to change
-[git]
-# Repository URL STRING = 'https://github.com/torvalds/linux.git'
-PROJECT_ORIGIN =
-# Select branch or tag, STRING, default = 'master', help = {'master' (default), 'develop', 'v3.1b', ...}
-PROJECT_BRANCH =
-# type = STRING, default = leave empty, help = if model branch is a TAG leave empty
-PROJECT_COMMIT =
-# type = STRING, default = leave empty and will load all submodules, help = loadThisSubmodule alsoloadthis anotherLoad ...
-PROJECT_SUBMODULES =
-# If PROJECT_TYPE is not svn, no need to change
-[svn]
-# type = STRING, help = 'https://svn.ec-earth.org/ecearth3'
-PROJECT_URL =
-# Select revision number. NUMERIC = 1778
-PROJECT_REVISION =
-
-# If PROJECT_TYPE is not local, no need to change
-[local]
-# type = STRING, help = /foo/bar/ecearth
-PROJECT_PATH =
-
-# If PROJECT_TYPE is none, no need to change
-[project_files]
-# Where is PROJECT CONFIGURATION file location relative to project root path
-FILE_PROJECT_CONF =
-# Where is JOBS CONFIGURATION file location relative to project root path
-FILE_JOBS_CONF =
-# Default job scripts type in the project. type = STRING, default = bash, supported = 'bash', 'python' or 'r'
-JOB_SCRIPTS_TYPE =
-
-[rerun]
-# Is a rerun or not? [Default: Do set FALSE]. BOOLEAN = TRUE, FALSE
-RERUN = FALSE
-# If RERUN = TRUE then supply the list of chunks to rerun
-# LIST = [ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]
-CHUNKLIST =
\ No newline at end of file
diff --git a/autosubmit_api/config/files/jobs.conf b/autosubmit_api/config/files/jobs.conf
deleted file mode 100644
index eaf192ce6b081598bb98537b9079061013f74d8f..0000000000000000000000000000000000000000
--- a/autosubmit_api/config/files/jobs.conf
+++ /dev/null
@@ -1,96 +0,0 @@
-# Example job with all options specified
-
-## Job name
-# [JOBNAME]
-## Script to execute. If not specified, job will be omitted from workflow.
-## Path relative to the project directory
-# FILE =
-## Platform to execute the job. If not specified, defaults to HPCARCH in expdef file.
-## LOCAL is always defined and represents the current machine
-# PLATFORM =
-## Queue to add the job to. If not specified, uses PLATFORM default.
-# QUEUE =
-## Defines dependencies from job as a list of parents jobs separated by spaces.
-## Dependencies to jobs in previous chunk, member o startdate, use -(DISTANCE)
-# DEPENDENCIES = INI SIM-1 CLEAN-2
-## Define if jobs runs once, once per stardate, once per member or once per chunk. Options: once, date, member, chunk.
-## If not specified, defaults to once
-# RUNNING = once
-## Specifies that job has only to be run after X dates, members or chunk. A job will always be created for the last
-## If not specified, defaults to 1
-# FREQUENCY = 3
-## Specifies if a job with FREQUENCY > 1 has only to wait for all the jobs in the previous chunks on its period or just
-## for the ones in the chunk it is going to execute
-## If not specified, defaults to True
-# WAIT = False
-## Defines if job is only to be executed in reruns. If not specified, defaults to false.
-# RERUN_ONLY = False
-## Defines jobs needed to be rerun if this job is going to be rerunned
-# RERUN_DEPENDENCIES = RERUN INI LOCAL_SETUP REMOTE_SETUP TRANSFER
-## Wallclock to be submitted to the HPC queue in format HH:MM. If not specified, defaults to empty.
-# WALLCLOCK = 00:05
-## Processors number to be submitted to the HPC. If not specified, defaults to 1.
-# PROCESSORS = 1
-## Threads number to be submitted to the HPC. If not specified, defaults to 1.
-# THREADS = 1
-## Tasks number (number of processes per node) to be submitted to the HPC. If not specified, defaults to empty.
-# TASKS = 16
-## Memory requirements for the job in MB. Optional. If not specified, then not defined for the scheduler.
-# MEMORY = 4096
-## Memory per task requirements for the job in MB. Optional. If not specified, then not defined for the scheduler.
-# MEMORY_PER_TASK = 1024
-## Scratch free space requirements for the job in percentage (%). If not specified, it won't be defined on the template.
-# SCRATCH_FREE_SPACE = 10
-## Number of retrials if a job fails. If not specified, defaults to the value given on experiment's autosubmit.conf
-# RETRIALS = 4
-## Some jobs can not be checked before running previous jobs. Set this option to false if that is the case
-# CHECK = False
-## Select the interpreter that will run the job. Options: bash, python, r Default: bash
-# TYPE = bash
-## Synchronize a chunk job with its dependency chunks at a 'date' or 'member' level
-# SYNCHRONIZE = date | member
-## Optional. Custom directives for the resource manager of the platform used for that job.
-## Put as many as you wish in json formatted array.
-# CUSTOM_DIRECTIVE = ["#PBS -v myvar=value, "#PBS -v othervar=value"]
-
-[LOCAL_SETUP]
-FILE = LOCAL_SETUP.sh
-PLATFORM = LOCAL
-
-[REMOTE_SETUP]
-FILE = REMOTE_SETUP.sh
-DEPENDENCIES = LOCAL_SETUP
-WALLCLOCK = 00:05
-
-[INI]
-FILE = INI.sh
-DEPENDENCIES = REMOTE_SETUP
-RUNNING = member
-WALLCLOCK = 00:05
-
-[SIM]
-FILE = SIM.sh
-DEPENDENCIES = INI SIM-1 CLEAN-2
-RUNNING = chunk
-WALLCLOCK = 00:05
-PROCESSORS = 2
-THREADS = 1
-TASKS = 1
-
-[POST]
-FILE = POST.sh
-DEPENDENCIES = SIM
-RUNNING = chunk
-WALLCLOCK = 00:05
-
-[CLEAN]
-FILE = CLEAN.sh
-DEPENDENCIES = POST
-RUNNING = chunk
-WALLCLOCK = 00:05
-
-[TRANSFER]
-FILE = TRANSFER.sh
-PLATFORM = LOCAL
-DEPENDENCIES = CLEAN
-RUNNING = member
diff --git a/autosubmit_api/config/files/platforms.conf b/autosubmit_api/config/files/platforms.conf
deleted file mode 100644
index a06f59dd0c0a234b8027d5fe180590487bdb8227..0000000000000000000000000000000000000000
--- a/autosubmit_api/config/files/platforms.conf
+++ /dev/null
@@ -1,55 +0,0 @@
-# Example platform with all options specified
-
-## Platform name
-# [PLATFORM]
-## Queue type. Options: PBS, SGE, PS, LSF, ecaccess, SLURM. Required
-# TYPE =
-## Version of queue manager to use. Needed only in PBS (options: 10, 11, 12) and ecaccess (options: pbs, loadleveler)
-# VERSION =
-## Hostname of the HPC. Required
-# HOST =
-## Project for the machine scheduler. Required
-# PROJECT =
-## Budget account for the machine scheduler. If omitted, takes the value defined in PROJECT
-# BUDGET =
-## Option to add project name to host. This is required for some HPCs.
-# ADD_PROJECT_TO_HOST = False
-## User for the machine scheduler. Required
-# USER =
-## Optional. If given, Autosubmit will change owner of files in given platform when using migrate_exp.
-# USER_TO =
-## Path to the scratch directory for the machine. Required.
-# SCRATCH_DIR = /scratch
-## Path to the machine's temporary directory for migrate purposes.
-# TEMP_DIR = /tmp
-## If true, Autosubmit test command can use this queue as a main queue. Defaults to False
-# TEST_SUITE = False
-## If given, Autosubmit will add jobs to the given queue. Required for some platforms.
-# QUEUE =
-## Optional. If given, Autosubmit will submit the serial jobs with the exclusivity directive.
-# EXCLUSIVITY =
-## Optional. If specified, autosubmit will run jobs with only one processor in the specified platform.
-# SERIAL_PLATFORM = SERIAL_PLATFORM_NAME
-## Optional. If specified, autosubmit will run jobs with only one processor in the specified queue.
-## Autosubmit will ignore this configuration if SERIAL_PLATFORM is provided
-# SERIAL_QUEUE = SERIAL_QUEUE_NAME
-## Optional. Default number of processors per node to be used in jobs
-# PROCESSORS_PER_NODE =
-## Optional. Integer. Scratch free space requirements for the platform in percentage (%).
-## If not specified, it won't be defined on the template.
-# SCRATCH_FREE_SPACE =
-## Optional. Integer. Default Maximum number of jobs to be waiting in any platform queue
-## Default = 3
-# MAX_WAITING_JOBS =
-## Optional. Integer. Default maximum number of jobs to be running at the same time at any platform
-## Default = 6
-# TOTAL_JOBS =
-## Max wallclock per job submitted to the HPC queue in format HH:MM. If not specified, defaults to empty.
-## Optional. Required for wrappers.
-# MAX_WALLCLOCK = 72:00
-## Max processors number per job submitted to the HPC. If not specified, defaults to empty.
-## Optional. Required for wrappers.
-# MAX_PROCESSORS = 1
-## Optional. Custom directives for the resource manager of the platform used.
-## Put as many as you wish in a json formatted array.
-# CUSTOM_DIRECTIVE = ["#PBS -v myvar=value, "#PBS -v othervar=value"]
diff --git a/autosubmit_api/database/__init__.py b/autosubmit_api/database/__init__.py
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..e2c02e22ffbd99e3569df2bf6ffeae33f39325d4 100644
--- a/autosubmit_api/database/__init__.py
+++ b/autosubmit_api/database/__init__.py
@@ -0,0 +1,19 @@
+from sqlalchemy import text
+from autosubmit_api.database.common import (
+ create_as_times_db_engine,
+ create_autosubmit_db_engine,
+)
+from autosubmit_api.database.tables import experiment_status_table, details_table
+
+
+def prepare_db():
+ with create_as_times_db_engine().connect() as conn:
+ experiment_status_table.create(conn, checkfirst=True)
+
+ with create_autosubmit_db_engine().connect() as conn:
+ details_table.create(conn, checkfirst=True)
+
+ view_name = "listexp"
+ view_from = "select id,name,user,created,model,branch,hpc,description from experiment left join details on experiment.id = details.exp_id"
+ new_view_stmnt = f"CREATE VIEW IF NOT EXISTS {view_name} as {view_from}"
+ conn.execute(text(new_view_stmnt))
diff --git a/autosubmit_api/database/common.py b/autosubmit_api/database/common.py
index c39b76ddb957058c931177958edf764a2707a16f..6891df5403feed41ba4a06ac72b50a3cd10274a7 100644
--- a/autosubmit_api/database/common.py
+++ b/autosubmit_api/database/common.py
@@ -1,27 +1,63 @@
import os
from typing import Any
-from sqlalchemy import Connection, Select, create_engine, select, text, func
+from sqlalchemy import Connection, Engine, Select, create_engine, select, text, func
+from autosubmit_api.builders import BaseBuilder
from autosubmit_api.logger import logger
from autosubmit_api.config.basicConfig import APIBasicConfig
-def create_main_db_conn():
- APIBasicConfig.read()
- autosubmit_db_path = os.path.abspath(APIBasicConfig.DB_PATH)
- as_times_db_path = os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB)
- engine = create_engine("sqlite://")
- conn = engine.connect()
- conn.execute(text(f"attach database '{autosubmit_db_path}' as autosubmit;"))
- conn.execute(text(f"attach database '{as_times_db_path}' as as_times;"))
- return conn
+class AttachedDatabaseConnBuilder(BaseBuilder):
+ """
+ SQLite utility to build attached databases.
+ """
+
+ def __init__(self) -> None:
+ super().__init__(False)
+ APIBasicConfig.read()
+ self.engine = create_engine("sqlite://")
+ self._product = self.engine.connect()
+
+ def attach_db(self, path: str, name: str):
+ self._product.execute(text(f"attach database '{path}' as {name};"))
+ def attach_autosubmit_db(self):
+ autosubmit_db_path = os.path.abspath(APIBasicConfig.DB_PATH)
+ self.attach_db(autosubmit_db_path, "autosubmit")
-def create_autosubmit_db_engine():
+ def attach_as_times_db(self):
+ as_times_db_path = os.path.join(
+ APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB
+ )
+ self.attach_db(as_times_db_path, "as_times")
+
+ @property
+ def product(self) -> Connection:
+ return super().product
+
+
+def create_main_db_conn() -> Connection:
+ """
+ Connection with the autosubmit and as_times DDBB.
+ """
+ builder = AttachedDatabaseConnBuilder()
+ builder.attach_autosubmit_db()
+ builder.attach_as_times_db()
+
+ return builder.product
+
+
+def create_autosubmit_db_engine() -> Engine:
+ """
+ Create an engine for the autosubmit DDBB. Usually named autosubmit.db
+ """
APIBasicConfig.read()
return create_engine(f"sqlite:///{ os.path.abspath(APIBasicConfig.DB_PATH)}")
-def create_as_times_db_engine():
+def create_as_times_db_engine() -> Engine:
+ """
+ Create an engine for the AS_TIMES DDBB. Usually named as_times.db
+ """
APIBasicConfig.read()
db_path = os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB)
return create_engine(f"sqlite:///{ os.path.abspath(db_path)}")
diff --git a/autosubmit_api/database/data/autosubmit.sql b/autosubmit_api/database/data/autosubmit.sql
deleted file mode 100644
index f1a1332c8fbff9963c0abb9159bace80ce055797..0000000000000000000000000000000000000000
--- a/autosubmit_api/database/data/autosubmit.sql
+++ /dev/null
@@ -1,8 +0,0 @@
-CREATE TABLE experiment(
- id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
- name VARCHAR NOT NULL,
- description VARCHAR NOT NULL,
- autosubmit_version VARCHAR);
-CREATE TABLE db_version(
- version INTEGER NOT NULL);
-INSERT INTO db_version (version) VALUES (1);
\ No newline at end of file
diff --git a/autosubmit_api/database/db_common.py b/autosubmit_api/database/db_common.py
index d9f714bca27e4860d1ddc25612df6c18f078e269..961bba69346eac7098dea4aca257d0f19c7b8eb3 100644
--- a/autosubmit_api/database/db_common.py
+++ b/autosubmit_api/database/db_common.py
@@ -22,7 +22,7 @@ Module containing functions to manage autosubmit's database.
"""
import os
from sqlite3 import Connection, Cursor
-import pysqlite3 as sqlite3
+import sqlite3
from bscearth.utils.log import Log
from autosubmit_api.config.basicConfig import APIBasicConfig
@@ -96,7 +96,7 @@ def open_conn(check_version=True) -> Tuple[Connection, Cursor]:
return conn, cursor
-def close_conn(conn, cursor):
+def close_conn(conn: Connection, cursor):
"""
Commits changes and close connection to database
@@ -231,7 +231,7 @@ def search_experiment_by_id(query, exp_type=None, only_active=None, owner=None):
experiment_times = dict()
if len(table) > 0:
experiment_status = DbRequests.get_experiment_status()
- experiment_times = DbRequests.get_experiment_times()
+ # REMOVED: experiment_times = DbRequests.get_experiment_times()
for row in table:
expid = str(row[1])
@@ -268,7 +268,7 @@ def search_experiment_by_id(query, exp_type=None, only_active=None, owner=None):
try:
current_run = ExperimentHistoryDirector(ExperimentHistoryBuilder(expid)).build_reader_experiment_history().manager.get_experiment_run_dc_with_max_id()
- if current_run and current_run.total > 0 and (current_run.total == total or current_run.modified_timestamp > last_modified_timestamp):
+ if current_run and current_run.total > 0:
completed = current_run.completed
total = current_run.total
submitted = current_run.submitted
@@ -313,7 +313,7 @@ def get_current_running_exp():
experiment_status = dict()
experiment_times = dict()
experiment_status = DbRequests.get_experiment_status()
- experiment_times = DbRequests.get_experiment_times()
+ # REMOVED: experiment_times = DbRequests.get_experiment_times()
for row in table:
expid = str(row[1])
status = "NOT RUNNING"
@@ -353,7 +353,7 @@ def get_current_running_exp():
# Try to retrieve experiment_run data
try:
current_run = ExperimentHistoryDirector(ExperimentHistoryBuilder(expid)).build_reader_experiment_history().manager.get_experiment_run_dc_with_max_id()
- if current_run and current_run.total > 0 and (current_run.total == total or current_run.modified_timestamp > last_modified_timestamp):
+ if current_run and current_run.total > 0:
completed = current_run.completed
total = current_run.total
submitted = current_run.submitted
diff --git a/autosubmit_api/database/db_jobdata.py b/autosubmit_api/database/db_jobdata.py
index d9fb4e17a0a9e01d92cadf98e259f60f905ddd83..25a70ad81b4495657b388e5e5303206c67613ff0 100644
--- a/autosubmit_api/database/db_jobdata.py
+++ b/autosubmit_api/database/db_jobdata.py
@@ -21,7 +21,7 @@ import os
import time
import textwrap
import traceback
-import pysqlite3 as sqlite3
+import sqlite3
import collections
import portalocker
from datetime import datetime, timedelta
@@ -38,6 +38,8 @@ from autosubmit_api.common.utils import get_jobs_with_no_outliers, Status, datec
# import autosubmitAPIwu.experiment.common_db_requests as DbRequests
from bscearth.utils.date import Log
+from autosubmit_api.persistance.experiment import ExperimentPaths
+
# Version 15 includes out err MaxRSS AveRSS and rowstatus
CURRENT_DB_VERSION = 15 # Used to be 10 or 0
@@ -532,9 +534,9 @@ class ExperimentGraphDrawing(MainDataBase):
MainDataBase.__init__(self, expid)
APIBasicConfig.read()
self.expid = expid
+ exp_paths = ExperimentPaths(expid)
self.folder_path = APIBasicConfig.LOCAL_ROOT_DIR
- self.database_path = os.path.join(
- self.folder_path, "as_metadata", "graph" , "graph_data_" + str(expid) + ".db")
+ self.database_path = exp_paths.graph_data_db
self.create_table_query = textwrap.dedent(
'''CREATE TABLE
IF NOT EXISTS experiment_graph_draw (
@@ -620,9 +622,10 @@ class ExperimentGraphDrawing(MainDataBase):
result = graph.create('dot', format="plain")
for u in result.split(b"\n"):
splitList = u.split(b" ")
- if len(splitList) > 1 and splitList[0] == "node":
- self.coordinates.append((splitList[1], int(
- float(splitList[2]) * 90), int(float(splitList[3]) * -90)))
+ if len(splitList) > 1 and splitList[0].decode() == "node":
+
+ self.coordinates.append((splitList[1].decode(), int(
+ float(splitList[2].decode()) * 90), int(float(splitList[3].decode()) * -90)))
# self.coordinates[splitList[1]] = (
# int(float(splitList[2]) * 90), int(float(splitList[3]) * -90))
self.insert_coordinates()
@@ -734,7 +737,7 @@ class ExperimentGraphDrawing(MainDataBase):
class JobDataStructure(MainDataBase):
- def __init__(self, expid, basic_config):
+ def __init__(self, expid: str, basic_config: APIBasicConfig):
"""Initializes the object based on the unique identifier of the experiment.
Args:
@@ -744,8 +747,8 @@ class JobDataStructure(MainDataBase):
# BasicConfig.read()
# self.expid = expid
self.folder_path = basic_config.JOBDATA_DIR
- self.database_path = os.path.join(
- self.folder_path, "job_data_" + str(expid) + ".db")
+ exp_paths = ExperimentPaths(expid)
+ self.database_path = exp_paths.job_data_db
# self.conn = None
self.db_version = None
# self.jobdata_list = JobDataList(self.expid)
diff --git a/autosubmit_api/database/db_structure.py b/autosubmit_api/database/db_structure.py
index 87fa177659b86f68bcd79aeadfb1f2d03cc82ee7..0866488dc470ff71fad8d81dc93cff953d530a1e 100644
--- a/autosubmit_api/database/db_structure.py
+++ b/autosubmit_api/database/db_structure.py
@@ -1,7 +1,9 @@
import os
import textwrap
import traceback
-import pysqlite3 as sqlite3
+import sqlite3
+
+from autosubmit_api.persistance.experiment import ExperimentPaths
def get_structure(expid, structures_path):
"""
@@ -12,10 +14,10 @@ def get_structure(expid, structures_path):
:rtype: Dictionary Key: String, Value: List(of String)
"""
try:
+ exp_paths = ExperimentPaths(expid)
+ db_structure_path = exp_paths.structure_db
#pkl_path = os.path.join(exp_path, expid, "pkl")
- if os.path.exists(structures_path):
- db_structure_path = os.path.join(
- structures_path, "structure_" + expid + ".db")
+ if os.path.exists(db_structure_path):
# Create file
os.umask(0)
if not os.path.exists(db_structure_path):
@@ -51,8 +53,8 @@ def get_structure(expid, structures_path):
return dict()
else:
# pkl folder not found
- raise Exception("structures folder not found " +
- str(structures_path))
+ raise Exception("structures db not found " +
+ str(db_structure_path))
except Exception as exp:
print((traceback.format_exc()))
@@ -70,7 +72,7 @@ def create_connection(db_file):
return None
-def create_table(conn, create_table_sql):
+def create_table(conn: sqlite3.Connection, create_table_sql):
""" create a table from the create_table_sql statement
:param conn: Connection object
:param create_table_sql: a CREATE TABLE statement
@@ -100,59 +102,3 @@ def _get_exp_structure(path):
except Exception as exp:
print((traceback.format_exc()))
return dict()
-
-
-def save_structure(graph, exp_id, structures_path):
- """
- Saves structure if path is valid
- """
- #pkl_path = os.path.join(exp_path, exp_id, "pkl")
- if os.path.exists(structures_path):
- db_structure_path = os.path.join(
- structures_path, "structure_" + exp_id + ".db")
- # with open(db_structure_path, "w"):
- conn = create_connection(db_structure_path)
- deleted = _delete_table_content(conn)
- if deleted == True:
- nodes_edges = {u for u, v in graph.edges()}
- nodes_edges.update({v for u, v in graph.edges()})
- independent_nodes = {
- u for u in graph.nodes() if u not in nodes_edges}
- data = {(u, v) for u, v in graph.edges()}
- data.update({(u, u) for u in independent_nodes})
- # save
- _create_edge(conn, data)
- #print("Created edge " + str(u) + str(v))
- conn.commit()
- else:
- # pkl folder not found
- raise Exception("pkl folder not found " + str(structures_path))
-
-
-def _create_edge(conn, data):
- """
- Create edge
- """
- try:
- sql = ''' INSERT INTO experiment_structure(e_from, e_to) VALUES(?,?) '''
- cur = conn.cursor()
- cur.executemany(sql, data)
- # return cur.lastrowid
- except sqlite3.Error as e:
- print(("Error on Insert : " + str(type(e).__name__)))
-
-
-def _delete_table_content(conn):
- """
- Deletes table content
- """
- try:
- sql = ''' DELETE FROM experiment_structure '''
- cur = conn.cursor()
- cur.execute(sql)
- conn.commit()
- return True
- except Exception as exp:
- # print(traceback.format_exc())
- print(("Error on Delete _delete_table_content: {}".format(str(exp))))
- return False
diff --git a/autosubmit_api/database/extended_db.py b/autosubmit_api/database/extended_db.py
deleted file mode 100644
index d2469013909652d5578bbd71cbfda3181f461c38..0000000000000000000000000000000000000000
--- a/autosubmit_api/database/extended_db.py
+++ /dev/null
@@ -1,36 +0,0 @@
-from autosubmit_api.config.basicConfig import APIBasicConfig
-from autosubmit_api.database.db_manager import DbManager
-from autosubmit_api.experiment.common_db_requests import prepare_completed_times_db, prepare_status_db
-from autosubmit_api.workers.populate_details.populate import DetailsProcessor
-
-class ExtendedDB:
- def __init__(self, root_path: str, db_name: str, as_times_db_name: str) -> None:
- self.root_path = root_path
- self.db_name = db_name
- self.main_db_manager = DbManager(root_path, db_name)
- self.as_times_db_manager = DbManager(root_path, as_times_db_name)
-
- def prepare_db(self):
- """
- Create tables and views that are required
- """
- self.prepare_main_db()
- self.prepare_as_times_db()
-
-
- def prepare_main_db(self):
- APIBasicConfig.read()
- DetailsProcessor(APIBasicConfig).create_details_table_if_not_exists()
- self.main_db_manager.create_view(
- 'listexp',
- 'select id,name,user,created,model,branch,hpc,description from experiment left join details on experiment.id = details.exp_id'
- )
-
- def prepare_as_times_db(self):
- prepare_completed_times_db()
- prepare_status_db()
- self.as_times_db_manager.create_view('currently_running',
- 'select s.name, s.status, t.total_jobs from experiment_status as s inner join experiment_times as t on s.name = t.name where s.status="RUNNING" ORDER BY t.total_jobs'
- )
-
-
diff --git a/autosubmit_api/database/models.py b/autosubmit_api/database/models.py
index db4a3e09eb5fa6427d8c1ee64e8e39480dd8c525..c63b2b056be1392b3b4bc8a2dc780d6eeaf81175 100644
--- a/autosubmit_api/database/models.py
+++ b/autosubmit_api/database/models.py
@@ -2,7 +2,7 @@ from typing import Optional
from pydantic import BaseModel
-class Experiment(BaseModel):
+class ExperimentModel(BaseModel):
id: int
name: str
description: str
diff --git a/autosubmit_api/database/queries.py b/autosubmit_api/database/queries.py
index d23d684908090957a5e68cfeab535cdb275a038e..ad3e3278658386af3b540adb1a7158981e9bc1a6 100644
--- a/autosubmit_api/database/queries.py
+++ b/autosubmit_api/database/queries.py
@@ -20,9 +20,6 @@ def generate_query_listexp_extended(
select(
tables.experiment_table,
tables.details_table,
- tables.experiment_times_table.c.exp_id,
- tables.experiment_times_table.c.total_jobs,
- tables.experiment_times_table.c.completed_jobs,
tables.experiment_status_table.c.exp_id,
tables.experiment_status_table.c.status,
)
@@ -31,11 +28,6 @@ def generate_query_listexp_extended(
tables.experiment_table.c.id == tables.details_table.c.exp_id,
isouter=True,
)
- .join(
- tables.experiment_times_table,
- tables.experiment_table.c.id == tables.experiment_times_table.c.exp_id,
- isouter=True,
- )
.join(
tables.experiment_status_table,
tables.experiment_table.c.id == tables.experiment_status_table.c.exp_id,
diff --git a/autosubmit_api/database/tables.py b/autosubmit_api/database/tables.py
index bc3913d34c9e3d25f5b415679112a278c414cf0a..7a42573b5b774d59c7cd6ff25c84f449042f526d 100644
--- a/autosubmit_api/database/tables.py
+++ b/autosubmit_api/database/tables.py
@@ -28,17 +28,6 @@ details_table = Table(
# AS_TIMES TABLES
-experiment_times_table = Table(
- "experiment_times",
- metadata_obj,
- Column("exp_id", Integer, primary_key=True),
- Column("name", Text, nullable=False),
- Column("created", Integer, nullable=False),
- Column("modified", Integer, nullable=False),
- Column("total_jobs", Integer, nullable=False),
- Column("completed_jobs", Integer, nullable=False),
-)
-
experiment_status_table = Table(
"experiment_status",
metadata_obj,
@@ -48,3 +37,15 @@ experiment_status_table = Table(
Column("seconds_diff", Integer, nullable=False),
Column("modified", Text, nullable=False),
)
+
+
+# Graph Data TABLES
+
+graph_data_table = Table(
+ "experiment_graph_draw",
+ metadata_obj,
+ Column("id", Integer, primary_key=True),
+ Column("job_name", Text, nullable=False),
+ Column("x", Integer, nullable=False),
+ Column("y", Integer, nullable=False),
+)
diff --git a/autosubmit_api/experiment/common_db_requests.py b/autosubmit_api/experiment/common_db_requests.py
index 752d5e8a78c1b078863e39bcb1cd1d54b3adf4f7..801e7e4447946140990bc2853c4f6d1b60e6f05d 100644
--- a/autosubmit_api/experiment/common_db_requests.py
+++ b/autosubmit_api/experiment/common_db_requests.py
@@ -1,11 +1,7 @@
import os
-import time
-import textwrap
import traceback
import sqlite3
from datetime import datetime
-from collections import OrderedDict
-from typing import List, Tuple, Dict
from autosubmit_api.config.basicConfig import APIBasicConfig
APIBasicConfig.read()
@@ -15,98 +11,8 @@ DB_FILES_STATUS = os.path.join(APIBasicConfig.LOCAL_ROOT_DIR, "as_metadata", "te
# PATH_DB_DATA = "/esarchive/autosubmit/as_metadata/data/"
-def prepare_status_db():
- """
- Creates table of experiment status if it does not exist.
- :return: Map from experiment name to (Id of experiment, Status, Seconds)
- :rtype: Dictionary Key: String, Value: Integer, String, Integer
- """
- conn = create_connection(DB_FILE_AS_TIMES)
- create_table_query = textwrap.dedent(
- '''CREATE TABLE
- IF NOT EXISTS experiment_status (
- exp_id integer PRIMARY KEY,
- name text NOT NULL,
- status text NOT NULL,
- seconds_diff integer NOT NULL,
- modified text NOT NULL,
- FOREIGN KEY (exp_id) REFERENCES experiment (id)
- );''')
- # drop_table_query = ''' DROP TABLE experiment_status '''
- # create_table(conn, drop_table_query)
- create_table(conn, create_table_query)
- current_table = _get_exp_status()
- current_table_expid = dict()
- # print(current_table)
- # print(type(current_table))
- for item in current_table:
- exp_id, expid, status, seconds = item
- current_table_expid[expid] = (exp_id, status, seconds)
- return current_table_expid
-
-
-def prepare_completed_times_db():
- """
- Creates two tables:
- "experiment_times" that stores general information about experiments' jobs completed updates
- "job_times" that stores information about completed time for jobs in the database as_times.db
- Then retrieves all the experiments data from "experiment_times".
- :return: Dictionary that maps experiment name to experiment data.
- :rtype: Dictionary: Key String, Value 6-tuple (exp_id, name, created, modified, total_jobs, completed_jobs)
- """
- conn = create_connection(DB_FILE_AS_TIMES)
- create_table_header_query = textwrap.dedent(
- '''CREATE TABLE
- IF NOT EXISTS experiment_times (
- exp_id integer PRIMARY KEY,
- name text NOT NULL,
- created int NOT NULL,
- modified int NOT NULL,
- total_jobs int NOT NULL,
- completed_jobs int NOT NULL,
- FOREIGN KEY (exp_id) REFERENCES experiment (id)
- );''')
-
- create_table_detail_query = textwrap.dedent(
- '''CREATE TABLE
- IF NOT EXISTS job_times (
- detail_id integer PRIMARY KEY AUTOINCREMENT,
- exp_id integer NOT NULL,
- job_name text NOT NULL,
- created integer NOT NULL,
- modified integer NOT NULL,
- submit_time int NOT NULL,
- start_time int NOT NULL,
- finish_time int NOT NULL,
- status text NOT NULL,
- FOREIGN KEY (exp_id) REFERENCES experiment (id)
- );''')
-
- drop_table_header_query = ''' DROP TABLE experiment_times '''
- drop_table_details_query = ''' DROP TABLE job_times '''
- # create_table(conn, drop_table_details_query)
- # create_table(conn, drop_table_header_query)
- # return
- create_table(conn, create_table_header_query)
- create_table(conn, create_table_detail_query)
- current_table = _get_exp_times()
- current_table_expid = dict()
-
- for item in current_table:
- try:
- exp_id, name, created, modified, total_jobs, completed_jobs = item
- current_table_expid[name] = (exp_id, int(created), int(
- modified), int(total_jobs), int(completed_jobs))
- except Exception as ex:
- print((traceback.format_exc()))
- print(item)
- print((str(name) + " ~ " + str(created) + "\t" + str(modified)))
-
- return current_table_expid
-
# STATUS ARCHIVE
-
def insert_archive_status(status, alatency, abandwidth, clatency, cbandwidth, rtime):
try:
@@ -146,99 +52,6 @@ def get_last_read_archive_status():
# INSERTIONS
-
-def insert_experiment_times_header(expid, timest, total_jobs, completed_jobs, debug=False, log=None):
- """
- Inserts into experiment times header. Requires ecearth.db connection.
- :param expid: Experiment name
- :type expid: String
- :param timest: timestamp of the pkl last modified date
- :type timest: Integer
- :param total_jobs: Total number of jobs
- :type total_jobs: Integer
- :param completed_jobs: Number of completed jobs
- :type completed_jobs: Integer
- :param debug: Flag (testing purposes)
- :type debug: Boolean
- :param conn_ecearth: ecearth.db connection
- :type conn: sqlite3 connection
- :return: exp_id of the experiment (not the experiment name)
- :rtype: Integer
- """
- try:
- current_id = _get_id_db(create_connection(APIBasicConfig.DB_PATH), expid)
- if (current_id):
- if (debug == True):
- print(("INSERTING INTO EXPERIMENT_TIMES " + str(current_id) + "\t" + str(expid) +
- "\t" + str(timest) + "\t" + str(total_jobs) + "\t" + str(completed_jobs)))
- return current_id
- row_content = (current_id, expid, int(timest), int(timest), total_jobs, completed_jobs)
- result = _create_exp_times(row_content)
- return current_id
- else:
- return -1
- except sqlite3.Error as e:
- print(("Error on Insert : " + str(type(e).__name__)))
- print(current_id)
-
-
-def _create_exp_times(row_content):
- """
- Create experiment times
- :param conn:
- :param details:
- :return:
- """
- try:
- conn = create_connection(DB_FILE_AS_TIMES)
- sql = ''' INSERT OR REPLACE INTO experiment_times(exp_id, name, created, modified, total_jobs, completed_jobs) VALUES(?,?,?,?,?,?) '''
- # print(row_content)
- cur = conn.cursor()
- cur.execute(sql, row_content)
- # print(cur)
- conn.commit()
- return cur.lastrowid
- except sqlite3.Error as e:
- print(("Error on Insert : " + str(type(e).__name__)))
- print(row_content)
-
-
-def create_many_job_times(list_job):
- try:
- conn = create_connection(DB_FILE_AS_TIMES)
- sql = ''' INSERT INTO job_times(exp_id, job_name, created, modified, submit_time, start_time, finish_time, status) VALUES(?,?,?,?,?,?,?,?) '''
- # print(row_content)
- cur = conn.cursor()
- cur.executemany(sql, list_job)
- # print(cur)
- # Commit outside the loop
- conn.commit()
- except sqlite3.Error as e:
- print((traceback.format_exc()))
- print(("Error on Insert : " + str(type(e).__name__)))
- # print((exp_id, job_name, timest, timest,
- # submit_time, start_time, finish_time, status))
-
-
-def _insert_into_ecearth_details(exp_id, user, created, model, branch, hpc):
- """
- Inserts into the details table of ecearth.db
- :return: Id
- :rtype: int
- """
- conn = create_connection(APIBasicConfig.DB_PATH)
- if conn:
- try:
- sql = ''' INSERT INTO details(exp_id, user, created, model, branch, hpc) VALUES(?,?,?,?,?,?) '''
- cur = conn.cursor()
- cur.execute(sql, (exp_id, user, created, model, branch, hpc))
- conn.commit()
- return cur.lastrowid
- except Exception as exp:
- print(exp)
- return False
-
-
def create_connection(db_file):
# type: (str) -> sqlite3.Connection
"""
@@ -253,63 +66,8 @@ def create_connection(db_file):
print(exp)
-def create_table(conn, create_table_sql):
- """ create a table from the create_table_sql statement
- :param conn: Connection object
- :param create_table_sql: a CREATE TABLE statement
- :return:
- """
- try:
- c = conn.cursor()
- c.execute(create_table_sql)
- except Exception as e:
- print(e)
-
-
# SELECTS
-def get_times_detail(exp_id):
- """
- Gets the current detail of the experiment from job_times
- :param exp_id: Id of the experiment
- :type exp_id: Integer
- :return: Dictionary Key: Job Name, Values: 5-tuple (submit time, start time, finish time, status, detail id)
- :rtype: dict
- """
- # conn = create_connection(DB_FILE_AS_TIMES)
- try:
- current_table_detail = dict()
- current_table = _get_job_times(exp_id)
- if current_table is None:
- return None
- for item in current_table:
- detail_id, exp_id, job_name, created, modified, submit_time, start_time, finish_time, status = item
- current_table_detail[job_name] = (
- submit_time, start_time, finish_time, status, detail_id)
- return current_table_detail
- except Exception as exp:
- print((traceback.format_exc()))
- return None
-
-
-def get_times_detail_by_expid(conn, expid):
- """
- Gets the detail of times of the experiment by expid (name of experiment).\n
- :param conn: ecearth.db connection
- :rtype conn: sqlite3 connection
- :param expid: Experiment name
- :type expid: str
- :return: Dictionary Key: Job Name, Values: 5-tuple (submit time, start time, finish time, status, detail id)
- :rtype: dict
- """
- # conn = create_connection(DB_FILE_AS_TIMES)
- exp_id = _get_id_db(conn, expid)
- if (exp_id):
- return get_times_detail(exp_id)
- else:
- return None
-
-
def get_experiment_status():
"""
Gets table experiment_status as dictionary
@@ -324,22 +82,6 @@ def get_experiment_status():
return experiment_status
-def get_currently_running_experiments():
- """
- Gets the list of currently running experiments ordered by total_jobs
- Connects to AS_TIMES
- :return: map of expid -> total_jobs
- :rtype: OrderedDict() str -> int
- """
- experiment_running = OrderedDict()
- current_running = _get_exp_only_active()
- if current_running:
- for item in current_running:
- name, status, total_jobs = item
- experiment_running[name] = total_jobs
- return experiment_running
-
-
def get_specific_experiment_status(expid):
"""
Gets the current status from database.\n
@@ -353,287 +95,6 @@ def get_specific_experiment_status(expid):
return (name, status)
-def get_experiment_times():
- # type: () -> Dict[str, Tuple[int, int, int]]
- """
- Gets table experiment_times as dictionary
- conn is expected to reference as_times.db
- """
- # conn = create_connection(DB_FILE_AS_TIMES)
- experiment_times = dict()
- current_table = _get_exp_times()
- for item in current_table:
- exp_id, name, created, modified, total_jobs, completed_jobs = item
- experiment_times[name] = (total_jobs, completed_jobs, modified)
- # if extended == True:
- # experiment_times[name] = (total_jobs, completed_jobs, created, modified)
- return experiment_times
-
-
-def get_experiment_times_by_expid(expid):
- """[summary]
- :return: exp_id, total number of jobs, number of completed jobs
- :rtype 3-tuple: (int, int, int)
- """
- current_row = _get_exp_times_by_expid(expid)
- if current_row:
- exp_id, name, created, modified, total_jobs, completed_jobs = current_row
- return (exp_id, total_jobs, completed_jobs)
- return None
-
-
-def get_experiment_times_group():
- """
- Gets table experiment_times as dictionary id -> name
- conn is expected to reference as_times.db
- """
- experiment_times = dict()
- current_table = _get_exp_times()
- for item in current_table:
- exp_id, name, created, modified, total_jobs, completed_jobs = item
- if name not in list(experiment_times.keys()):
- experiment_times[name] = list()
- experiment_times[name].append(exp_id)
- # if extended == True:
- # experiment_times[name] = (total_jobs, completed_jobs, created, modified)
- return experiment_times
-
-
-def get_exps_base():
- """
- Get exp name and id from experiment table in ecearth.db
- :param conn: ecearth.db connection
- :param expid:
- :return:
- """
- conn = create_connection(APIBasicConfig.DB_PATH)
- result = dict()
- conn.text_factory = str
- cur = conn.cursor()
- cur.execute(
- "SELECT name, id FROM experiment WHERE autosubmit_version IS NOT NULL")
- rows = cur.fetchall()
- cur.close()
- conn.close()
- if (rows):
- for row in rows:
- _name, _id = row
- result[_name] = _id
- else:
- return None
- return result
-
-
-def get_exps_detailed_complete():
- """
- Get information from details table
- :param conn: ecearth.db connection
- :param expid:
- :return: Dictionary exp_id -> (user, created, model, branch, hpc)
- """
- all_details = _get_exps_detailed_complete(create_connection(APIBasicConfig.DB_PATH))
- result = dict()
- if (all_details):
- for item in all_details:
- exp_id, user, created, model, branch, hpc = item
- result[exp_id] = (user, created, model, branch, hpc)
- return result
-
-
-def get_latest_completed_jobs(seconds=300):
- """
- Get latest completed jobs
- """
- result = dict()
- latest_completed_detail = _get_latest_completed_jobs(seconds)
- if (latest_completed_detail):
- for item in latest_completed_detail:
- detail_id, exp_id, job_name, created, modified, submit_time, start_time, finish_time, status = item
- result[job_name] = (detail_id, submit_time,
- start_time, finish_time, status)
- return result
-
-
-def _get_exps_detailed_complete(conn):
- """
- Get information from details table
- :param conn: ecearth.db connection
- :param expid:
- :return:
- """
- conn.text_factory = str
- cur = conn.cursor()
- cur.execute(
- "SELECT * FROM details")
- rows = cur.fetchall()
- cur.close()
- conn.close()
- return rows
-
-
-def _get_exp_times():
- # type: () -> List[Tuple[int, str, int, int, int, int]]
- """
- Get all experiments from table experiment_times.\n
- :return: Row content (exp_id, name, created, modified, total_jobs, completed_jobs)
- :rtype: 6-tuple (int, str, int, int, int, int)
- """
- try:
- conn = create_connection(os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB))
- conn.text_factory = str
- cur = conn.cursor()
- cur.execute(
- "SELECT exp_id, name, created, modified, total_jobs, completed_jobs FROM experiment_times")
- rows = cur.fetchall()
- conn.close()
- return rows
- except Exception as exp:
- print((traceback.format_exc()))
- return list()
-
-
-def _get_exp_times_by_expid(expid):
- """
- Returns data from experiment_time table by expid.
-
- :param expid: expid/name
- :type expid: str
- :return: Row content (exp_id, name, created, modified, total_jobs, completed_jobs)
- :rtype: 6-tuple (int, str, str, str, int, int)
- """
- try:
- conn = create_connection(os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB))
- conn.text_factory = str
- cur = conn.cursor()
- cur.execute(
- "SELECT exp_id, name, created, modified, total_jobs, completed_jobs FROM experiment_times WHERE name=?", (expid,))
- rows = cur.fetchall()
- conn.close()
- return rows[0] if len(rows) > 0 else None
- except Exception as exp:
- print((traceback.format_exc()))
- return None
-
-
-def _get_job_times(exp_id):
- """
- Get exp job times detail for a given expid from job_times.
- :param exp_id: Experiment id (not name)
- :type exp_id: Integer
- :return: Detail content
- :rtype: list of 9-tuple: (detail_id, exp_id, job_name, created, modified, submit_time, start_time, finish_time, status)
- """
- try:
- conn = create_connection(os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB))
- cur = conn.cursor()
- cur.execute("SELECT detail_id, exp_id, job_name, created, modified, submit_time, start_time, finish_time, status FROM job_times WHERE exp_id=?", (exp_id,))
- rows = cur.fetchall()
- conn.close()
- return rows
- except Exception as ex:
- print((traceback.format_exc()))
- print((str(exp_id)))
- print((str(ex)))
- return None
-
-
-def _get_latest_completed_jobs(seconds=300):
- """
- get latest completed jobs, defaults at 300 seconds, 5 minutes
- """
- current_ts = time.time()
- try:
- conn = create_connection(os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB))
- cur = conn.cursor()
- cur.execute("SELECT detail_id, exp_id, job_name, created, modified, submit_time, start_time, finish_time, status FROM job_times WHERE (? - modified)<=? AND status='COMPLETED'", (current_ts, seconds))
- rows = cur.fetchall()
- conn.close()
- return rows
- except Exception as ex:
- print((traceback.format_exc()))
- print((str(seconds)))
- print((str(ex)))
- return None
-
-
-def _delete_many_from_job_times_detail(detail_list):
- try:
- conn = create_connection(os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB))
- cur = conn.cursor()
- #print("Cursor defined")
- cur.executemany(
- "DELETE FROM job_times WHERE detail_id=?", detail_list)
- # print(cur.rowcount)
- cur.close()
- conn.commit()
- conn.close()
- # No reliable way to get any feedback from cursor at this point, so let's just return 1
- return True
- except Exception as exp:
- # print("Error while trying to delete " +
- # str(detail_id) + " from job_times.")
- print((traceback.format_exc()))
- return None
-
-
-def delete_experiment_data(exp_id):
- # type: (int) -> bool
- """
- Deletes experiment data from experiment_times and job_times in as_times.db
-
- :param exp_id: Id of experiment
- :type exp_id: int
- """
- deleted_e = _delete_from_experiment_times(exp_id)
- deleted_j = _delete_from_job_times(exp_id)
- print(("Main exp " + str(deleted_e) + "\t" + str(deleted_j)))
- if deleted_e and deleted_j:
- return True
- return False
-
-
-def _delete_from_experiment_times(exp_id):
- """
- Deletes an experiment from experiment_times in as_times.db
-
- :param exp_id: Id of experiment
- :type exp_id: int
- """
- try:
- conn = create_connection(os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB))
- cur = conn.cursor()
- cur.execute("DELETE FROM experiment_times WHERE exp_id=?", (exp_id,))
- conn.commit()
- conn.close()
- return True
- except Exception as exp:
- print(("Error while trying to delete " +
- str(exp_id) + " from experiment_times."))
- print((traceback.format_exc()))
- return None
-
-
-def _delete_from_job_times(exp_id):
- """
- Deletes an experiment from job_times in as_times.db
-
- :param exp_id: Id of experiment
- :type exp_id: int
- """
- try:
- conn = create_connection(os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB))
- cur = conn.cursor()
- cur.execute("DELETE FROM job_times WHERE exp_id=?", (exp_id,))
- conn.commit()
- conn.close()
- return True
- except Exception as exp:
- print(("Error while trying to delete " +
- str(exp_id) + " from job_times."))
- print((traceback.format_exc()))
- return None
-
-
def _get_exp_status():
"""
Get all registers from experiment_status.\n
@@ -653,23 +114,6 @@ def _get_exp_status():
return dict()
-def _get_exp_only_active():
- """
- Get all registers of experiments ACTIVE
- """
- try:
- conn = create_connection(os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB))
- conn.text_factory = str
- cur = conn.cursor()
- cur.execute("select name, status, total_jobs from currently_running")
- rows = cur.fetchall()
- return rows
- except Exception as exp:
- print((traceback.format_exc()))
- print(exp)
- return None
-
-
def _get_specific_exp_status(expid):
"""
Get all registers from experiment_status.\n
@@ -692,167 +136,4 @@ def _get_specific_exp_status(expid):
return (0, expid, "NOT RUNNING", 0)
-def _get_id_db(conn, expid):
- """
- Get exp_id of the experiment (different than the experiment name).
- :param conn: ecearth.db connection
- :type conn: sqlite3 connection
- :param expid: Experiment name
- :type expid: String
- :return: Id of the experiment
- :rtype: Integer or None
- """
- try:
- cur = conn.cursor()
- cur.execute("SELECT id FROM experiment WHERE name=?", (expid,))
- row = cur.fetchone()
- return row[0]
- except Exception as exp:
- print(("Couldn't get exp_id for {0}".format(expid)))
- print(traceback.format_exc())
- return None
-
-
# UPDATES
-
-
-def update_exp_status(expid, status, seconds_diff):
- """
- Update existing experiment_status.
- :param expid: Experiment name
- :type expid: String
- :param status: Experiment status
- :type status: String
- :param seconds_diff: Indicator of how long it has been active since the last time it was checked
- :type seconds_diff: Integer
- :return: Id of register
- :rtype: Integer
- """
- try:
- conn = create_connection(os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB))
- sql = ''' UPDATE experiment_status SET status = ?, seconds_diff = ?, modified = ? WHERE name = ? '''
- cur = conn.cursor()
- cur.execute(sql, (status, seconds_diff,
- datetime.today().strftime('%Y-%m-%d-%H:%M:%S'), expid))
- conn.commit()
- return cur.lastrowid
- except sqlite3.Error as e:
- print(("Error while trying to update " +
- str(expid) + " in experiment_status."))
- print((traceback.format_exc()))
- print(("Error on Update: " + str(type(e).__name__)))
-
-
-def _update_ecearth_details(exp_id, user, created, model, branch, hpc):
- """
- Updates ecearth.db table details.
- """
- conn = create_connection(APIBasicConfig.DB_PATH)
- if conn:
- try:
- sql = ''' UPDATE details SET user=?, created=?, model=?, branch=?, hpc=? where exp_id=? '''
- cur = conn.cursor()
- cur.execute(sql, (user, created, model, branch, hpc, exp_id))
- conn.commit()
- return True
- except Exception as exp:
- print(exp)
- return False
- return False
-
-
-def update_experiment_times(exp_id, modified, completed_jobs, total_jobs, debug=False):
- """
- Update existing experiment times header
- """
- try:
- conn = create_connection(os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB))
- if (debug == True):
- print(("UPDATE experiment_times " + str(exp_id) + "\t" +
- str(completed_jobs) + "\t" + str(total_jobs)))
- return
- sql = ''' UPDATE experiment_times SET modified = ?, completed_jobs = ?, total_jobs = ? WHERE exp_id = ? '''
- cur = conn.cursor()
- cur.execute(sql, (int(modified), completed_jobs, total_jobs, exp_id))
- conn.commit()
- # print("Updated header")
- return exp_id
- except sqlite3.Error as e:
- print(("Error while trying to update " +
- str(exp_id) + " in experiment_times."))
- print((traceback.format_exc()))
- print(("Error on Update: " + str(type(e).__name__)))
-
-
-def update_experiment_times_only_modified(exp_id, modified):
- try:
- conn = create_connection(os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB))
- sql = ''' UPDATE experiment_times SET modified = ? WHERE exp_id = ? '''
- cur = conn.cursor()
- cur.execute(sql, (int(modified), exp_id))
- conn.commit()
- # print("Updated header")
- return exp_id
- except sqlite3.Error as e:
- print(("Error while trying to update " +
- str(exp_id) + " in experiment_times."))
- print((traceback.format_exc()))
- print(("Error on Update: " + str(type(e).__name__)))
-
-
-def update_job_times(detail_id, modified, submit_time, start_time, finish_time, status, debug=False, no_modify_time=False):
- """
- Update single experiment job detail \n
- :param conn: Connection to as_times.db. \n
- :type conn: sqlite3 connection. \n
- :param detail_id: detail_id in as_times.job_times. \n
- :type detail_id: Integer. \n
- :param modified: Timestamp of current date of pkl \n
- :type modified: Integer. \n
- """
- try:
- conn = create_connection(os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB))
- if (debug == True):
- print(("UPDATING JOB TIMES " + str(detail_id) + " ~ " + str(modified) + "\t" +
- str(submit_time) + "\t" + str(start_time) + "\t" + str(finish_time) + "\t" + str(status)))
- return
- if no_modify_time == False:
- sql = ''' UPDATE job_times SET modified = ?, submit_time = ?, start_time = ?, finish_time = ?, status = ? WHERE detail_id = ? '''
- cur = conn.cursor()
- cur.execute(sql, (modified, submit_time, start_time,
- finish_time, status, detail_id))
- # Commit outside the loop
- conn.commit()
- cur.close()
- conn.close()
- else:
- sql = ''' UPDATE job_times SET submit_time = ?, start_time = ?, finish_time = ?, status = ? WHERE detail_id = ? '''
- cur = conn.cursor()
- cur.execute(sql, (submit_time, start_time,
- finish_time, status, detail_id))
- # Commit outside the loop
- conn.commit()
- cur.close()
- conn.close()
-
- except sqlite3.Error as e:
- print(("Error while trying to update " +
- str(detail_id) + " in job_times."))
- print((traceback.format_exc()))
- print(("Error on Update: " + str(type(e).__name__)))
-
-
-def update_many_job_times(list_jobs):
- try:
- conn = create_connection(os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB))
- sql = ''' UPDATE job_times SET modified = ?, submit_time = ?, start_time = ?, finish_time = ?, status = ? WHERE detail_id = ? '''
- cur = conn.cursor()
- cur.executemany(sql, list_jobs)
- # Commit outside the loop
- conn.commit()
- cur.close()
- conn.close()
- except sqlite3.Error as e:
- print("Error while trying to update many in update_many_job_times.")
- print((traceback.format_exc()))
- print(("Error on Update: " + str(type(e).__name__)))
diff --git a/autosubmit_api/experiment/common_requests.py b/autosubmit_api/experiment/common_requests.py
index dce9f43513c8f12385cedf647fb844d4d5ae53ac..0028357e2741388767409e58024102091d66e7b7 100644
--- a/autosubmit_api/experiment/common_requests.py
+++ b/autosubmit_api/experiment/common_requests.py
@@ -43,6 +43,7 @@ from autosubmit_api.logger import logger
from autosubmit_api.performance.utils import calculate_SYPD_perjob
from autosubmit_api.monitor.monitor import Monitor
+from autosubmit_api.persistance.experiment import ExperimentPaths
from autosubmit_api.statistics.statistics import Statistics
@@ -172,7 +173,8 @@ def get_experiment_data(expid):
result["completed_jobs"] = experiment_run.completed
result["db_historic_version"] = experiment_history.manager.db_version
else:
- _, result["total_jobs"], result["completed_jobs"] = DbRequests.get_experiment_times_by_expid(expid)
+ result["total_jobs"] = 0
+ result["completed_jobs"] = 0
result["db_historic_version"] = "NA"
except Exception as exp:
@@ -211,10 +213,9 @@ def _is_exp_running(expid, time_condition=300):
definite_log_path = None
try:
APIBasicConfig.read()
- pathlog_aslog = APIBasicConfig.LOCAL_ROOT_DIR + '/' + expid + '/' + \
- APIBasicConfig.LOCAL_TMP_DIR + '/' + APIBasicConfig.LOCAL_ASLOG_DIR
- pathlog_tmp = APIBasicConfig.LOCAL_ROOT_DIR + '/' + \
- expid + '/' + APIBasicConfig.LOCAL_TMP_DIR
+ exp_paths = ExperimentPaths(expid)
+ pathlog_aslog = exp_paths.tmp_as_logs_dir
+ pathlog_tmp = exp_paths.tmp_dir
# Basic Configuration
look_old_folder = False
current_version = None
@@ -479,7 +480,8 @@ def get_experiment_log_last_lines(expid):
try:
APIBasicConfig.read()
- path = APIBasicConfig.LOCAL_ROOT_DIR + '/' + expid + '/' + APIBasicConfig.LOCAL_TMP_DIR + '/' + APIBasicConfig.LOCAL_ASLOG_DIR
+ exp_paths = ExperimentPaths(expid)
+ path = exp_paths.tmp_as_logs_dir
reading = os.popen('ls -t ' + path + ' | grep "run.log"').read() if (os.path.exists(path)) else ""
# Finding log files
@@ -528,7 +530,8 @@ def get_job_log(expid, logfile, nlines=150):
logcontent = []
reading = ""
APIBasicConfig.read()
- logfilepath = os.path.join(APIBasicConfig.LOCAL_ROOT_DIR, expid, APIBasicConfig.LOCAL_TMP_DIR, "LOG_{0}".format(expid), logfile)
+ exp_paths = ExperimentPaths(expid)
+ logfilepath = os.path.join(exp_paths.tmp_log_dir, logfile)
try:
if os.path.exists(logfilepath):
current_stat = os.stat(logfilepath)
@@ -726,7 +729,7 @@ def get_experiment_tree_rundetail(expid, run_id):
print((traceback.format_exc()))
return {'tree': [], 'jobs': [], 'total': 0, 'reference': [], 'error': True, 'error_message': str(e), 'pkl_timestamp': 0}
base_list['error'] = False
- base_list['error_message'] = 'None'
+ base_list['error_message'] = ''
base_list['pkl_timestamp'] = pkl_timestamp
return base_list
@@ -769,53 +772,6 @@ def get_experiment_tree_structured(expid, log):
return {'tree': [], 'jobs': [], 'total': 0, 'reference': [], 'error': True, 'error_message': str(e), 'pkl_timestamp': 0}
-def verify_last_completed(seconds=300):
- """
- Verifying last 300 seconds by default
- """
- # Basic info
- t0 = time.time()
- APIBasicConfig.read()
- # Current timestamp
- current_st = time.time()
- # Connection
- path = APIBasicConfig.LOCAL_ROOT_DIR
- db_file = os.path.join(path, DbRequests.DB_FILE_AS_TIMES)
- conn = DbRequests.create_connection(db_file)
- # Current latest detail
- td0 = time.time()
- latest_detail = DbRequests.get_latest_completed_jobs(seconds)
- t_data = time.time() - td0
- # Main Loop
- for job_name, detail in list(latest_detail.items()):
- tmp_path = os.path.join(
- APIBasicConfig.LOCAL_ROOT_DIR, job_name[:4], APIBasicConfig.LOCAL_TMP_DIR)
- detail_id, submit, start, finish, status = detail
- submit_time, start_time, finish_time, status_text_res = JUtils.get_job_total_stats(
- common_utils.Status.COMPLETED, job_name, tmp_path)
- submit_ts = int(time.mktime(submit_time.timetuple())) if len(
- str(submit_time)) > 0 else 0
- start_ts = int(time.mktime(start_time.timetuple())) if len(
- str(start_time)) > 0 else 0
- finish_ts = int(time.mktime(finish_time.timetuple())) if len(
- str(finish_time)) > 0 else 0
- if (finish_ts != finish):
- #print("\tMust Update")
- DbRequests.update_job_times(detail_id,
- int(current_st),
- submit_ts,
- start_ts,
- finish_ts,
- status,
- debug=False,
- no_modify_time=True)
- t1 = time.time()
- # Timer safeguard
- if (t1 - t0) > SAFE_TIME_LIMIT:
- raise Exception(
- "Time limit reached {0:06.2f} seconds on verify_last_completed while reading {1}. Time spent on reading data {2:06.2f} seconds.".format((t1 - t0), job_name, t_data))
-
-
def get_experiment_counters(expid):
"""
Returns status counters of the experiment.
@@ -863,7 +819,8 @@ def get_quick_view(expid):
total_count = completed_count = failed_count = running_count = queuing_count = 0
try:
APIBasicConfig.read()
- path_to_logs = os.path.join(APIBasicConfig.LOCAL_ROOT_DIR, expid, "tmp", "LOG_" + expid)
+ exp_paths = ExperimentPaths(expid)
+ path_to_logs = exp_paths.tmp_log_dir
# Retrieving packages
now_ = time.time()
@@ -952,7 +909,8 @@ def get_job_history(expid, job_name):
result = None
try:
APIBasicConfig.read()
- path_to_job_logs = os.path.join(APIBasicConfig.LOCAL_ROOT_DIR, expid, "tmp", "LOG_" + expid)
+ exp_paths = ExperimentPaths(expid)
+ path_to_job_logs = exp_paths.tmp_log_dir
result = ExperimentHistoryDirector(ExperimentHistoryBuilder(expid)).build_reader_experiment_history().get_historic_job_data(job_name)
except Exception as exp:
print((traceback.format_exc()))
diff --git a/autosubmit_api/experiment/experiment_db_manager.py b/autosubmit_api/experiment/experiment_db_manager.py
deleted file mode 100644
index f07f39df6d4dc2da96aa39b8d7d9e42299b8c9eb..0000000000000000000000000000000000000000
--- a/autosubmit_api/experiment/experiment_db_manager.py
+++ /dev/null
@@ -1,38 +0,0 @@
-# References BasicConfig.DB_PATH
-
-import history.database_managers.database_models as Models
-from history.database_managers.database_manager import DatabaseManager
-from config.basicConfig import APIBasicConfig
-from typing import List
-
-class ExperimentDbManager(DatabaseManager):
- # The current implementation only handles the experiment table. The details table is ignored because it is handled by a different worker.
- def __init__(self, basic_config, expid):
- # type: (APIBasicConfig, str) -> None
- super(ExperimentDbManager, self).__init__(expid, basic_config)
- self.basic_config = basic_config
- self._ecearth_file_path = self.basic_config.DB_PATH
-
- def get_experiment_row_by_expid(self, expid):
- # type: (str) -> Models.ExperimentRow | None
- """
- Get the experiment from ecearth.db by expid as Models.ExperimentRow.
- """
- statement = self.get_built_select_statement("experiment", "name=?")
- current_rows = self.get_from_statement_with_arguments(self._ecearth_file_path, statement, (expid,))
- if len(current_rows) <= 0:
- return None
- # raise ValueError("Experiment {0} not found in {1}".format(expid, self._ecearth_file_path))
- return Models.ExperimentRow(*current_rows[0])
-
- def get_experiments_with_valid_version(self):
- # type: () -> List[Models.ExperimentRow]
- statement = self.get_built_select_statement("experiment", "autosubmit_version IS NOT NULL")
- rows = self.get_from_statement(self._ecearth_file_path, statement)
- return [Models.ExperimentRow(*row) for row in rows]
-
- # def insert_experiment_details(self, exp_id, user, created, model, branch, hpc):
-
- # statement = ''' INSERT INTO details(exp_id, user, created, model, branch, hpc) VALUES(?,?,?,?,?,?) '''
- # arguments = (exp_id, user, status, 0, HUtils.get_current_datetime())
- # return self.insert_statement_with_arguments(self._as_times_file_path, statement, arguments)
\ No newline at end of file
diff --git a/autosubmit_api/experiment/old_code.txt b/autosubmit_api/experiment/old_code.txt
deleted file mode 100644
index d9cac6a20157ecd71fc3a9a8f85f340f664028ac..0000000000000000000000000000000000000000
--- a/autosubmit_api/experiment/old_code.txt
+++ /dev/null
@@ -1,412 +0,0 @@
-def get_experiment_metrics(expid):
- """
- Gets metrics
- """
- error = False
- error_message = ""
- SYPD = ASYPD = CHSY = JPSY = RSYPD = Parallelization = 0
- seconds_in_a_day = 86400
- list_considered = []
- core_hours_year = []
- warnings_job_data = []
- total_run_time = 0
- total_queue_time = total_CHSY = total_JPSY = energy_count = 0
-
- try:
- # Basic info
- BasicConfig.read()
- path = BasicConfig.LOCAL_ROOT_DIR + '/' + expid + '/pkl'
- tmp_path = os.path.join(
- BasicConfig.LOCAL_ROOT_DIR, expid, BasicConfig.LOCAL_TMP_DIR)
- pkl_filename = "job_list_" + str(expid) + ".pkl"
- path_pkl = path + "/" + pkl_filename
-
- as_conf = AutosubmitConfig(
- expid, BasicConfig, ConfigParserFactory())
- if not as_conf.check_conf_files():
- # Log.critical('Can not run with invalid configuration')
- raise Exception(
- 'Autosubmit GUI might not have permission to access necessary configuration files.')
-
- # Chunk Information
- chunk_unit = as_conf.get_chunk_size_unit()
- chunk_size = as_conf.get_chunk_size()
- year_per_sim = datechunk_to_year(chunk_unit, chunk_size)
- if year_per_sim <= 0:
- raise Exception("The system couldn't calculate year per SIM value " + str(year_per_sim) +
- ", for chunk size " + str(chunk_size) + " and chunk unit " + str(chunk_unit))
-
- # From database
- # db_file = os.path.join(BasicConfig.LOCAL_ROOT_DIR, "ecearth.db")
- # conn = DbRequests.create_connection(db_file)
- # job_times = DbRequests.get_times_detail_by_expid(conn, expid)
-
- # Job time information
- # Try to get packages
- job_to_package = {}
- package_to_jobs = {}
- job_to_package, package_to_jobs, _, _ = JobList.retrieve_packages(
- BasicConfig, expid)
- # Basic data
- job_running_to_seconds = {}
- # SIM POST TRANSFER jobs (COMPLETED) in experiment
- sim_jobs_in_pkl = []
- post_jobs_in_pkl = []
- transfer_jobs_in_pkl = []
- clean_jobs_in_pkl = []
- sim_jobs_info = []
- post_jobs_info = []
- transfer_jobs_info = []
- clean_jobs_info = []
- sim_jobs_info_asypd = []
- sim_jobs_info_rsypd = []
- outlied = []
- # Get pkl information
- if os.path.exists(path_pkl):
- fd = open(path_pkl, 'r')
- pickle_content = pickle.load(fd)
- # pickle 0: Name, 2: StatusCode
- sim_jobs_in_pkl = [item[0]
- for item in pickle_content if 'SIM' in item[0].split('_') and int(item[2]) == Status.COMPLETED]
- post_jobs_in_pkl = [item[0]
- for item in pickle_content if 'POST' in item[0].split('_') and int(item[2]) == Status.COMPLETED]
- transfer_member = [item[0]
- for item in pickle_content if item[0].find('TRANSFER_MEMBER') > 0 and int(item[2]) == Status.COMPLETED]
- transfer_jobs_in_pkl = [item[0]
- for item in pickle_content if 'TRANSFER' in item[0].split('_') and int(item[2]) == Status.COMPLETED and item[0] not in transfer_member]
- clean_member = [item[0]
- for item in pickle_content if item[0].find('CLEAN_MEMBER') > 0 and int(item[2]) == Status.COMPLETED]
- clean_jobs_in_pkl = [item[0]
- for item in pickle_content if 'CLEAN' in item[0].split('_') and int(item[2]) == Status.COMPLETED and item[0] not in clean_member]
-
- # print(transfer_jobs_in_pkl)
- fakeAllJobs = [SimpleJob(item[0], tmp_path, int(item[2]))
- for item in pickle_content]
- del pickle_content
- job_running_to_seconds, _, warnings_job_data = JobList.get_job_times_collection(
- BasicConfig, fakeAllJobs, expid, job_to_package, package_to_jobs, timeseconds=True)
- # ASYPD - RSYPD warnings
- if len(post_jobs_in_pkl) == 0:
- warnings_job_data.append(
- "ASYPD | There are no (COMPLETED) POST jobs in the experiment, ASYPD cannot be computed.")
- if len(transfer_jobs_in_pkl) == 0 and len(clean_jobs_in_pkl) == 0:
- warnings_job_data.append(
- "RSYPD | There are no TRANSFER nor CLEAN (COMPLETED) jobs in the experiment, RSYPD cannot be computed.")
- if len(transfer_jobs_in_pkl) == 0 and len(clean_jobs_in_pkl) > 0:
- warnings_job_data.append(
- "RSYPD | There are no TRANSFER (COMPLETED) jobs in the experiment, we resort to use (COMPLETED) CLEAN jobs to compute RSYPD.")
-
- Parallelization = 0
- try:
- processors_value = as_conf.get_processors("SIM")
- if processors_value.find(":") >= 0:
- # It is an expression
- components = processors_value.split(":")
- Parallelization = int(sum(
- [math.ceil(float(x) / 36.0) * 36.0 for x in components]))
- warnings_job_data.append("Parallelization parsing | {0} was interpreted as {1} cores.".format(
- processors_value, Parallelization))
- else:
- # It is int
- Parallelization = int(processors_value)
- except Exception as exp:
- # print(exp)
- warnings_job_data.append(
- "CHSY Critical | Autosubmit API could not parse the number of processors for the SIM job.")
- pass
-
- # ASYPD
- # Main Loop
- # Times exist
- if len(job_running_to_seconds) > 0:
- # job_info attributes: ['name', 'queue_time', 'run_time', 'status', 'energy', 'submit', 'start', 'finish', 'ncpus']
- sim_jobs_info = [job_running_to_seconds[job_name]
- for job_name in sim_jobs_in_pkl if job_running_to_seconds.get(job_name, None) is not None]
- sim_jobs_info.sort(key=lambda x: tostamp(x.finish), reverse=True)
-
- # SIM outlier detection
- data_run_times = [job.run_time for job in sim_jobs_info]
- mean_1 = np.mean(data_run_times) if len(data_run_times) > 0 else 0
- std_1 = np.std(data_run_times) if len(data_run_times) > 0 else 0
- threshold = 2
-
- # ASYPD Pre
- post_jobs_info = [job_running_to_seconds[job_name]
- for job_name in post_jobs_in_pkl if job_running_to_seconds.get(job_name, None) is not None and job_running_to_seconds[job_name].finish is not None]
- post_jobs_info.sort(key=lambda x: tostamp(x.finish), reverse=True)
- # End ASYPD Pre
- for job_info in sim_jobs_info:
- # JobRow object
- z_score = (job_info.run_time - mean_1) / \
- std_1 if std_1 > 0 else 0
- # print("{} : {}".format(job_info.name, z_score, threshold))
- if np.abs(z_score) <= threshold and job_info.run_time > 0:
- status = job_info.status if job_info else "UNKNOWN"
- # Energy
- energy = round(job_info.energy, 2) if job_info else 0
- if energy == 0:
- warnings_job_data.append(
- "Considered | Job {0} (Package {1}) has no energy information and is not going to be considered for energy calculations.".format(job_info.name, job_to_package.get(job_info.name, "")))
- total_queue_time += max(int(job_info.queue_time), 0)
- total_run_time += max(int(job_info.run_time), 0)
- seconds_per_year = (Parallelization *
- job_info.run_time) / year_per_sim
- job_JPSY = round(energy / year_per_sim,
- 2) if year_per_sim > 0 else 0
- job_SYPD = round((year_per_sim * seconds_in_a_day) /
- max(int(job_info.run_time), 0), 2) if job_info.run_time > 0 else 0
- job_ASYPD = round((year_per_sim * seconds_in_a_day) / (int(job_info.queue_time) + int(job_info.run_time) + sum(
- job.queue_time + job.run_time for job in post_jobs_info) / len(post_jobs_info)) if len(post_jobs_info) > 0 else 0, 2)
-
- # Maximum finish time
- # max_sim_finish = tostamp(job_info.finish) if job_info.finish is not None and tostamp(
- # job_info.finish) > max_sim_finish else max_sim_finish
- # sim_count += 1
- total_CHSY += round(seconds_per_year / 3600, 2)
- total_JPSY += job_JPSY
- if job_JPSY > 0:
- # Ignore for mean calculation
- energy_count += 1
- # core_hours_year.append(year_seconds/3600)
- list_considered.append(
- {"name": job_info.name,
- "queue": int(job_info.queue_time),
- "running": int(job_info.run_time),
- "CHSY": round(seconds_per_year / 3600, 2),
- "SYPD": job_SYPD,
- "ASYPD": job_ASYPD,
- "JPSY": job_JPSY,
- "energy": energy})
- else:
- # print("Outlied {}".format(job_info.name))
- outlied.append(job_info.name)
- warnings_job_data.append(
- "Outlier | Job {0} (Package {1} - Running time {2} seconds) has been considered an outlier (mean {3}, std {4}, z_score {5}) and will be ignored for performance calculations.".format(job_info.name, job_to_package.get(job_info.name, "NA"), str(job_info.run_time), str(round(mean_1, 2)), str(round(std_1, 2)), str(round(z_score, 2))))
-
- # ASYPD Pre
- sim_jobs_info_asypd = [job for job in sim_jobs_info if job.name not in outlied] if len(
- post_jobs_info) > 0 else []
- sim_jobs_info_asypd.sort(
- key=lambda x: tostamp(x.finish), reverse=True)
-
- # RSYPD
- transfer_jobs_info = [job_running_to_seconds[job_name]
- for job_name in transfer_jobs_in_pkl if job_running_to_seconds.get(job_name, None) is not None and job_running_to_seconds[job_name].finish is not None]
- transfer_jobs_info.sort(
- key=lambda x: tostamp(x.finish), reverse=True)
- if len(transfer_jobs_info) <= 0:
- clean_jobs_info = [job_running_to_seconds[job_name]
- for job_name in clean_jobs_in_pkl if job_running_to_seconds.get(job_name, None) is not None and job_running_to_seconds[job_name].finish is not None]
- clean_jobs_info.sort(
- key=lambda x: tostamp(x.finish), reverse=True)
- sim_jobs_info_rsypd = [job for job in sim_jobs_info if job.name not in outlied and tostamp(job.finish) <= tostamp(
- clean_jobs_info[0].finish)] if len(clean_jobs_info) > 0 else []
- else:
- sim_jobs_info_rsypd = [job for job in sim_jobs_info if job.name not in outlied and tostamp(job.finish) <= tostamp(
- transfer_jobs_info[0].finish)]
- sim_jobs_info_rsypd.sort(
- key=lambda x: tostamp(x.finish), reverse=True)
-
- SYPD = ((year_per_sim * len(list_considered) * seconds_in_a_day) /
- (total_run_time)) if total_run_time > 0 else 0
- SYPD = round(SYPD, 2)
- # Old
- # ASYPD = ((year_per_sim * len(list_considered) * seconds_in_a_day) /
- # (total_run_time + total_queue_time) if (total_run_time +
- # total_queue_time) > 0 else 0)
- # Paper Implementation
- # ASYPD = ((year_per_sim * len(list_considered) * seconds_in_a_day) / (max_sim_finish -
- # min_submit)) if (max_sim_finish - min_submit) > 0 else 0
- # ASYPD New Implementation
- ASYPD = (year_per_sim * len(sim_jobs_info_asypd) * seconds_in_a_day) / (sum(job.queue_time + job.run_time for job in sim_jobs_info_asypd) +
- sum(job.queue_time + job.run_time for job in post_jobs_info) / len(post_jobs_info)) if len(sim_jobs_info_asypd) > 0 and len(post_jobs_info) > 0 else 0
-
- # RSYPD
- RSYPD = 0
- if len(transfer_jobs_info) > 0:
- RSYPD = (year_per_sim * len(sim_jobs_info_rsypd) * seconds_in_a_day) / (tostamp(transfer_jobs_info[0].finish) - tostamp(sim_jobs_info_rsypd[-1].start)) if len(
- sim_jobs_info_rsypd) > 0 and len(transfer_jobs_info) > 0 and (tostamp(transfer_jobs_info[0].finish) - tostamp(sim_jobs_info_rsypd[-1].start)) > 0 else 0
- else:
- RSYPD = (year_per_sim * len(sim_jobs_info_rsypd) * seconds_in_a_day) / (tostamp(clean_jobs_info[0].finish) - tostamp(sim_jobs_info_rsypd[-1].start)) if len(
- sim_jobs_info_rsypd) > 0 and len(clean_jobs_info) > 0 and (tostamp(clean_jobs_info[0].finish) - tostamp(sim_jobs_info_rsypd[-1].start)) > 0 else 0
-
- ASYPD = round(ASYPD, 4)
- RSYPD = round(RSYPD, 4)
- CHSY = round(total_CHSY / len(list_considered),
- 2) if len(list_considered) > 0 else total_CHSY
- JPSY = round(
- total_JPSY / energy_count, 2) if energy_count > 0 else total_JPSY
- except Exception as ex:
- print(traceback.format_exc())
- error = True
- error_message = str(ex)
- pass
-
- return {"SYPD": SYPD,
- "ASYPD": ASYPD,
- "RSYPD": RSYPD,
- "CHSY": CHSY,
- "JPSY": JPSY,
- "Parallelization": Parallelization,
- "considered": list_considered,
- "error": error,
- "error_message": error_message,
- "warnings_job_data": warnings_job_data,
- }
-
-def get_experiment_graph_format_test(expid):
- """
- Some testing. Does not serve any purpose now, but the code might be useful.
- """
- base_list = dict()
- pkl_timestamp = 10000000
- try:
- notransitive = False
- print("Received " + str(expid))
- BasicConfig.read()
- exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid)
- tmp_path = os.path.join(exp_path, BasicConfig.LOCAL_TMP_DIR)
- as_conf = AutosubmitConfig(
- expid, BasicConfig, ConfigParserFactory())
- if not as_conf.check_conf_files():
- # Log.critical('Can not run with invalid configuration')
- raise Exception(
- 'Autosubmit GUI might not have permission to access necessary configuration files.')
-
- job_list = Autosubmit.load_job_list(
- expid, as_conf, notransitive=notransitive)
-
- job_list.sort_by_id()
- base_list = job_list.get_graph_representation(BasicConfig)
- except Exception as e:
- return {'nodes': [],
- 'edges': [],
- 'error': True,
- 'error_message': str(e),
- 'graphviz': False,
- 'max_children': 0,
- 'max_parents': 0,
- 'total_jobs': 0,
- 'pkl_timestamp': 0}
- name_to_id = dict()
- # name_to_weight = dict()
- list_nodes = list()
- list_edges = list()
- i = 0
- with open('/home/Earth/wuruchi/Documents/Personal/spectralgraph/data/graph_' + expid + '.txt', 'w') as the_file:
- for item in base_list['nodes']:
- # the_file.write(str(i) + "\n")
- name_to_id[item['id']] = i
- list_nodes.append(i)
- i += 1
- for item in base_list['edges']:
- the_file.write(str(name_to_id[item['from']]) + " " + str(
- name_to_id[item['to']]) + " " + ("10" if item['is_wrapper'] == True else "1") + "\n")
- list_edges.append(
- (name_to_id[item['from']], name_to_id[item['to']]))
- for item in base_list['fake_edges']:
- the_file.write(str(name_to_id[item['from']]) + " " + str(
- name_to_id[item['to']]) + " " + ("10" if item['is_wrapper'] == True else "1") + "\n")
- list_edges.append(
- (name_to_id[item['from']], name_to_id[item['to']]))
- return list_nodes, list_edges
-
-
-
-def update_running_experiments(time_condition=600):
- """
- Tests if an experiment is running and updates database as_times.db accordingly.\n
- :return: Nothing
- """
- t0 = time.time()
- experiment_to_modified_ts = dict() # type: Dict[str, int]
- try:
- BasicConfig.read()
- path = BasicConfig.LOCAL_ROOT_DIR
- # List of experiments from pkl
- tp0 = time.time()
- currentDirectories = subprocess.Popen(['ls', '-t', path],
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT) if (os.path.exists(path)) else None
- stdOut, stdErr = currentDirectories.communicate(
- ) if currentDirectories else (None, None)
-
- # Build connection to ecearth.db
- db_file = os.path.join(path, "ecearth.db")
- conn = DbRequests.create_connection(db_file)
- current_table = DbRequests.prepare_status_db()
- readingpkl = stdOut.split() if stdOut else []
-
- tp1 = time.time()
- for expid in readingpkl:
- pkl_path = os.path.join(path, expid, "pkl", "job_list_{0}.pkl".format(expid))
- if not len(expid) == 4 or not os.path.exists(pkl_path):
- continue
- t1 = time.time()
- # Timer safeguard
- if (t1 - t0) > SAFE_TIME_LIMIT_STATUS:
- raise Exception(
- "Time limit reached {0:06.2f} seconds on update_running_experiments while processing {1}. \
- Time spent on reading data {2:06.2f} seconds.".format((t1 - t0), expid, (tp1 - tp0)))
- current_stat = os.stat(pkl_path)
- time_diff = int(time.time()) - int(current_stat.st_mtime)
- if (time_diff < time_condition):
- experiment_to_modified_ts[expid] = time_diff
- if current_table.get(expid, None) is not None:
- # UPDATE RUNNING
- _exp_id, _status, _seconds = current_table[expid]
- if _status != "RUNNING":
- DbRequests.update_exp_status(
- expid, "RUNNING", time_diff)
- else:
- # Insert new experiment
- current_id = DbRequests.insert_experiment_status(
- conn, expid, time_diff)
- current_table[expid] = (
- current_id, 'RUNNING', time_diff)
- elif (time_diff <= 3600):
- # If it has been running in the last 1 hour
- # It must have been added before
- error, error_message, is_running, timediff, _ = _is_exp_running(
- expid)
- if is_running == True:
- if current_table.get(expid, None):
- _exp_id, _status, _seconds = current_table[expid]
- if _status != "RUNNING" and is_running == True:
- DbRequests.update_exp_status(
- expid, "RUNNING", _seconds)
- else:
- current_id = DbRequests.insert_experiment_status(
- conn, expid, time_diff)
- current_table[expid] = (
- current_id, 'RUNNING', time_diff)
-
- for expid in current_table:
- exp_id, status, seconds = current_table[expid]
- if status == "RUNNING" and experiment_to_modified_ts.get(expid, None) is None:
- # Perform exhaustive check
- error, error_message, is_running, timediff, _ = _is_exp_running(
- expid)
- # UPDATE NOT RUNNING
- if (is_running == False):
- # print("Update NOT RUNNING for " + expid)
- DbRequests.update_exp_status(
- expid, "NOT RUNNING", timediff)
- except Exception as e:
- # print(expid)
- print(e.message)
- # print(traceback.format_exc())
-
-def get_experiment_run_detail(expid, run_id):
- error = False
- error_message = ""
- result = None
- try:
- result = JobDataStructure(expid).get_current_run_job_data_json(run_id)
- tags = {"source": JobList.get_sourcetag(), "target": JobList.get_targettag(), "sync": JobList.get_synctag(), "check": JobList.get_checkmark(
- ), "completed": JobList.get_completed_tag(), "running_tag": JobList.get_running_tag(), "queuing_tag": JobList.get_queuing_tag(), "failed_tag": JobList.get_failed_tag()}
- except Exception as exp:
- error = True
- error_message = str(exp)
- pass
- return {"error": error, "error_message": error_message, "rundata": result}
\ No newline at end of file
diff --git a/autosubmit_api/experiment/test.py b/autosubmit_api/experiment/test.py
deleted file mode 100644
index d3e98f75b37feb4be7e9b5b49917f7566f1dc2e1..0000000000000000000000000000000000000000
--- a/autosubmit_api/experiment/test.py
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2015 Earth Sciences Department, BSC-CNS
-
-# This file is part of Autosubmit.
-
-# Autosubmit is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# Autosubmit is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with Autosubmit. If not, see .
-
-import unittest
-
-from experiment.common_requests import get_job_history, get_experiment_data
-
-class TestCommonRequests(unittest.TestCase):
- def setUp(self):
- pass
-
- def test_get_history(self):
- result = get_job_history("a3z4", "a3z4_19951101_fc8_1_SIM")
- print(result)
- self.assertTrue(result != None)
-
- def test_get_experiment_data(self):
- result = get_experiment_data("a29z")
- print(result)
- result2 = get_experiment_data("a4a0")
- print(result2)
- result3 = get_experiment_data("a2am")
- print(result3)
- self.assertTrue(result != None)
- self.assertTrue(result2 != None)
- self.assertTrue(result3 != None)
-
-if __name__ == "__main__":
- unittest.main()
\ No newline at end of file
diff --git a/autosubmit_api/history/database_managers/database_manager.py b/autosubmit_api/history/database_managers/database_manager.py
index 0bd01db3db68dcc4b09f25efa3d576cda72d735a..c7cb2d950dac36d35b963754000e8eeb2d8f1e12 100644
--- a/autosubmit_api/history/database_managers/database_manager.py
+++ b/autosubmit_api/history/database_managers/database_manager.py
@@ -16,12 +16,12 @@
# You should have received a copy of the GNU General Public License
# along with Autosubmit. If not, see .
-import pysqlite3 as sqlite3
+import sqlite3
import os
-from .. import utils as HUtils
-from . import database_models as Models
-from ...config.basicConfig import APIBasicConfig
-from abc import ABCMeta, abstractmethod
+from autosubmit_api.history import utils as HUtils
+from autosubmit_api.history.database_managers import database_models as Models
+from autosubmit_api.config.basicConfig import APIBasicConfig
+from abc import ABCMeta
DEFAULT_JOBDATA_DIR = os.path.join('/esarchive', 'autosubmit', 'as_metadata', 'data')
DEFAULT_HISTORICAL_LOGS_DIR = os.path.join('/esarchive', 'autosubmit', 'as_metadata', 'logs')
diff --git a/autosubmit_api/history/database_managers/experiment_history_db_manager.py b/autosubmit_api/history/database_managers/experiment_history_db_manager.py
index 6d03428da268cd30831df7d2e221a827eeee03a7..1f35a92154782623839b84dbe7c055e34abb4b45 100644
--- a/autosubmit_api/history/database_managers/experiment_history_db_manager.py
+++ b/autosubmit_api/history/database_managers/experiment_history_db_manager.py
@@ -15,16 +15,16 @@
# You should have received a copy of the GNU General Public License
# along with Autosubmit. If not, see .
-import pysqlite3 as sqlite3
import os
-import traceback
import textwrap
-from .. import utils as HUtils
-from .. database_managers import database_models as Models
-from .. data_classes.job_data import JobData
-from ..data_classes.experiment_run import ExperimentRun
-from ...config.basicConfig import APIBasicConfig
-from .database_manager import DatabaseManager, DEFAULT_JOBDATA_DIR
+
+from autosubmit_api.persistance.experiment import ExperimentPaths
+from autosubmit_api.history import utils as HUtils
+from autosubmit_api.history.database_managers import database_models as Models
+from autosubmit_api.history.data_classes.job_data import JobData
+from autosubmit_api.history.data_classes.experiment_run import ExperimentRun
+from autosubmit_api.config.basicConfig import APIBasicConfig
+from autosubmit_api.history.database_managers.database_manager import DatabaseManager
from typing import List
from collections import namedtuple
@@ -39,7 +39,8 @@ class ExperimentHistoryDbManager(DatabaseManager):
super(ExperimentHistoryDbManager, self).__init__(expid, basic_config)
self._set_schema_changes()
self._set_table_queries()
- self.historicaldb_file_path = os.path.join(self.JOBDATA_DIR, "job_data_{0}.db".format(self.expid)) # type : str
+ exp_paths = ExperimentPaths(expid)
+ self.historicaldb_file_path = exp_paths.job_data_db
if self.my_database_exists():
self.set_db_version_models()
@@ -176,10 +177,7 @@ class ExperimentHistoryDbManager(DatabaseManager):
def get_experiment_run_dc_with_max_id(self):
""" Get Current (latest) ExperimentRun data class. """
- if self.db_version >= Models.DatabaseVersion.EXPERIMENT_HEADER_SCHEMA_CHANGES.value:
- return ExperimentRun.from_model(self._get_experiment_run_with_max_id())
- else:
- return ExperimentRun(run_id=0)
+ return ExperimentRun.from_model(self._get_experiment_run_with_max_id())
def register_experiment_run_dc(self, experiment_run_dc):
self._insert_experiment_run(experiment_run_dc)
diff --git a/autosubmit_api/history/database_managers/experiment_status_db_manager.py b/autosubmit_api/history/database_managers/experiment_status_db_manager.py
deleted file mode 100644
index b7092a4653d6f9960a5db03a23d37426634dd41f..0000000000000000000000000000000000000000
--- a/autosubmit_api/history/database_managers/experiment_status_db_manager.py
+++ /dev/null
@@ -1,155 +0,0 @@
-#!/usr/bin/env python
-
-
-# Copyright 2015-2020 Earth Sciences Department, BSC-CNS
-# This file is part of Autosubmit.
-
-# Autosubmit is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# Autosubmit is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with Autosubmit. If not, see .
-
-import os
-import textwrap
-import time
-
-from autosubmit_api.experiment.common_db_requests import prepare_status_db
-from .database_manager import DatabaseManager, DEFAULT_LOCAL_ROOT_DIR
-from ...config.basicConfig import APIBasicConfig
-from ...history import utils as HUtils
-from ...history.database_managers import database_models as Models
-from typing import List
-
-class ExperimentStatusDbManager(DatabaseManager):
- """ Manages the actions on the status database """
- def __init__(self, expid, basic_config):
- # type: (str, APIBasicConfig) -> None
- super(ExperimentStatusDbManager, self).__init__(expid, basic_config)
- self._as_times_file_path = os.path.join(APIBasicConfig.DB_DIR, self.AS_TIMES_DB_NAME)
- self._ecearth_file_path = os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.DB_FILE)
- self._pkl_file_path = os.path.join(APIBasicConfig.LOCAL_ROOT_DIR, self.expid, "pkl", "job_list_{0}.pkl".format(self.expid))
- self.default_experiment_status_row = Models.ExperimentStatusRow(0, "DEFAULT", "NOT RUNNING", 0, "")
-
-
- def validate_status_database(self):
- """ Creates experiment_status table if it does not exist """
- prepare_status_db()
- # Redundant code
- # create_table_query = textwrap.dedent(
- # '''CREATE TABLE
- # IF NOT EXISTS experiment_status (
- # exp_id integer PRIMARY KEY,
- # name text NOT NULL,
- # status text NOT NULL,
- # seconds_diff integer NOT NULL,
- # modified text NOT NULL
- # );'''
- # )
- # self.execute_statement_on_dbfile(self._as_times_file_path, create_table_query)
-
- def print_current_table(self):
- for experiment in self._get_experiment_status_content():
- print(experiment)
- if self.current_experiment_status_row:
- print(("Current Row:\n\t" + self.current_experiment_status_row.name + "\n\t" +
- str(self.current_experiment_status_row.exp_id) + "\n\t" + self.current_experiment_status_row.status))
-
- def get_experiment_table_content(self):
- # type: () -> List[Models.ExperimentStatusRow]
- return self._get_experiment_status_content()
-
- def is_running(self, time_condition=600):
- # type : (int) -> bool
- """ True if experiment is running, False otherwise. """
- if (os.path.exists(self._pkl_file_path)):
- current_stat = os.stat(self._pkl_file_path)
- timest = int(current_stat.st_mtime)
- timesys = int(time.time())
- time_diff = int(timesys - timest)
- if (time_diff < time_condition):
- return True
- else:
- return False
- return False
-
- def set_existing_experiment_status_as_running(self, expid):
- """ Set the experiment_status row as running. """
- self.update_exp_status(expid, Models.RunningStatus.RUNNING)
-
- def create_experiment_status_as_running(self, experiment):
- """ Create a new experiment_status row for the Models.Experiment item."""
- self.create_exp_status(experiment.id, experiment.name, Models.RunningStatus.RUNNING)
-
-
- def get_experiment_status_row_by_expid(self, expid):
- # type: (str) -> Models.ExperimentStatusRow | None
- """
- Get Models.ExperimentStatusRow by expid. Uses exp_id (int id) as an intermediate step and validation.
- """
- experiment_row = self.get_experiment_row_by_expid(expid)
- return self.get_experiment_status_row_by_exp_id(experiment_row.id) if experiment_row else None
-
- def get_experiment_row_by_expid(self, expid):
- # type: (str) -> Models.ExperimentRow | None
- """
- Get the experiment from ecearth.db by expid as Models.ExperimentRow.
- """
- statement = self.get_built_select_statement("experiment", "name=?")
- current_rows = self.get_from_statement_with_arguments(self._ecearth_file_path, statement, (expid,))
- if len(current_rows) <= 0:
- return None
- # raise ValueError("Experiment {0} not found in {1}".format(expid, self._ecearth_file_path))
- return Models.ExperimentRow(*current_rows[0])
-
- def _get_experiment_status_content(self):
- # type: () -> List[Models.ExperimentStatusRow]
- """
- Get all registers from experiment_status as List of Models.ExperimentStatusRow.\n
- """
- statement = self.get_built_select_statement("experiment_status")
- current_rows = self.get_from_statement(self._as_times_file_path, statement)
- return [Models.ExperimentStatusRow(*row) for row in current_rows]
-
- def get_experiment_status_row_by_exp_id(self, exp_id):
- # type: (int) -> Models.ExperimentStatusRow
- """ Get Models.ExperimentStatusRow from as_times.db by exp_id (int)"""
- statement = self.get_built_select_statement("experiment_status", "exp_id=?")
- arguments = (exp_id,)
- current_rows = self.get_from_statement_with_arguments(self._as_times_file_path, statement, arguments)
- if len(current_rows) <= 0:
- return None
- return Models.ExperimentStatusRow(*current_rows[0])
-
-
- def create_exp_status(self, exp_id, expid, status):
- # type: (int, str, str) -> None
- """
- Create experiment status
- """
- statement = ''' INSERT INTO experiment_status(exp_id, name, status, seconds_diff, modified) VALUES(?,?,?,?,?) '''
- arguments = (exp_id, expid, status, 0, HUtils.get_current_datetime())
- return self.insert_statement_with_arguments(self._as_times_file_path, statement, arguments)
-
- def update_exp_status(self, expid, status="RUNNING"):
- # type: (str, str) -> None
- """
- Update status, seconds_diff, modified in experiment_status.
- """
- statement = ''' UPDATE experiment_status SET status = ?, seconds_diff = ?, modified = ? WHERE name = ? '''
- arguments = (status, 0, HUtils.get_current_datetime(), expid)
- self.execute_statement_with_arguments_on_dbfile(self._as_times_file_path, statement, arguments)
-
- def delete_exp_status(self, expid):
- # type: (str) -> None
- """ Deletes experiment_status row by expid. Useful for testing purposes. """
- statement = ''' DELETE FROM experiment_status where name = ? '''
- arguments = (expid,)
- self.execute_statement_with_arguments_on_dbfile(self._as_times_file_path, statement, arguments)
\ No newline at end of file
diff --git a/autosubmit_api/history/database_managers/test.py b/autosubmit_api/history/database_managers/test.py
deleted file mode 100644
index 1b88b37aa04a6eac228ed2cf363b3ff07addc727..0000000000000000000000000000000000000000
--- a/autosubmit_api/history/database_managers/test.py
+++ /dev/null
@@ -1,261 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2015-2020 Earth Sciences Department, BSC-CNS
-# This file is part of Autosubmit.
-
-# Autosubmit is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# Autosubmit is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with Autosubmit. If not, see .
-
-import unittest
-import time
-import random
-import os
-from shutil import copy2
-from autosubmit_api.history.database_managers.experiment_history_db_manager import ExperimentHistoryDbManager
-from autosubmit_api.history.database_managers.experiment_status_db_manager import ExperimentStatusDbManager
-# from experiment_status_db_manager import ExperimentStatusDbManager
-from autosubmit_api.history.data_classes.experiment_run import ExperimentRun
-from autosubmit_api.history.data_classes.job_data import JobData
-import autosubmit_api.history.database_managers.database_models as Models
-from autosubmit_api.config.basicConfig import APIBasicConfig
-import autosubmit_api.history.utils as HUtils
-
-EXPID_TT00_SOURCE = "test_database.db~"
-EXPID_TT01_SOURCE = "test_database_no_run.db~"
-EXPID = "t024"
-EXPID_NONE = "t027"
-# BasicConfig.read()
-JOBDATA_DIR = APIBasicConfig.JOBDATA_DIR
-LOCAL_ROOT_DIR = APIBasicConfig.LOCAL_ROOT_DIR
-
-class TestExperimentStatusDatabaseManager(unittest.TestCase):
- """ Covers Experiment Status Database Manager """
- def setUp(self):
- APIBasicConfig.read()
- self.exp_status_db = ExperimentStatusDbManager(EXPID, APIBasicConfig)
- if self.exp_status_db.get_experiment_status_row_by_expid(EXPID) == None:
- self.exp_status_db.create_exp_status(7375, "t024", "NOT RUNNING")
-
-
- def test_get_current_experiment_status_row(self):
- exp_status_row = self.exp_status_db.get_experiment_status_row_by_expid(EXPID)
- self.assertIsNotNone(exp_status_row)
- exp_status_row_none = self.exp_status_db.get_experiment_status_row_by_expid(EXPID_NONE)
- self.assertIsNone(exp_status_row_none)
- exp_row_direct = self.exp_status_db.get_experiment_status_row_by_exp_id(exp_status_row.exp_id)
- self.assertTrue(exp_status_row.exp_id == exp_row_direct.exp_id)
-
-
- def test_update_exp_status(self):
- self.exp_status_db.update_exp_status(EXPID, "RUNNING")
- exp_status_row_current = self.exp_status_db.get_experiment_status_row_by_expid(EXPID)
- self.assertTrue(exp_status_row_current.status == "RUNNING")
- self.exp_status_db.update_exp_status(EXPID, "NOT RUNNING")
- exp_status_row_current = self.exp_status_db.get_experiment_status_row_by_expid(EXPID)
- self.assertTrue(exp_status_row_current.status == "NOT RUNNING")
-
- def test_create_exp_status(self):
- experiment = self.exp_status_db.get_experiment_row_by_expid(EXPID_NONE)
- self.exp_status_db.create_experiment_status_as_running(experiment)
- experiment_status = self.exp_status_db.get_experiment_status_row_by_expid(EXPID_NONE)
- self.assertIsNotNone(experiment_status)
- self.exp_status_db.delete_exp_status(EXPID_NONE)
- experiment_status = self.exp_status_db.get_experiment_status_row_by_expid(EXPID_NONE)
- self.assertIsNone(experiment_status)
-
-
-class TestExperimentHistoryDbManager(unittest.TestCase):
- """ Covers Experiment History Database Manager and Data Models """
- def setUp(self):
- APIBasicConfig.read()
- source_path_tt00 = os.path.join(JOBDATA_DIR, EXPID_TT00_SOURCE)
- self.target_path_tt00 = os.path.join(JOBDATA_DIR, "job_data_{0}.db".format(EXPID))
- copy2(source_path_tt00, self.target_path_tt00)
- source_path_tt01 = os.path.join(JOBDATA_DIR, EXPID_TT01_SOURCE)
- self.target_path_tt01 = os.path.join(JOBDATA_DIR, "job_data_{0}.db".format(EXPID_NONE))
- copy2(source_path_tt01, self.target_path_tt01)
- self.experiment_database = ExperimentHistoryDbManager(EXPID, APIBasicConfig)
- self.experiment_database.initialize()
-
- def tearDown(self):
- os.remove(self.target_path_tt00)
- os.remove(self.target_path_tt01)
-
- def test_get_max_id(self):
- max_item = self.experiment_database.get_experiment_run_dc_with_max_id()
- self.assertTrue(max_item.run_id > 0)
- self.assertTrue(max_item.run_id >= 18) # Max is 18
-
- def test_pragma(self):
- self.assertTrue(self.experiment_database._get_pragma_version() == Models.DatabaseVersion.CURRENT_DB_VERSION.value) # Update version on changes
-
- def test_get_job_data(self):
- job_data = self.experiment_database._get_job_data_last_by_name("a29z_20000101_fc0_1_SIM")
- self.assertTrue(len(job_data) > 0)
- self.assertTrue(job_data[0].last == 1)
- self.assertTrue(job_data[0].job_name == "a29z_20000101_fc0_1_SIM")
-
- job_data = self.experiment_database.get_job_data_dcs_by_name("a29z_20000101_fc0_1_SIM")
- self.assertTrue(job_data[0].job_name == "a29z_20000101_fc0_1_SIM")
-
- job_data = self.experiment_database._get_job_data_last_by_run_id(18) # Latest
- self.assertTrue(len(job_data) > 0)
-
- job_data = self.experiment_database._get_job_data_last_by_run_id_and_finished(18)
- self.assertTrue(len(job_data) > 0)
-
- job_data = self.experiment_database.get_job_data_all()
- self.assertTrue(len(job_data) > 0)
-
- def test_insert_and_delete_experiment_run(self):
- new_run = ExperimentRun(19)
- new_id = self.experiment_database._insert_experiment_run(new_run)
- self.assertIsNotNone(new_id)
- last_experiment_run = self.experiment_database.get_experiment_run_dc_with_max_id()
- self.assertTrue(new_id == last_experiment_run.run_id)
- self.experiment_database.delete_experiment_run(new_id)
- last_experiment_run = self.experiment_database.get_experiment_run_dc_with_max_id()
- self.assertTrue(new_id != last_experiment_run.run_id)
-
- def test_insert_and_delete_job_data(self):
- max_run_id = self.experiment_database.get_experiment_run_dc_with_max_id().run_id
- new_job_data_name = "test_001_name_{0}".format(int(time.time()))
- new_job_data = JobData(_id=1, job_name=new_job_data_name, run_id=max_run_id)
- new_job_data_id = self.experiment_database._insert_job_data(new_job_data)
- self.assertIsNotNone(new_job_data_id)
- self.experiment_database.delete_job_data(new_job_data_id)
- job_data = self.experiment_database.get_job_data_dcs_by_name(new_job_data_name)
- self.assertTrue(len(job_data) == 0)
-
-
- def test_update_experiment_run(self):
- experiment_run_data_class = self.experiment_database.get_experiment_run_dc_with_max_id() # 18
- backup_run = self.experiment_database.get_experiment_run_dc_with_max_id()
- experiment_run_data_class.chunk_unit = "unouno"
- experiment_run_data_class.running = random.randint(1, 100)
- experiment_run_data_class.queuing = random.randint(1, 100)
- experiment_run_data_class.suspended = random.randint(1, 100)
- self.experiment_database._update_experiment_run(experiment_run_data_class)
- last_experiment_run = self.experiment_database.get_experiment_run_dc_with_max_id() # 18
- self.assertTrue(self.experiment_database.db_version == Models.DatabaseVersion.CURRENT_DB_VERSION.value)
- self.assertTrue(last_experiment_run.chunk_unit == experiment_run_data_class.chunk_unit)
- self.assertTrue(last_experiment_run.running == experiment_run_data_class.running)
- self.assertTrue(last_experiment_run.queuing == experiment_run_data_class.queuing)
- self.assertTrue(last_experiment_run.suspended == experiment_run_data_class.suspended)
- self.experiment_database._update_experiment_run(backup_run)
-
- def test_job_data_from_model(self):
- job_data_rows = self.experiment_database._get_job_data_last_by_name("a29z_20000101_fc0_1_SIM")
- job_data_row_first = job_data_rows[0]
- job_data_data_class = JobData.from_model(job_data_row_first)
- self.assertTrue(job_data_row_first.job_name == job_data_data_class.job_name)
-
- def test_update_job_data_processed(self):
- current_time = time.time()
- job_data_rows = self.experiment_database._get_job_data_last_by_name("a29z_20000101_fc0_1_SIM")
- job_data_row_first = job_data_rows[0]
- job_data_data_class = JobData.from_model(job_data_row_first)
- backup_job_dc = JobData.from_model(job_data_row_first)
- job_data_data_class.nnodes = random.randint(1, 1000)
- job_data_data_class.ncpus = random.randint(1, 1000)
- job_data_data_class.status = "DELAYED"
- job_data_data_class.finish = current_time
- self.experiment_database._update_job_data_by_id(job_data_data_class)
- job_data_rows_current = self.experiment_database._get_job_data_last_by_name("a29z_20000101_fc0_1_SIM")
- job_data_row_first = job_data_rows_current[0]
- self.assertTrue(job_data_row_first.nnodes == job_data_data_class.nnodes)
- self.assertTrue(job_data_row_first.ncpus == job_data_data_class.ncpus)
- self.assertTrue(job_data_row_first.status == job_data_data_class.status)
- self.assertTrue(job_data_row_first.finish == job_data_data_class.finish)
- self.experiment_database._update_job_data_by_id(backup_job_dc)
-
- def test_bulk_update(self):
- current_time = time.time()
- all_job_data_rows = self.experiment_database.get_job_data_all()
- job_data_rows_test = [job for job in all_job_data_rows if job.run_id == 3]
- backup = [JobData.from_model(job) for job in job_data_rows_test]
- list_job_data_class = [JobData.from_model(job) for job in job_data_rows_test]
- backup_changes = [(HUtils.get_current_datetime(), job.status, job.rowstatus, job._id) for job in list_job_data_class]
- changes = [(HUtils.get_current_datetime(), "DELAYED", job.rowstatus, job._id) for job in list_job_data_class]
- self.experiment_database.update_many_job_data_change_status(changes)
- all_job_data_rows = self.experiment_database.get_job_data_all()
- job_data_rows_validate = [job for job in all_job_data_rows if job.run_id == 3]
- for (job_val, change_item) in zip(job_data_rows_validate, changes):
- modified, status, rowstatus, _id = change_item
- # self.assertTrue(job_val.finish == finish)
- self.assertTrue(job_val.modified == modified)
- self.assertTrue(job_val.status == status)
- self.assertTrue(job_val.rowstatus == rowstatus)
- self.assertTrue(job_val.id == _id)
- self.experiment_database.update_many_job_data_change_status(backup_changes)
-
- def test_job_data_maxcounter(self):
- new_job_data = ExperimentHistoryDbManager(EXPID_NONE, APIBasicConfig)
- new_job_data.initialize()
- max_empty_table_counter = new_job_data.get_job_data_max_counter()
- self.assertTrue(max_empty_table_counter == 0)
- max_existing_counter = self.experiment_database.get_job_data_max_counter()
- self.assertTrue(max_existing_counter > 0)
-
- def test_register_submitted_job_data_dc(self):
- job_data_dc = self.experiment_database.get_job_data_dc_unique_latest_by_job_name("a29z_20000101_fc0_1_SIM")
- max_counter = self.experiment_database.get_job_data_max_counter()
- self.assertTrue(max_counter > 0)
- self.assertTrue(job_data_dc.counter > 0)
- next_counter = max(max_counter, job_data_dc.counter + 1)
- self.assertTrue(next_counter >= max_counter)
- self.assertTrue(next_counter >= job_data_dc.counter + 1)
- job_data_dc.counter = next_counter
- job_data_dc_current = self.experiment_database.register_submitted_job_data_dc(job_data_dc)
- self.assertTrue(job_data_dc._id < job_data_dc_current._id)
- job_data_last_list = self.experiment_database._get_job_data_last_by_name(job_data_dc.job_name)
- self.assertTrue(len(job_data_last_list) == 1)
- self.experiment_database.delete_job_data(job_data_last_list[0].id)
- job_data_dc.last = 1
- updated_job_data_dc = self.experiment_database.update_job_data_dc_by_id(job_data_dc)
- self.assertTrue(job_data_dc._id == updated_job_data_dc._id)
- job_data_dc = self.experiment_database.get_job_data_dc_unique_latest_by_job_name("a29z_20000101_fc0_1_SIM")
- self.assertTrue(job_data_dc._id == updated_job_data_dc._id)
-
- def test_update_children_and_platform_output(self):
- job_data_dc = self.experiment_database.get_job_data_dc_unique_latest_by_job_name("a29z_20000101_fc0_1_SIM")
- children_str = "a00, a01, a02"
- platform_output_str = " SLURM OUTPUT "
- job_data_dc.children = children_str
- job_data_dc.platform_output = platform_output_str
- self.assertTrue(self.experiment_database.db_version == Models.DatabaseVersion.CURRENT_DB_VERSION.value)
- self.experiment_database.update_job_data_dc_by_id(job_data_dc)
- job_data_dc_updated = self.experiment_database.get_job_data_dc_unique_latest_by_job_name("a29z_20000101_fc0_1_SIM")
- self.assertTrue(job_data_dc_updated.children == children_str)
- self.assertTrue(job_data_dc_updated.platform_output == platform_output_str)
- # Back to normal
- job_data_dc.children = ""
- job_data_dc.platform_output = "NO OUTPUT"
- self.experiment_database.update_job_data_dc_by_id(job_data_dc)
- job_data_dc_updated = self.experiment_database.get_job_data_dc_unique_latest_by_job_name("a29z_20000101_fc0_1_SIM")
- self.assertTrue(job_data_dc_updated.children == "")
- self.assertTrue(job_data_dc_updated.platform_output == "NO OUTPUT")
-
-
-
- def test_experiment_run_dc(self):
- experiment_run = self.experiment_database.get_experiment_run_dc_with_max_id()
- self.assertIsNotNone(experiment_run)
-
- def test_if_database_not_exists(self):
- exp_manager = ExperimentHistoryDbManager("0000", APIBasicConfig)
- self.assertTrue(exp_manager.my_database_exists() == False)
-
-
-if __name__ == '__main__':
- unittest.main()
\ No newline at end of file
diff --git a/autosubmit_api/history/experiment_history.py b/autosubmit_api/history/experiment_history.py
index f36521d25b3d9fbb1aeb942d64ad8101936159b9..f9ae4231d0196aaabc39f9c1b28ada0fe70a2a69 100644
--- a/autosubmit_api/history/experiment_history.py
+++ b/autosubmit_api/history/experiment_history.py
@@ -15,20 +15,14 @@
# You should have received a copy of the GNU General Public License
# along with Autosubmit. If not, see .
-import os
import traceback
-from .database_managers import database_models as Models
-from ..history import utils as HUtils
-from ..performance import utils as PUtils
-from time import time, sleep
-from ..history.database_managers.experiment_history_db_manager import ExperimentHistoryDbManager
-from ..history.database_managers.database_manager import DEFAULT_JOBDATA_DIR, DEFAULT_HISTORICAL_LOGS_DIR
-from ..history.strategies import PlatformInformationHandler, SingleAssociationStrategy, StraightWrapperAssociationStrategy, TwoDimWrapperDistributionStrategy, GeneralizedWrapperDistributionStrategy
-from ..history.data_classes.job_data import JobData
-from ..history.data_classes.experiment_run import ExperimentRun
-from ..history.platform_monitor.slurm_monitor import SlurmMonitor
-from ..history.internal_logging import Logging
-from ..config.basicConfig import APIBasicConfig
+from autosubmit_api.history.database_managers import database_models as Models
+from autosubmit_api.performance import utils as PUtils
+from autosubmit_api.history.database_managers.experiment_history_db_manager import ExperimentHistoryDbManager
+from autosubmit_api.history.data_classes.job_data import JobData
+from autosubmit_api.history.data_classes.experiment_run import ExperimentRun
+from autosubmit_api.history.internal_logging import Logging
+from autosubmit_api.config.basicConfig import APIBasicConfig
from typing import List, Dict, Tuple, Any
SECONDS_WAIT_PLATFORM = 60
@@ -47,133 +41,11 @@ class ExperimentHistory():
self._log.log(str(exp), traceback.format_exc())
self.manager = None
- def initialize_database(self):
- try:
- self.manager.initialize()
- except Exception as exp:
- self._log.log(str(exp), traceback.format_exc())
- self.manager = None
-
def is_header_ready(self):
if self.manager:
return self.manager.is_header_ready_db_version()
return False
-
- def write_submit_time(self, job_name, submit=0, status="UNKNOWN", ncpus=0, wallclock="00:00", qos="debug", date="",
- member="", section="", chunk=0, platform="NA", job_id=0, wrapper_queue=None, wrapper_code=None, children=""):
- try:
- next_counter = self._get_next_counter_by_job_name(job_name)
- current_experiment_run = self.manager.get_experiment_run_dc_with_max_id()
- job_data_dc = JobData(_id=0,
- counter=next_counter,
- job_name=job_name,
- submit=submit,
- status=status,
- rowtype=self._get_defined_rowtype(wrapper_code),
- ncpus=ncpus,
- wallclock=wallclock,
- qos=self._get_defined_queue_name(wrapper_queue, wrapper_code, qos),
- date=date,
- member=member,
- section=section,
- chunk=chunk,
- platform=platform,
- job_id=job_id,
- children=children,
- run_id=current_experiment_run.run_id)
- return self.manager.register_submitted_job_data_dc(job_data_dc)
- except Exception as exp:
- self._log.log(str(exp), traceback.format_exc())
- return None
-
- def write_start_time(self, job_name, start=0, status="UNKWOWN", ncpus=0, wallclock="00:00", qos="debug", date="",
- member="", section="", chunk=0, platform="NA", job_id=0, wrapper_queue=None, wrapper_code=None, children=""):
- try:
- job_data_dc_last = self.manager.get_job_data_dc_unique_latest_by_job_name(job_name)
- if not job_data_dc_last:
- job_data_dc_last = self.write_submit_time(job_name=job_name,
- status=status,
- ncpus=ncpus,
- wallclock=wallclock,
- qos=qos,
- date=date,
- member=member,
- section=section,
- chunk=chunk,
- platform=platform,
- job_id=job_id,
- wrapper_queue=wrapper_queue,
- wrapper_code=wrapper_code)
- self._log.log("write_start_time {0} start not found.".format(job_name))
- job_data_dc_last.start = start
- job_data_dc_last.qos = self._get_defined_queue_name(wrapper_queue, wrapper_code, qos)
- job_data_dc_last.status = status
- job_data_dc_last.rowtype = self._get_defined_rowtype(wrapper_code)
- job_data_dc_last.job_id = job_id
- job_data_dc_last.children = children
- return self.manager.update_job_data_dc_by_id(job_data_dc_last)
- except Exception as exp:
- self._log.log(str(exp), traceback.format_exc())
-
- def write_finish_time(self, job_name, finish=0, status="UNKNOWN", ncpus=0, wallclock="00:00", qos="debug", date="",
- member="", section="", chunk=0, platform="NA", job_id=0, out_file=None, err_file=None,
- wrapper_queue=None, wrapper_code=None, children=""):
- try:
- job_data_dc_last = self.manager.get_job_data_dc_unique_latest_by_job_name(job_name)
- if not job_data_dc_last:
- job_data_dc_last = self.write_submit_time(job_name=job_name,
- status=status,
- ncpus=ncpus,
- wallclock=wallclock,
- qos=qos,
- date=date,
- member=member,
- section=section,
- chunk=chunk,
- platform=platform,
- job_id=job_id,
- wrapper_queue=wrapper_queue,
- wrapper_code=wrapper_code,
- children=children)
- self._log.log("write_finish_time {0} submit not found.".format(job_name))
- job_data_dc_last.finish = finish if finish > 0 else int(time())
- job_data_dc_last.status = status
- job_data_dc_last.job_id = job_id
- job_data_dc_last.rowstatus = Models.RowStatus.PENDING_PROCESS
- job_data_dc_last.out = out_file if out_file else ""
- job_data_dc_last.err = err_file if err_file else ""
- return self.manager.update_job_data_dc_by_id(job_data_dc_last)
- except Exception as exp:
- self._log.log(str(exp), traceback.format_exc())
-
- def write_platform_data_after_finish(self, job_data_dc, platform_obj):
- """
- Call it in a thread.
- """
- try:
- sleep(SECONDS_WAIT_PLATFORM)
- ssh_output = platform_obj.check_job_energy(job_data_dc.job_id)
- slurm_monitor = SlurmMonitor(ssh_output)
- self._verify_slurm_monitor(slurm_monitor, job_data_dc)
- job_data_dcs_in_wrapper = self.manager.get_job_data_dcs_last_by_wrapper_code(job_data_dc.wrapper_code)
- job_data_dcs_to_update = []
- if len(job_data_dcs_in_wrapper) > 0:
- info_handler = PlatformInformationHandler(StraightWrapperAssociationStrategy(self._historiclog_dir_path))
- job_data_dcs_to_update = info_handler.execute_distribution(job_data_dc, job_data_dcs_in_wrapper, slurm_monitor)
- if len(job_data_dcs_to_update) == 0:
- info_handler.strategy = TwoDimWrapperDistributionStrategy(self._historiclog_dir_path)
- job_data_dcs_to_update = info_handler.execute_distribution(job_data_dc, job_data_dcs_in_wrapper, slurm_monitor)
- if len(job_data_dcs_to_update) == 0:
- info_handler.strategy = GeneralizedWrapperDistributionStrategy(self._historiclog_dir_path)
- job_data_dcs_to_update = info_handler.execute_distribution(job_data_dc, job_data_dcs_in_wrapper, slurm_monitor)
- else:
- info_handler = PlatformInformationHandler(SingleAssociationStrategy(self._historiclog_dir_path))
- job_data_dcs_to_update = info_handler.execute_distribution(job_data_dc, job_data_dcs_in_wrapper, slurm_monitor)
- return self.manager.update_list_job_data_dc_by_each_id(job_data_dcs_to_update)
- except Exception as exp:
- self._log.log(str(exp), traceback.format_exc())
-
def get_historic_job_data(self, job_name):
# type: (str) -> List[Dict[str, Any]]
result = []
@@ -231,166 +103,3 @@ class ExperimentHistory():
"err": job_data_dc.err
})
return result
-
-
- def update_job_finish_time_if_zero(self, job_name, finish_ts):
- # type: (str, int) -> JobData
- try:
- job_data_dc = self.manager.get_job_data_dc_unique_latest_by_job_name(job_name)
- if job_data_dc and job_data_dc.finish == 0:
- job_data_dc.finish = finish_ts
- return self.manager.update_job_data_dc_by_id(job_data_dc)
- except Exception as exp:
- self._log.log(str(exp), traceback.format_exc())
-
- def _verify_slurm_monitor(self, slurm_monitor, job_data_dc):
- try:
- if slurm_monitor.header.status not in ["COMPLETED", "FAILED"]:
- self._log.log("Assertion Error on job {0} with ssh_output {1}".format(job_data_dc.job_name, slurm_monitor.original_input),
- "Slurm status {0} is not COMPLETED nor FAILED for ID {1}.\n".format(slurm_monitor.header.status, slurm_monitor.header.name))
- if not slurm_monitor.steps_plus_extern_approximate_header_energy():
- self._log.log("Assertion Error on job {0} with ssh_output {1}".format(job_data_dc.job_name, slurm_monitor.original_input),
- "Steps + extern != total energy for ID {0}. Number of steps {1}.\n".format(slurm_monitor.header.name, slurm_monitor.step_count))
- except Exception as exp:
- self._log.log(str(exp), traceback.format_exc())
-
- def process_status_changes(self, job_list=None, chunk_unit="NA", chunk_size=0, current_config=""):
- """ Detect status differences between job_list and current job_data rows, and update. Creates a new run if necessary. """
- try:
- current_experiment_run_dc = self.manager.get_experiment_run_dc_with_max_id()
- update_these_changes = self._get_built_list_of_changes(job_list)
- should_create_new_run = self.should_we_create_a_new_run(job_list, len(update_these_changes), current_experiment_run_dc, chunk_unit, chunk_size)
- if len(update_these_changes) > 0 and should_create_new_run == False:
- self.manager.update_many_job_data_change_status(update_these_changes)
- if should_create_new_run:
- return self.create_new_experiment_run(chunk_unit, chunk_size, current_config, job_list)
- return self.update_counts_on_experiment_run_dc(current_experiment_run_dc, job_list)
- except Exception as exp:
- self._log.log(str(exp), traceback.format_exc())
-
- def _get_built_list_of_changes(self, job_list):
- """ Return: List of (current timestamp, current datetime str, status, rowstatus, id in job_data). One tuple per change. """
- job_data_dcs = self.detect_changes_in_job_list(job_list)
- return [(HUtils.get_current_datetime(), job.status, Models.RowStatus.CHANGED, job._id) for job in job_data_dcs]
-
- def process_job_list_changes_to_experiment_totals(self, job_list=None):
- """ Updates current experiment_run row with totals calculated from job_list. """
- try:
- current_experiment_run_dc = self.manager.get_experiment_run_dc_with_max_id()
- return self.update_counts_on_experiment_run_dc(current_experiment_run_dc, job_list)
- except Exception as exp:
- self._log.log(str(exp), traceback.format_exc())
-
- def should_we_create_a_new_run(self, job_list, changes_count, current_experiment_run_dc, new_chunk_unit, new_chunk_size):
- if len(job_list) != current_experiment_run_dc.total:
- return True
- if changes_count > int(self._get_date_member_completed_count(job_list)):
- return True
- return self._chunk_config_has_changed(current_experiment_run_dc, new_chunk_unit, new_chunk_size)
-
- def _chunk_config_has_changed(self, current_exp_run_dc, new_chunk_unit, new_chunk_size):
- if not current_exp_run_dc:
- return True
- if current_exp_run_dc.chunk_unit != new_chunk_unit or current_exp_run_dc.chunk_size != new_chunk_size:
- return True
- return False
-
- def update_counts_on_experiment_run_dc(self, experiment_run_dc, job_list=None):
- """ Return updated row as Models.ExperimentRun. """
- status_counts = self.get_status_counts_from_job_list(job_list)
- experiment_run_dc.completed = status_counts[HUtils.SupportedStatus.COMPLETED]
- experiment_run_dc.failed = status_counts[HUtils.SupportedStatus.FAILED]
- experiment_run_dc.queuing = status_counts[HUtils.SupportedStatus.QUEUING]
- experiment_run_dc.submitted = status_counts[HUtils.SupportedStatus.SUBMITTED]
- experiment_run_dc.running = status_counts[HUtils.SupportedStatus.RUNNING]
- experiment_run_dc.suspended = status_counts[HUtils.SupportedStatus.SUSPENDED]
- experiment_run_dc.total = status_counts["TOTAL"]
- return self.manager.update_experiment_run_dc_by_id(experiment_run_dc)
-
- def finish_current_experiment_run(self):
- if self.manager.is_there_a_last_experiment_run():
- current_experiment_run_dc = self.manager.get_experiment_run_dc_with_max_id()
- current_experiment_run_dc.finish = int(time())
- return self.manager.update_experiment_run_dc_by_id(current_experiment_run_dc)
- return None
-
- def create_new_experiment_run(self, chunk_unit="NA", chunk_size=0, current_config="", job_list=None):
- """ Also writes the finish timestamp of the previous run. """
- self.finish_current_experiment_run()
- return self._create_new_experiment_run_dc_with_counts(chunk_unit=chunk_unit, chunk_size=chunk_size, current_config=current_config, job_list=job_list)
-
- def _create_new_experiment_run_dc_with_counts(self, chunk_unit, chunk_size, current_config="", job_list=None):
- """ Create new experiment_run row and return the new Models.ExperimentRun data class from database. """
- status_counts = self.get_status_counts_from_job_list(job_list)
- experiment_run_dc = ExperimentRun(0,
- chunk_unit=chunk_unit,
- chunk_size=chunk_size,
- metadata=current_config,
- start=int(time()),
- completed=status_counts[HUtils.SupportedStatus.COMPLETED],
- total=status_counts["TOTAL"],
- failed=status_counts[HUtils.SupportedStatus.FAILED],
- queuing=status_counts[HUtils.SupportedStatus.QUEUING],
- running=status_counts[HUtils.SupportedStatus.RUNNING],
- submitted=status_counts[HUtils.SupportedStatus.SUBMITTED],
- suspended=status_counts[HUtils.SupportedStatus.SUSPENDED])
- return self.manager.register_experiment_run_dc(experiment_run_dc)
-
- def detect_changes_in_job_list(self, job_list):
- """ Detect changes in job_list compared to the current contents of job_data table. Returns a list of JobData data classes where the status of each item is the new status."""
- job_name_to_job = {job.name: job for job in job_list}
- current_job_data_dcs = self.manager.get_all_last_job_data_dcs()
- differences = []
- for job_dc in current_job_data_dcs:
- if job_dc.job_name in job_name_to_job and job_dc.status != job_name_to_job[job_dc.job_name].status_str:
- job_dc.status = job_name_to_job[job_dc.job_name].status_str
- differences.append(job_dc)
- return differences
-
- def _get_defined_rowtype(self, code):
- if code:
- return code
- else:
- return Models.RowType.NORMAL
-
- def _get_defined_queue_name(self, wrapper_queue, wrapper_code, qos):
- if wrapper_code and wrapper_code > 2 and wrapper_queue is not None:
- return wrapper_queue
- return qos
-
- def _get_next_counter_by_job_name(self, job_name):
- """ Return the counter attribute from the latest job data row by job_name. """
- job_data_dc = self.manager.get_job_data_dc_unique_latest_by_job_name(job_name)
- max_counter = self.manager.get_job_data_max_counter()
- if job_data_dc:
- return max(max_counter, job_data_dc.counter + 1)
- else:
- return max_counter
-
- def _get_date_member_completed_count(self, job_list):
- """ Each item in the job_list must have attributes: date, member, status_str. """
- job_list = job_list if job_list else []
- return sum(1 for job in job_list if job.date is not None and job.member is not None and job.status_str == HUtils.SupportedStatus.COMPLETED)
-
- def get_status_counts_from_job_list(self, job_list):
- """
- Return dict with keys COMPLETED, FAILED, QUEUING, SUBMITTED, RUNNING, SUSPENDED, TOTAL.
- """
- result = {
- HUtils.SupportedStatus.COMPLETED: 0,
- HUtils.SupportedStatus.FAILED: 0,
- HUtils.SupportedStatus.QUEUING: 0,
- HUtils.SupportedStatus.SUBMITTED: 0,
- HUtils.SupportedStatus.RUNNING: 0,
- HUtils.SupportedStatus.SUSPENDED: 0,
- "TOTAL": 0
- }
-
- if not job_list:
- job_list = []
-
- for job in job_list:
- if job.status_str in result:
- result[job.status_str] += 1
- result["TOTAL"] = len(job_list)
- return result
diff --git a/autosubmit_api/history/experiment_status.py b/autosubmit_api/history/experiment_status.py
deleted file mode 100644
index d9046aaab3adbc776fb7cd846ab688dd9c50f339..0000000000000000000000000000000000000000
--- a/autosubmit_api/history/experiment_status.py
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2015-2020 Earth Sciences Department, BSC-CNS
-# This file is part of Autosubmit.
-
-# Autosubmit is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# Autosubmit is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with Autosubmit. If not, see .
-
-import traceback
-from .database_managers.experiment_status_db_manager import ExperimentStatusDbManager
-from .database_managers.database_manager import DEFAULT_LOCAL_ROOT_DIR, DEFAULT_HISTORICAL_LOGS_DIR
-from .internal_logging import Logging
-from ..config.basicConfig import APIBasicConfig
-from typing import List
-from .database_managers.database_models import ExperimentStatusRow
-
-class ExperimentStatus():
- """ Represents the Experiment Status Mechanism that keeps track of currently active experiments """
- def __init__(self, expid):
- # type: (str) -> None
- self.expid = expid # type: str
- print(expid)
- APIBasicConfig.read()
- try:
- self.manager = ExperimentStatusDbManager(self.expid, APIBasicConfig)
- except Exception as exp:
- message = "Error while trying to update {0} in experiment_status.".format(str(self.expid))
- print(message)
- print(str(exp))
- print()
- Logging(self.expid, APIBasicConfig).log(message, traceback.format_exc())
- self.manager = None
-
- def validate_database(self):
- # type: () -> None
- self.manager.validate_status_database()
-
- def get_current_table_content(self):
- # type: () -> List[ExperimentStatusRow]
- return self.manager.get_experiment_table_content()
-
- def set_as_running(self):
- # type: () -> ExperimentStatusRow
- """ Set the status of the experiment in experiment_status of as_times.db as RUNNING. Inserts row if necessary."""
- if not self.manager:
- raise Exception("ExperimentStatus: The database manager is not available.")
- exp_status_row = self.manager.get_experiment_status_row_by_expid(self.expid)
- if exp_status_row:
- self.manager.set_existing_experiment_status_as_running(exp_status_row.name)
- return self.manager.get_experiment_status_row_by_expid(self.expid)
- else:
- exp_row = self.manager.get_experiment_row_by_expid(self.expid)
- if exp_row:
- self.manager.create_experiment_status_as_running(exp_row)
- return self.manager.get_experiment_status_row_by_expid(self.expid)
- else:
- print(("Couldn't find {} in the main database. There is not enough information to set it as running.".format(self.expid)))
- return self.manager.default_experiment_status_row
-
-
- def set_as_not_running(self):
- # type: () -> None
- """ Deletes row by expid. """
- if not self.manager:
- raise Exception("ExperimentStatus: The database manager is not available.")
- exp_status_row = self.manager.get_experiment_status_row_by_expid(self.expid)
- if not exp_status_row:
- # raise Exception("ExperimentStatus: Query error, experiment {} not found in status table.".format(self.expid))
- pass # If it is not in the status table, we don't need to worry about it.
- else:
- self.manager.delete_exp_status(exp_status_row.name)
diff --git a/autosubmit_api/history/experiment_status_manager.py b/autosubmit_api/history/experiment_status_manager.py
deleted file mode 100644
index 8a79477c0c9445a835221115aea35d8071af57ab..0000000000000000000000000000000000000000
--- a/autosubmit_api/history/experiment_status_manager.py
+++ /dev/null
@@ -1,89 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2015-2020 Earth Sciences Department, BSC-CNS
-# This file is part of Autosubmit.
-
-# Autosubmit is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# Autosubmit is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with Autosubmit. If not, see .
-
-import os
-import time
-import subprocess
-from .experiment_status import ExperimentStatus
-from ..config.basicConfig import APIBasicConfig
-from ..experiment.common_requests import _is_exp_running
-from ..common.utils import get_experiments_from_folder
-from typing import Dict, Set
-from .utils import SAFE_TIME_LIMIT_STATUS
-from . import utils as HUtils
-
-
-
-class ExperimentStatusManager(object):
- """ Manages the update of the status table. """
- def __init__(self):
- APIBasicConfig.read()
- self._basic_config = APIBasicConfig
- self._experiments_updated = set()
- self._local_root_path = self._basic_config.LOCAL_ROOT_DIR
- self._base_experiment_status = ExperimentStatus("0000")
- self._base_experiment_status.validate_database()
- self._validate_configuration()
- self._creation_timestamp = int(time.time())
-
- def _validate_configuration(self):
- # type: () -> None
- if not os.path.exists(self._local_root_path):
- raise Exception("Experiment Status Manager: LOCAL ROOT DIR not found.")
-
- def update_running_experiments(self, time_condition=600):
- # type: (int) -> None
- """
- Tests if an experiment is running and updates database as_times.db accordingly.\n
- :return: Nothing
- """
-
- # start_reading_folders = int(time.time())
- # currentDirectories = subprocess.Popen(['ls', '-t', self._local_root_path], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- # stdOut, _ = currentDirectories.communicate()
- # readingpkl = stdOut.split()
- # time_reading_folders = int(time.time()) - start_reading_folders
- readingpkl = get_experiments_from_folder(self._local_root_path)
- # Update those that are RUNNING
- for expid in readingpkl:
- pkl_path = os.path.join(self._local_root_path, str(expid), "pkl", "job_list_{0}.pkl".format(str(expid)))
- if not len(expid) == 4 or not os.path.exists(pkl_path):
- continue
- time_spent = int(time.time()) - self._creation_timestamp
- if time_spent > SAFE_TIME_LIMIT_STATUS:
- raise Exception(
- "Time limit reached {0} seconds on update_running_experiments while processing {1}. \
- Time spent on reading data {2} seconds.".format(time_spent, expid, time_reading_folders))
- time_diff = int(time.time()) - int(os.stat(pkl_path).st_mtime)
- if (time_diff < time_condition):
- self._experiments_updated.add(ExperimentStatus(expid).set_as_running().exp_id)
- elif (time_diff <= 3600):
- _, _ , is_running, _, _ = _is_exp_running(expid) # Exhaustive validation
- if is_running == True:
- self._experiments_updated.add(ExperimentStatus(expid).set_as_running().exp_id)
- # Update those that were RUNNING
- self._detect_and_delete_not_running()
-
- def _detect_and_delete_not_running(self):
- # type: () -> None
- current_rows = self._base_experiment_status.get_current_table_content()
- for experiment_status_row in current_rows:
- if experiment_status_row.status == "RUNNING" and experiment_status_row.exp_id not in self._experiments_updated:
- _, _, is_running, _, _ = _is_exp_running(experiment_status_row.name) # Exhaustive validation
- if is_running == False:
- ExperimentStatus(experiment_status_row.name).set_as_not_running()
\ No newline at end of file
diff --git a/autosubmit_api/history/platform_monitor/output_examples/pending.txt b/autosubmit_api/history/platform_monitor/output_examples/pending.txt
deleted file mode 100644
index 007e88d0886e0514329881d82c47db9c4ae5109e..0000000000000000000000000000000000000000
--- a/autosubmit_api/history/platform_monitor/output_examples/pending.txt
+++ /dev/null
@@ -1 +0,0 @@
- 17838842 PENDING 4 1 2021-10-11T10:55:53 Unknown Unknown
diff --git a/autosubmit_api/history/platform_monitor/output_examples/wrapper1.txt b/autosubmit_api/history/platform_monitor/output_examples/wrapper1.txt
deleted file mode 100644
index 61b855cd1b46e5c03a8e8298f333d8315a5b4b2e..0000000000000000000000000000000000000000
--- a/autosubmit_api/history/platform_monitor/output_examples/wrapper1.txt
+++ /dev/null
@@ -1,3 +0,0 @@
- 12535498 COMPLETED 2 1 2020-11-18T13:54:24 2020-11-18T13:55:55 2020-11-18T13:56:10 2.77K
- 12535498.batch COMPLETED 2 1 2020-11-18T13:55:55 2020-11-18T13:55:55 2020-11-18T13:56:10 2.69K 659K 659K
- 12535498.extern COMPLETED 2 1 2020-11-18T13:55:55 2020-11-18T13:55:55 2020-11-18T13:56:10 2.77K 24K 24K
\ No newline at end of file
diff --git a/autosubmit_api/history/platform_monitor/output_examples/wrapper2.txt b/autosubmit_api/history/platform_monitor/output_examples/wrapper2.txt
deleted file mode 100644
index 082eb0105bf17788d6f9a058f6530b74490e0faa..0000000000000000000000000000000000000000
--- a/autosubmit_api/history/platform_monitor/output_examples/wrapper2.txt
+++ /dev/null
@@ -1,3 +0,0 @@
- 12535498 COMPLETED 2 1 2020-11-18T13:54:24 2020-11-18T13:55:55 2020-11-18T13:56:10 2.77K
- 12535498.batch COMPLETED 2 1 2020-11-18T13:55:55 2020-11-18T13:55:55 2020-11-18T13:56:10 2.69K 659K 659K
- 12535498.0 COMPLETED 2 1 2020-11-18T13:55:55 2020-11-18T13:55:55 2020-11-18T13:56:10 2.77K 24K 24K
\ No newline at end of file
diff --git a/autosubmit_api/history/platform_monitor/output_examples/wrapper_big.txt b/autosubmit_api/history/platform_monitor/output_examples/wrapper_big.txt
deleted file mode 100644
index 65c6c119128b035bdb21cb2133838c5353ab9bbd..0000000000000000000000000000000000000000
--- a/autosubmit_api/history/platform_monitor/output_examples/wrapper_big.txt
+++ /dev/null
@@ -1,33 +0,0 @@
- 17857525 COMPLETED 10 1 2021-10-13T15:51:16 2021-10-13T15:51:17 2021-10-13T15:52:47 19.05K
- 17857525.batch COMPLETED 10 1 2021-10-13T15:51:17 2021-10-13T15:51:17 2021-10-13T15:52:47 13.38K 6264K 6264K
- 17857525.extern COMPLETED 10 1 2021-10-13T15:51:17 2021-10-13T15:51:17 2021-10-13T15:52:47 13.66K 473K 68K
- 17857525.0 COMPLETED 10 1 2021-10-13T15:51:21 2021-10-13T15:51:21 2021-10-13T15:51:22 186 352K 312.30K
- 17857525.1 COMPLETED 10 1 2021-10-13T15:51:23 2021-10-13T15:51:23 2021-10-13T15:51:24 186 420K 306.70K
- 17857525.2 COMPLETED 10 1 2021-10-13T15:51:24 2021-10-13T15:51:24 2021-10-13T15:51:27 188 352K 325.80K
- 17857525.3 COMPLETED 10 1 2021-10-13T15:51:28 2021-10-13T15:51:28 2021-10-13T15:51:29 192 352K 341.90K
- 17857525.4 COMPLETED 10 1 2021-10-13T15:51:29 2021-10-13T15:51:29 2021-10-13T15:51:31 186 352K 335.20K
- 17857525.5 COMPLETED 10 1 2021-10-13T15:51:31 2021-10-13T15:51:31 2021-10-13T15:51:32 186 352K 329.80K
- 17857525.6 COMPLETED 10 1 2021-10-13T15:51:32 2021-10-13T15:51:32 2021-10-13T15:51:33 184 428K 311.10K
- 17857525.7 COMPLETED 10 1 2021-10-13T15:51:34 2021-10-13T15:51:34 2021-10-13T15:51:35 185 416K 341.40K
- 17857525.8 COMPLETED 10 1 2021-10-13T15:51:35 2021-10-13T15:51:35 2021-10-13T15:51:37 180 428K 317.40K
- 17857525.9 COMPLETED 10 1 2021-10-13T15:51:39 2021-10-13T15:51:39 2021-10-13T15:51:42 17 424K 272.70K
- 17857525.10 COMPLETED 10 1 2021-10-13T15:51:42 2021-10-13T15:51:42 2021-10-13T15:51:44 185 356K 304.20K
- 17857525.11 COMPLETED 10 1 2021-10-13T15:51:44 2021-10-13T15:51:44 2021-10-13T15:51:45 189 352K 322.20K
- 17857525.12 COMPLETED 10 1 2021-10-13T15:51:45 2021-10-13T15:51:45 2021-10-13T15:51:47 184 388K 310.70K
- 17857525.13 COMPLETED 10 1 2021-10-13T15:51:48 2021-10-13T15:51:48 2021-10-13T15:51:49 183 352K 336.90K
- 17857525.14 COMPLETED 10 1 2021-10-13T15:51:49 2021-10-13T15:51:49 2021-10-13T15:51:51 183 428K 346.60K
- 17857525.15 COMPLETED 10 1 2021-10-13T15:51:51 2021-10-13T15:51:51 2021-10-13T15:51:53 187 352K 335.90K
- 17857525.16 COMPLETED 10 1 2021-10-13T15:51:54 2021-10-13T15:51:54 2021-10-13T15:51:55 184 424K 270K
- 17857525.17 COMPLETED 10 1 2021-10-13T15:51:55 2021-10-13T15:51:55 2021-10-13T15:51:57 186 352K 304.80K
- 17857525.18 COMPLETED 10 1 2021-10-13T15:51:57 2021-10-13T15:51:57 2021-10-13T15:51:59 182 428K 357K
- 17857525.19 COMPLETED 10 1 2021-10-13T15:51:59 2021-10-13T15:51:59 2021-10-13T15:52:01 185 420K 280.60K
- 17857525.20 COMPLETED 10 1 2021-10-13T15:52:01 2021-10-13T15:52:01 2021-10-13T15:52:03 185 352K 339.90K
- 17857525.21 COMPLETED 10 1 2021-10-13T15:52:04 2021-10-13T15:52:04 2021-10-13T15:52:05 188 356K 340.20K
- 17857525.22 COMPLETED 10 1 2021-10-13T15:52:06 2021-10-13T15:52:06 2021-10-13T15:52:08 185 352K 287.50K
- 17857525.23 COMPLETED 10 1 2021-10-13T15:52:08 2021-10-13T15:52:08 2021-10-13T15:52:11 187 420K 349.40K
- 17857525.24 COMPLETED 10 1 2021-10-13T15:52:14 2021-10-13T15:52:14 2021-10-13T15:52:16 185 420K 353.70K
- 17857525.25 COMPLETED 10 1 2021-10-13T15:52:20 2021-10-13T15:52:20 2021-10-13T15:52:22 187 352K 340.30K
- 17857525.26 COMPLETED 10 1 2021-10-13T15:52:24 2021-10-13T15:52:24 2021-10-13T15:52:32 186 420K 345.80K
- 17857525.27 COMPLETED 10 1 2021-10-13T15:52:37 2021-10-13T15:52:37 2021-10-13T15:52:39 184 352K 341K
- 17857525.28 COMPLETED 10 1 2021-10-13T15:52:41 2021-10-13T15:52:41 2021-10-13T15:52:43 184 352K 326.20K
- 17857525.29 COMPLETED 10 1 2021-10-13T15:52:44 2021-10-13T15:52:44 2021-10-13T15:52:47 183 352K 319.30K
diff --git a/autosubmit_api/history/platform_monitor/platform_monitor.py b/autosubmit_api/history/platform_monitor/platform_monitor.py
deleted file mode 100644
index 46249781e0b739235df6e6d91c708ab1ee573d55..0000000000000000000000000000000000000000
--- a/autosubmit_api/history/platform_monitor/platform_monitor.py
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2015-2020 Earth Sciences Department, BSC-CNS
-# This file is part of Autosubmit.
-
-# Autosubmit is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# Autosubmit is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with Autosubmit. If not, see .
-
-from abc import ABCMeta, abstractmethod
-
-class PlatformMonitor(metaclass=ABCMeta):
- def __init__(self, platform_output):
- self.original_input = platform_output
- self.input = str(platform_output).strip()
-
-
- @abstractmethod
- def _identify_input_rows(self):
- """ """
-
diff --git a/autosubmit_api/history/platform_monitor/platform_utils.py b/autosubmit_api/history/platform_monitor/platform_utils.py
deleted file mode 100644
index dc5081678a62dfd0ddc4a32be8838879fd8ffc3c..0000000000000000000000000000000000000000
--- a/autosubmit_api/history/platform_monitor/platform_utils.py
+++ /dev/null
@@ -1,70 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2015-2020 Earth Sciences Department, BSC-CNS
-# This file is part of Autosubmit.
-
-# Autosubmit is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# Autosubmit is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with Autosubmit. If not, see .
-
-import os
-from time import mktime
-from datetime import datetime
-
-SLURM_DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S"
-
-def parse_output_number(string_number):
- """
- Parses number in format 1.0K 1.0M 1.0G
-
- :param string_number: String representation of number
- :type string_number: str
- :return: number in float format
- :rtype: float
- """
- number = 0.0
- if (string_number):
- last_letter = string_number.strip()[-1]
- multiplier = 1.0
- if last_letter == "G":
- multiplier = 1000000000.0 # Billion
- number = float(string_number[:-1])
- elif last_letter == "M":
- multiplier = 1000000.0 # Million
- number = float(string_number[:-1])
- elif last_letter == "K":
- multiplier = 1000.0 # Thousand
- number = float(string_number[:-1])
- else:
- number = float(string_number)
- try:
- number = float(number) * multiplier
- except Exception as exp:
- number = 0.0
- pass
- return number
-
-def try_parse_time_to_timestamp(input):
- """
- Receives a string in format "%Y-%m-%dT%H:%M:%S" and tries to parse it to timestamp.
- """
- try:
- return int(mktime(datetime.strptime(input, SLURM_DATETIME_FORMAT).timetuple()))
- except:
- return 0
-
-def read_example(example_name):
- source_path = "autosubmit_api/history/platform_monitor/output_examples/"
- file_path = os.path.join(source_path, example_name)
- with open(file_path, "r") as fp:
- output_ssh = fp.read()
- return output_ssh
\ No newline at end of file
diff --git a/autosubmit_api/history/platform_monitor/slurm_monitor.py b/autosubmit_api/history/platform_monitor/slurm_monitor.py
deleted file mode 100644
index 30d7e4b7db6b48802ab3c1ad6924b28f4da09bb0..0000000000000000000000000000000000000000
--- a/autosubmit_api/history/platform_monitor/slurm_monitor.py
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2015-2020 Earth Sciences Department, BSC-CNS
-# This file is part of Autosubmit.
-
-# Autosubmit is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# Autosubmit is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with Autosubmit. If not, see .
-
-from .platform_monitor import PlatformMonitor
-from .slurm_monitor_item import SlurmMonitorItem
-
-class SlurmMonitor(PlatformMonitor):
- """ Manages Slurm commands interpretation. """
- def __init__(self, platform_output):
- super(SlurmMonitor, self).__init__(platform_output)
- self._identify_input_rows()
-
- @property
- def steps_energy(self):
- return sum([step.energy for step in self.input_items if step.is_step])
-
- @property
- def total_energy(self):
- return max(self.header.energy, self.steps_energy + self.extern.energy)
-
- @property
- def step_count(self):
- return len([step for step in self.input_items if step.is_step])
-
- def _identify_input_rows(self):
- lines = self.input.split("\n")
- self.input_items = [SlurmMonitorItem.from_line(line) for line in lines]
-
- @property
- def steps(self):
- return [item for item in self.input_items if item.is_step]
-
- @property
- def header(self):
- return next((header for header in self.input_items if header.is_header), None)
-
- @property
- def batch(self):
- return next((batch for batch in self.input_items if batch.is_batch), None)
-
- @property
- def extern(self):
- return next((extern for extern in self.input_items if extern.is_extern), None)
-
- def steps_plus_extern_approximate_header_energy(self):
- return abs(self.steps_energy + self.extern.energy - self.header.energy) <= 10
-
- def print_items(self):
- for item in self.input_items:
- print(item)
diff --git a/autosubmit_api/history/platform_monitor/slurm_monitor_item.py b/autosubmit_api/history/platform_monitor/slurm_monitor_item.py
deleted file mode 100644
index 65ed78758799d121b21e432352de4bae7d7f0bba..0000000000000000000000000000000000000000
--- a/autosubmit_api/history/platform_monitor/slurm_monitor_item.py
+++ /dev/null
@@ -1,98 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2015-2020 Earth Sciences Department, BSC-CNS
-# This file is part of Autosubmit.
-
-# Autosubmit is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# Autosubmit is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with Autosubmit. If not, see .
-
-from . import platform_utils as utils
-
-class SlurmMonitorItem():
- def __init__(self, name, status, ncpus, nnodes, submit, start, finish, energy="0", MaxRSS=0.0, AveRSS=0.0):
- self.name = str(name)
- self.status = str(status)
- self.ncpus = int(ncpus)
- self.nnodes = int(nnodes)
- self.submit = utils.try_parse_time_to_timestamp(submit)
- self.start = utils.try_parse_time_to_timestamp(start)
- self.finish = utils.try_parse_time_to_timestamp(finish)
- self.energy_str = energy
- self.energy = utils.parse_output_number(energy)
- self.MaxRSS = utils.parse_output_number(MaxRSS)
- self.AveRSS = utils.parse_output_number(AveRSS)
-
- @property
- def is_header(self):
- return not self.is_detail
-
- @property
- def is_detail(self):
- if self.name.find(".") >= 0:
- return True
- return False
-
- @property
- def is_extern(self):
- if self.name.find(".ext") >= 0:
- return True
- return False
-
- @property
- def is_batch(self):
- if self.name.find(".bat") >= 0:
- return True
- return False
-
- @property
- def step_number(self):
- if self.is_step == True:
- point_loc = self.name.find(".")
- return int(self.name[point_loc+1:])
- return -1
-
- @property
- def is_step(self):
- if self.name.find(".") >= 0 and self.is_batch == False and self.is_extern == False:
- return True
- return False
-
- @classmethod
- def from_line(cls, line):
- line = line.strip().split()
- if len(line) < 2:
- raise Exception("Slurm parser found a line too short {0}".format(line))
- new_item = cls(line[0],
- line[1],
- str(line[2]) if len(line) > 2 else 0,
- str(line[3]) if len(line) > 3 else 0,
- str(line[4]) if len(line) > 4 else 0,
- str(line[5]) if len(line) > 5 else 0,
- str(line[6]) if len(line) > 6 else 0,
- str(line[7]) if len(line) > 7 else 0,
- str(line[8]) if len(line) > 8 else 0,
- str(line[9]) if len(line) > 9 else 0)
- return new_item
-
- def get_as_dict(self):
- return {"ncpus": self.ncpus,
- "nnodes": self.nnodes,
- "submit": self.submit,
- "start": self.start,
- "finish": self.finish,
- "energy": self.energy,
- "MaxRSS": self.MaxRSS,
- "AveRSS": self.AveRSS}
-
- def __str__(self):
- return "Name {0}, Status {1}, NCpus {2}, NNodes {3}, Submit {4}, Start {5}, Finish {6}, Energy {7}, MaxRSS {8}, AveRSS {9} [Energy Str {10}]".format(self.name, self.status, self.ncpus, self.nnodes, self.submit, self.start, self.finish, self.energy, self.MaxRSS, self.AveRSS, self.energy_str, self.is_batch)
\ No newline at end of file
diff --git a/autosubmit_api/history/platform_monitor/test.py b/autosubmit_api/history/platform_monitor/test.py
deleted file mode 100644
index 9ae89976c0559d519526b7700a8231258a1046ab..0000000000000000000000000000000000000000
--- a/autosubmit_api/history/platform_monitor/test.py
+++ /dev/null
@@ -1,94 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2015-2020 Earth Sciences Department, BSC-CNS
-# This file is part of Autosubmit.
-
-# Autosubmit is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# Autosubmit is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with Autosubmit. If not, see .
-
-import unittest
-from autosubmit_api.history.platform_monitor import platform_utils as utils
-from autosubmit_api.history.platform_monitor.slurm_monitor import SlurmMonitor
-
-class TestSlurmMonitor(unittest.TestCase):
- def test_reader_on_simple_wrapper_example_1(self):
- ssh_output = utils.read_example("wrapper1.txt")
- slurm_monitor = SlurmMonitor(ssh_output)
- # Header
- self.assertTrue(slurm_monitor.input_items[0].is_batch == False)
- self.assertTrue(slurm_monitor.input_items[0].is_detail == False)
- self.assertTrue(slurm_monitor.input_items[0].is_extern == False)
- self.assertTrue(slurm_monitor.input_items[0].is_header == True)
- self.assertTrue(slurm_monitor.input_items[0].is_detail == False)
- # Batch
- self.assertTrue(slurm_monitor.input_items[1].is_batch == True)
- self.assertTrue(slurm_monitor.input_items[1].is_detail == True)
- self.assertTrue(slurm_monitor.input_items[1].is_extern == False)
- self.assertTrue(slurm_monitor.input_items[1].is_header == False)
- self.assertTrue(slurm_monitor.input_items[1].is_detail == True)
- # Extern
- self.assertTrue(slurm_monitor.input_items[2].is_batch == False)
- self.assertTrue(slurm_monitor.input_items[2].is_detail == True)
- self.assertTrue(slurm_monitor.input_items[2].is_extern == True)
- self.assertTrue(slurm_monitor.input_items[2].is_header == False)
- self.assertTrue(slurm_monitor.input_items[2].is_detail == True)
- header = slurm_monitor.header
- batch = slurm_monitor.batch
- extern = slurm_monitor.extern
- self.assertIsNotNone(header)
- self.assertIsNotNone(batch)
- self.assertIsNotNone(extern)
- # print("{0} {1} <- {2}".format(batch.name, batch.energy, batch.energy_str))
- # print("{0} {1} <- {2}".format(extern.name, extern.energy, extern.energy_str))
- # print("{0} {1} <- {2}".format(header.name, header.energy, header.energy_str))
- self.assertTrue(slurm_monitor.steps_plus_extern_approximate_header_energy())
-
-
- def test_reader_on_simple_wrapper_example_2(self):
- ssh_output = utils.read_example("wrapper2.txt") # not real
- slurm_monitor = SlurmMonitor(ssh_output)
- # Header
- self.assertTrue(slurm_monitor.input_items[0].is_batch == False)
- self.assertTrue(slurm_monitor.input_items[0].is_detail == False)
- self.assertTrue(slurm_monitor.input_items[0].is_step == False)
- self.assertTrue(slurm_monitor.input_items[0].is_extern == False)
- self.assertTrue(slurm_monitor.input_items[0].is_header == True)
- # Batch
- self.assertTrue(slurm_monitor.input_items[1].is_batch == True)
- self.assertTrue(slurm_monitor.input_items[1].is_detail == True)
- self.assertTrue(slurm_monitor.input_items[1].is_step == False)
- self.assertTrue(slurm_monitor.input_items[1].is_extern == False)
- self.assertTrue(slurm_monitor.input_items[1].is_header == False)
- # Step 0
- self.assertTrue(slurm_monitor.input_items[2].is_batch == False)
- self.assertTrue(slurm_monitor.input_items[2].is_detail == True)
- self.assertTrue(slurm_monitor.input_items[2].is_step == True)
- self.assertTrue(slurm_monitor.input_items[2].is_extern == False)
- self.assertTrue(slurm_monitor.input_items[2].is_header == False)
- self.assertTrue(slurm_monitor.input_items[2].step_number >= 0)
-
- def test_reader_on_big_wrapper(self):
- ssh_output = utils.read_example("wrapper_big.txt")
- slurm_monitor = SlurmMonitor(ssh_output)
- self.assertTrue(slurm_monitor.step_count == 30)
- header = slurm_monitor.header
- batch = slurm_monitor.batch
- extern = slurm_monitor.extern
- self.assertIsNotNone(header)
- self.assertIsNotNone(batch)
- self.assertIsNotNone(extern)
- self.assertTrue(slurm_monitor.steps_plus_extern_approximate_header_energy())
-
-
-if __name__ == '__main__':
- unittest.main()
\ No newline at end of file
diff --git a/autosubmit_api/history/strategies.py b/autosubmit_api/history/strategies.py
deleted file mode 100644
index 8fdf88dabcb0a6f4031ac3b4c207863d82459a6d..0000000000000000000000000000000000000000
--- a/autosubmit_api/history/strategies.py
+++ /dev/null
@@ -1,220 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2015-2020 Earth Sciences Department, BSC-CNS
-# This file is part of Autosubmit.
-
-# Autosubmit is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# Autosubmit is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with Autosubmit. If not, see .
-
-from abc import ABCMeta, abstractmethod
-from .database_managers import database_models as Models
-import traceback
-from .internal_logging import Logging
-from .database_managers.database_manager import DEFAULT_LOCAL_ROOT_DIR, DEFAULT_HISTORICAL_LOGS_DIR
-
-class PlatformInformationHandler():
- def __init__(self, strategy):
- self._strategy = strategy
-
- @property
- def strategy(self):
- return self._strategy
-
- @strategy.setter
- def strategy(self, strategy):
- self._strategy = strategy
-
- def execute_distribution(self, job_data_dc, job_data_dcs_in_wrapper, slurm_monitor):
- return self._strategy.apply_distribution(job_data_dc, job_data_dcs_in_wrapper, slurm_monitor)
-
-
-class Strategy(metaclass=ABCMeta):
- """ Strategy Interface """
-
- def __init__(self, historiclog_dir_path=DEFAULT_HISTORICAL_LOGS_DIR):
- self.historiclog_dir_path = historiclog_dir_path
-
- @abstractmethod
- def apply_distribution(self, job_data_dc, job_data_dcs_in_wrapper, slurm_monitor):
- pass
-
- def set_job_data_dc_as_processed(self, job_data_dc, original_ssh_output):
- job_data_dc.platform_output = original_ssh_output
- job_data_dc.rowstatus = Models.RowStatus.PROCESSED
- return job_data_dc
-
- def set_job_data_dc_as_process_failed(self, job_data_dc, original_ssh_output):
- job_data_dc.platform_output = original_ssh_output
- job_data_dc.rowstatus = Models.RowStatus.FAULTY
- return job_data_dc
-
- def get_calculated_weights_of_jobs_in_wrapper(self, job_data_dcs_in_wrapper):
- """ Based on computational weight: running time in seconds * number of cpus. """
- total_weight = sum(job.computational_weight for job in job_data_dcs_in_wrapper)
- return {job.job_name: round(job.computational_weight/total_weight, 4) for job in job_data_dcs_in_wrapper}
-
-
-class SingleAssociationStrategy(Strategy):
-
- def __init__(self, historiclog_dir_path=DEFAULT_HISTORICAL_LOGS_DIR):
- super(SingleAssociationStrategy, self).__init__(historiclog_dir_path=historiclog_dir_path)
-
- def apply_distribution(self, job_data_dc, job_data_dcs_in_wrapper, slurm_monitor):
- try:
- if len(job_data_dcs_in_wrapper) > 0:
- return []
- # job_data_dc.submit = slurm_monitor.header.submit
- # job_data_dc.start = slurm_monitor.header.start
- # job_data_dc.finish = slurm_monitor.header.finish
- job_data_dc.ncpus = slurm_monitor.header.ncpus
- job_data_dc.nnodes = slurm_monitor.header.nnodes
- job_data_dc.energy = slurm_monitor.header.energy
- job_data_dc.MaxRSS = max(slurm_monitor.header.MaxRSS, slurm_monitor.batch.MaxRSS if slurm_monitor.batch else 0, slurm_monitor.extern.MaxRSS if slurm_monitor.extern else 0) # TODO: Improve this rule
- job_data_dc.AveRSS = max(slurm_monitor.header.AveRSS, slurm_monitor.batch.AveRSS if slurm_monitor.batch else 0, slurm_monitor.extern.AveRSS if slurm_monitor.extern else 0)
- job_data_dc = self.set_job_data_dc_as_processed(job_data_dc, slurm_monitor.original_input)
- return [job_data_dc]
- except Exception as exp:
- Logging("strategies", self.historiclog_dir_path).log("SingleAssociationStrategy failed for {0}. Using ssh_output: {1}. Exception message: {2}".format(job_data_dc.job_name, slurm_monitor.original_input, str(exp)),
- traceback.format_exc())
- job_data_dc = self.set_job_data_dc_as_process_failed(job_data_dc, slurm_monitor.original_input)
- return [job_data_dc]
-
-class StraightWrapperAssociationStrategy(Strategy):
-
- def __init__(self, historiclog_dir_path=DEFAULT_HISTORICAL_LOGS_DIR):
- super(StraightWrapperAssociationStrategy, self).__init__(historiclog_dir_path=historiclog_dir_path)
-
- def apply_distribution(self, job_data_dc, job_data_dcs_in_wrapper, slurm_monitor):
- """ """
- try:
- if len(job_data_dcs_in_wrapper) != slurm_monitor.step_count:
- return []
- result = []
- computational_weights = self.get_calculated_weights_of_jobs_in_wrapper(job_data_dcs_in_wrapper)
- for job_dc, step in zip(job_data_dcs_in_wrapper, slurm_monitor.steps):
- job_dc.energy = step.energy + computational_weights.get(job_dc.job_name, 0) * slurm_monitor.extern.energy
- job_dc.AveRSS = step.AveRSS
- job_dc.MaxRSS = step.MaxRSS
- job_dc.platform_output = ""
- if job_dc.job_name == job_data_dc.job_name:
- job_data_dc.energy = job_dc.energy
- job_data_dc.AveRSS = job_dc.AveRSS
- job_data_dc.MaxRSS = job_dc.MaxRSS
- result.append(job_dc)
- job_data_dc = self.set_job_data_dc_as_processed(job_data_dc, slurm_monitor.original_input)
- result.append(job_data_dc)
- return result
- except Exception as exp:
- Logging("strategies", self.historiclog_dir_path).log("StraightWrapperAssociationStrategy failed for {0}. Using ssh_output: {1}. Exception message: {2}".format(job_data_dc.job_name,
- slurm_monitor.original_input,
- str(exp)),
- traceback.format_exc())
- job_data_dc = self.set_job_data_dc_as_process_failed(job_data_dc, slurm_monitor.original_input)
- return [job_data_dc]
-
-class GeneralizedWrapperDistributionStrategy(Strategy):
-
- def __init__(self, historiclog_dir_path=DEFAULT_HISTORICAL_LOGS_DIR):
- super(GeneralizedWrapperDistributionStrategy, self).__init__(historiclog_dir_path=historiclog_dir_path)
-
- def apply_distribution(self, job_data_dc, job_data_dcs_in_wrapper, slurm_monitor):
- try:
- result = []
- computational_weights = self.get_calculated_weights_of_jobs_in_wrapper(job_data_dcs_in_wrapper)
- for job_dc in job_data_dcs_in_wrapper:
- job_dc.energy = round(computational_weights.get(job_dc.job_name, 0) * slurm_monitor.total_energy,2)
- job_dc.platform_output = ""
- if job_dc.job_name == job_data_dc.job_name:
- job_data_dc.energy = job_dc.energy
- result.append(job_dc)
- job_data_dc = self.set_job_data_dc_as_processed(job_data_dc, slurm_monitor.original_input)
- result.append(job_data_dc)
- return result
- except Exception as exp:
- Logging("strategies", self.historiclog_dir_path).log("GeneralizedWrapperDistributionStrategy failed for {0}. Using ssh_output: {1}. Exception message: {2}".format(job_data_dc.job_name, slurm_monitor.original_input, str(exp)),
- traceback.format_exc())
- job_data_dc = self.set_job_data_dc_as_process_failed(job_data_dc, slurm_monitor.original_input)
- return [job_data_dc]
-
-class TwoDimWrapperDistributionStrategy(Strategy):
-
- def __init__(self, historiclog_dir_path=DEFAULT_HISTORICAL_LOGS_DIR):
- super(TwoDimWrapperDistributionStrategy, self).__init__(historiclog_dir_path=historiclog_dir_path)
-
- def apply_distribution(self, job_data_dc, job_data_dcs_in_wrapper, slurm_monitor):
- try:
- result = []
- self.jobs_per_level = self.get_jobs_per_level(job_data_dcs_in_wrapper)
- if len(self.jobs_per_level) != slurm_monitor.step_count:
- return []
- comp_weight_per_level = self.get_comp_weight_per_level(self.jobs_per_level)
- level_energy = []
- for i, step in enumerate(slurm_monitor.steps):
- level_energy.append(step.energy + comp_weight_per_level[i] * slurm_monitor.extern.energy)
- for i, jobs in enumerate(self.jobs_per_level):
- weights = self.get_comp_weight_per_group_of_job_dcs(jobs)
- for j, job_dc in enumerate(jobs):
- job_dc.energy = round(level_energy[i] * weights[j], 2)
- if job_dc.job_name == job_data_dc.job_name:
- job_data_dc.energy = job_dc.energy
- result.append(job_dc)
- job_data_dc = self.set_job_data_dc_as_processed(job_data_dc, slurm_monitor.original_input)
- result.append(job_data_dc)
- return result
- except Exception as exp:
- Logging("strategies", self.historiclog_dir_path).log("TwoDimWrapperDistributionStrategy failed for {0}. Using ssh_output: {1}. Exception message: {2}".format(job_data_dc.job_name, slurm_monitor.original_input, str(exp)),
- traceback.format_exc())
- job_data_dc = self.set_job_data_dc_as_process_failed(job_data_dc, slurm_monitor.original_input)
- return [job_data_dc]
-
- def get_jobs_per_level(self, job_data_dcs_in_wrapper):
- """ List of Lists, index of list is the level. """
- job_name_to_object = {job.job_name: job for job in job_data_dcs_in_wrapper}
- levels = []
- roots_dcs = self._get_roots(job_data_dcs_in_wrapper)
- levels.append(roots_dcs)
- next_level = self.get_level(roots_dcs, job_name_to_object)
- while len(next_level) > 0:
- levels.append([job for job in next_level])
- next_level = self.get_level(next_level, job_name_to_object)
- return levels
-
- def _get_roots(self, job_data_dcs_in_wrapper):
- children_names = self._get_all_children(job_data_dcs_in_wrapper)
- return [job for job in job_data_dcs_in_wrapper if job.job_name not in children_names]
-
- def _get_all_children(self, job_data_dcs_in_wrapper):
- result = []
- for job_dc in job_data_dcs_in_wrapper:
- result.extend(job_dc.children_list)
- return result
-
- def get_comp_weight_per_group_of_job_dcs(self, jobs):
- total = sum(job.computational_weight for job in jobs)
- return [round(job.computational_weight/total, 4) for job in jobs]
-
- def get_comp_weight_per_level(self, jobs_per_level):
- level_weight = []
- total_weight = 0
- for jobs in jobs_per_level:
- computational_weight = sum(job.computational_weight for job in jobs)
- total_weight += computational_weight
- level_weight.append(computational_weight)
- return [round(weight/total_weight, 4) for weight in level_weight]
-
- def get_level(self, previous_level_dcs, job_name_to_object):
- children_names = []
- for job_dc in previous_level_dcs:
- children_names.extend(job_dc.children_list)
- level_dcs = [job_name_to_object[job_name] for job_name in children_names if job_name in job_name_to_object]
- return level_dcs
diff --git a/autosubmit_api/history/test.py b/autosubmit_api/history/test.py
deleted file mode 100644
index 9d0bbfdcc528c5b8d38a341c494c8d194218bb08..0000000000000000000000000000000000000000
--- a/autosubmit_api/history/test.py
+++ /dev/null
@@ -1,367 +0,0 @@
-#!/usr/bin/python
-
-# Copyright 2015-2020 Earth Sciences Department, BSC-CNS
-# This file is part of Autosubmit.
-
-# Autosubmit is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# Autosubmit is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with Autosubmit. If not, see .
-
-import unittest
-import traceback
-import os
-import time
-import autosubmit_api.common.utils_for_testing as UtilsForTesting
-import autosubmit_api.performance.utils as PUtils
-from shutil import copy2
-from collections import namedtuple
-from autosubmit_api.history.internal_logging import Logging
-from autosubmit_api.history.strategies import StraightWrapperAssociationStrategy, GeneralizedWrapperDistributionStrategy, PlatformInformationHandler
-from autosubmit_api.config.basicConfig import APIBasicConfig
-from autosubmit_api.history.platform_monitor.slurm_monitor import SlurmMonitor
-from autosubmit_api.builders.experiment_history_builder import ExperimentHistoryDirector, ExperimentHistoryBuilder
-EXPID_TT00_SOURCE = "test_database.db~"
-EXPID_TT01_SOURCE = "test_database_no_run.db~"
-EXPID = "tt00"
-EXPID_NONE = "tt01"
-# BasicConfig.read()
-JOBDATA_DIR = APIBasicConfig.JOBDATA_DIR
-LOCAL_ROOT_DIR = APIBasicConfig.LOCAL_ROOT_DIR
-job = namedtuple("Job", ["name", "date", "member", "status_str", "children"])
-
-class TestExperimentHistory(unittest.TestCase):
- # @classmethod
- # def setUpClass(cls):
- # cls.exp = ExperimentHistory("tt00") # example database
- def setUp(self):
- APIBasicConfig.read()
- source_path_tt00 = os.path.join(JOBDATA_DIR, EXPID_TT00_SOURCE)
- self.target_path_tt00 = os.path.join(JOBDATA_DIR, "job_data_{0}.db".format(EXPID))
- copy2(source_path_tt00, self.target_path_tt00)
- source_path_tt01 = os.path.join(JOBDATA_DIR, EXPID_TT01_SOURCE)
- self.target_path_tt01 = os.path.join(JOBDATA_DIR, "job_data_{0}.db".format(EXPID_NONE))
- copy2(source_path_tt01, self.target_path_tt01)
- self.job_list = [
- job("a29z_20000101_fc2_1_POST", "2000-01-01 00:00:00", "POST", "COMPLETED", ""),
- job("a29z_20000101_fc1_1_CLEAN", "2000-01-01 00:00:00", "CLEAN", "COMPLETED", ""),
- job("a29z_20000101_fc3_1_POST", "2000-01-01 00:00:00", "POST", "RUNNING", ""),
- job("a29z_20000101_fc2_1_CLEAN", "2000-01-01 00:00:00", "CLEAN", "COMPLETED", ""),
- job("a29z_20000101_fc0_3_SIM", "2000-01-01 00:00:00", "SIM", "COMPLETED", ""),
- job("a29z_20000101_fc1_2_POST", "2000-01-01 00:00:00", "POST", "QUEUING", ""),
- ] # 2 differences, all COMPLETED
- self.job_list_large = [
- job("a29z_20000101_fc2_1_POST", "2000-01-01 00:00:00", "POST", "COMPLETED", ""),
- job("a29z_20000101_fc1_1_CLEAN", "2000-01-01 00:00:00", "CLEAN", "COMPLETED", ""),
- job("a29z_20000101_fc3_1_POST", "2000-01-01 00:00:00", "POST", "RUNNING", ""),
- job("a29z_20000101_fc2_1_CLEAN", "2000-01-01 00:00:00", "CLEAN", "COMPLETED", ""),
- job("a29z_20000101_fc0_3_SIM", "2000-01-01 00:00:00", "SIM", "COMPLETED", ""),
- job("a29z_20000101_fc1_2_POST", "2000-01-01 00:00:00", "POST", "QUEUING", ""),
- job("a29z_20000101_fc1_5_POST", "2000-01-01 00:00:00", "POST", "SUSPENDED", ""),
- job("a29z_20000101_fc1_4_POST", "2000-01-01 00:00:00", "POST", "FAILED", ""),
- job("a29z_20000101_fc2_5_CLEAN", "2000-01-01 00:00:00", "CLEAN", "SUBMITTED", ""),
- job("a29z_20000101_fc0_1_POST", "2000-01-01 00:00:00", "POST", "RUNNING", ""),
- ]
- # self.logging_tt00 = Logging("tt00", BasicConfig)
- # self.exp_db_manager_tt00 = ExperimentHistoryDbManager("tt00", BasicConfig)
- # self.exp_db_manager_tt00.initialize()
-
- def tearDown(self):
- os.remove(self.target_path_tt00)
- os.remove(self.target_path_tt01)
-
- def test_get_POST_jobs(self):
- exp_history = ExperimentHistoryDirector(ExperimentHistoryBuilder("a28v")).build_reader_experiment_history(UtilsForTesting.get_mock_basic_config())
- completed_post_jobs = exp_history.manager.get_job_data_dcs_COMPLETED_by_section("POST")
- completed_sim_jobs = exp_history.manager.get_job_data_dcs_COMPLETED_by_section("SIM")
- self.assertTrue(len(completed_post_jobs) == 0)
- self.assertTrue(len(completed_sim_jobs) == 42)
-
- def test_get_job_data_dc_COMPLETED_by_wrapper_run_id(self):
- exp_history = ExperimentHistoryDirector(ExperimentHistoryBuilder("a28v")).build_reader_experiment_history(UtilsForTesting.get_mock_basic_config())
- completed_jobs_by_wrapper_and_run_id = exp_history.manager.get_job_data_dc_COMPLETED_by_wrapper_run_id(1644619612435, 6)
- self.assertTrue(len(completed_jobs_by_wrapper_and_run_id) == 5)
-
- def test_queue_time_considering_wrapper_info(self):
- exp_history = ExperimentHistoryDirector(ExperimentHistoryBuilder("a28v")).build_reader_experiment_history(UtilsForTesting.get_mock_basic_config())
- sim_job = exp_history.manager.get_job_data_dc_unique_latest_by_job_name("a28v_19500101_fc0_580_SIM")
- package_jobs = exp_history.manager.get_job_data_dc_COMPLETED_by_wrapper_run_id(sim_job.rowtype, sim_job.run_id)
- self.assertTrue(sim_job.queuing_time == 79244)
- # print("Adjusted queue time: {}".format(sim_job.queuing_time_considering_package(package_jobs)))
- self.assertTrue(sim_job.queuing_time_considering_package(package_jobs) == (79244 - (63179 + 16065)))
-
- def test_performance_metrics_historic(self):
- exp_history = ExperimentHistoryDirector(ExperimentHistoryBuilder("a28v")).build_reader_experiment_history(UtilsForTesting.get_mock_basic_config())
- sim_job = exp_history.manager.get_job_data_dc_unique_latest_by_job_name("a28v_19500101_fc0_580_SIM")
- experiment_run = exp_history.manager.get_experiment_run_by_id(sim_job.run_id)
- package_jobs = exp_history.manager.get_job_data_dc_COMPLETED_by_wrapper_run_id(sim_job.rowtype, sim_job.run_id)
- calculated_ASYPD = PUtils.calculate_ASYPD_perjob(experiment_run.chunk_unit, experiment_run.chunk_size, sim_job.chunk, sim_job.queuing_time_considering_package(package_jobs) + sim_job.running_time, 0.0, sim_job.status_code)
- calculated_SYPD = PUtils.calculate_SYPD_perjob(experiment_run.chunk_unit, experiment_run.chunk_size, sim_job.chunk, sim_job.running_time, sim_job.status_code)
- # print("calculated ASYPD: {}".format(calculated_ASYPD))
- # print("calculated SYPD: {}".format(calculated_SYPD))
- self.assertTrue(calculated_ASYPD == round(((1.0/12.0) * 86400) / 15552, 2))
- self.assertTrue(calculated_SYPD == round(((1.0/12.0) * 86400) / 15552, 2))
-
- def test_get_historic(self):
- exp_history = ExperimentHistoryDirector(ExperimentHistoryBuilder("a28v")).build_reader_experiment_history(UtilsForTesting.get_mock_basic_config())
- result = exp_history.get_historic_job_data("a28v_19500101_fc0_580_SIM")
- self.assertTrue(result[0]["ASYPD"] == 0.46)
- self.assertTrue(result[0]["SYPD"] == 0.46)
-
- def test_db_exists(self):
- exp_history = ExperimentHistoryDirector(ExperimentHistoryBuilder(EXPID)).build_current_experiment_history() # ExperimentHistory("tt00", BasicConfig, self.exp_db_manager_tt00, self.logging_tt00)
- # # exp_history.initialize_database()
- self.assertTrue(exp_history.manager.my_database_exists() == True)
-
- def test_db_not_exists(self):
- # exp_history_db_manager = ExperimentHistoryDbManager("tt99", BasicConfig)
- # exp_history_logging = Logging("tt99", BasicConfig)
- exp_history = ExperimentHistoryDirector(ExperimentHistoryBuilder("tt99")).build_reader_experiment_history() # ExperimentHistory("tt99", BasicConfig, exp_history_db_manager, exp_history_logging)
- self.assertTrue(exp_history.manager.my_database_exists() == False)
-
- def test_is_header_ready(self):
- exp_history = ExperimentHistoryDirector(ExperimentHistoryBuilder(EXPID)).build_current_experiment_history() # ExperimentHistory("tt00", BasicConfig, self.exp_db_manager_tt00, self.logging_tt00)
- self.assertTrue(exp_history.is_header_ready() == True)
-
- def test_detect_differences_job_list(self):
- exp_history = ExperimentHistoryDirector(ExperimentHistoryBuilder(EXPID)).build_current_experiment_history() # ExperimentHistory("tt00", BasicConfig, self.exp_db_manager_tt00, self.logging_tt00)
- # exp_history.initialize_database()
- differences = exp_history.detect_changes_in_job_list(self.job_list)
- expected_differences = ["a29z_20000101_fc3_1_POST", "a29z_20000101_fc1_2_POST"]
- for job_dc in differences:
- self.assertTrue(job_dc.job_name in expected_differences)
- self.assertTrue(len(differences) == 2)
-
- def test_built_list_of_changes(self):
- exp_history = ExperimentHistoryDirector(ExperimentHistoryBuilder(EXPID)).build_current_experiment_history() # ExperimentHistory("tt00", BasicConfig, self.exp_db_manager_tt00, self.logging_tt00)
- # exp_history.initialize_database()
- built_differences = exp_history._get_built_list_of_changes(self.job_list)
- expected_ids_differences = [90, 101]
- for item in built_differences:
- self.assertTrue(item[3] in expected_ids_differences)
-
- def test_get_date_member_count(self):
- exp_history = ExperimentHistoryDirector(ExperimentHistoryBuilder(EXPID)).build_current_experiment_history() # ExperimentHistory("tt00", BasicConfig, self.exp_db_manager_tt00, self.logging_tt00)
- # exp_history.initialize_database()
- dm_count = exp_history._get_date_member_completed_count(self.job_list)
- self.assertTrue(dm_count > 0)
-
- def test_should_we_create_new_run(self):
- exp_history = ExperimentHistoryDirector(ExperimentHistoryBuilder(EXPID)).build_current_experiment_history() # ExperimentHistory("tt00", BasicConfig, self.exp_db_manager_tt00, self.logging_tt00)
- # exp_history.initialize_database()
- CHANGES_COUNT = 1
- TOTAL_COUNT = 6
- current_experiment_run_dc = exp_history.manager.get_experiment_run_dc_with_max_id()
- current_experiment_run_dc.total = TOTAL_COUNT
- should_we = exp_history.should_we_create_a_new_run(self.job_list, CHANGES_COUNT, current_experiment_run_dc, current_experiment_run_dc.chunk_unit, current_experiment_run_dc.chunk_size)
- self.assertTrue(should_we == False)
- TOTAL_COUNT_DIFF = 5
- current_experiment_run_dc.total = TOTAL_COUNT_DIFF
- should_we = exp_history.should_we_create_a_new_run(self.job_list, CHANGES_COUNT, current_experiment_run_dc, current_experiment_run_dc.chunk_unit, current_experiment_run_dc.chunk_size)
- self.assertTrue(should_we == True)
- CHANGES_COUNT = 5
- should_we = exp_history.should_we_create_a_new_run(self.job_list, CHANGES_COUNT, current_experiment_run_dc, current_experiment_run_dc.chunk_unit, current_experiment_run_dc.chunk_size)
- self.assertTrue(should_we == True)
- CHANGES_COUNT = 1
- current_experiment_run_dc.total = TOTAL_COUNT
- should_we = exp_history.should_we_create_a_new_run(self.job_list, CHANGES_COUNT, current_experiment_run_dc, current_experiment_run_dc.chunk_unit, current_experiment_run_dc.chunk_size*20)
- self.assertTrue(should_we == True)
- should_we = exp_history.should_we_create_a_new_run(self.job_list, CHANGES_COUNT, current_experiment_run_dc, current_experiment_run_dc.chunk_unit, current_experiment_run_dc.chunk_size)
- self.assertTrue(should_we == False)
- should_we = exp_history.should_we_create_a_new_run(self.job_list, CHANGES_COUNT, current_experiment_run_dc, "day", current_experiment_run_dc.chunk_size)
- self.assertTrue(should_we == True)
-
- def test_status_counts(self):
- exp_history = ExperimentHistoryDirector(ExperimentHistoryBuilder(EXPID)).build_current_experiment_history() # ExperimentHistory("tt00", BasicConfig, self.exp_db_manager_tt00, self.logging_tt00)
- # exp_history.initialize_database()
- result = exp_history.get_status_counts_from_job_list(self.job_list_large)
- self.assertTrue(result["COMPLETED"] == 4)
- self.assertTrue(result["QUEUING"] == 1)
- self.assertTrue(result["RUNNING"] == 2)
- self.assertTrue(result["FAILED"] == 1)
- self.assertTrue(result["SUSPENDED"] == 1)
- self.assertTrue(result["TOTAL"] == len(self.job_list_large))
-
- def test_create_new_experiment_run_with_counts(self):
- exp_history = ExperimentHistoryDirector(ExperimentHistoryBuilder(EXPID)).build_current_experiment_history() # ExperimentHistory("tt00", BasicConfig, self.exp_db_manager_tt00, self.logging_tt00)
- # exp_history.initialize_database()
- exp_run = exp_history.create_new_experiment_run(job_list=self.job_list)
- self.assertTrue(exp_run.chunk_size == 0)
- self.assertTrue(exp_run.chunk_unit == "NA")
- self.assertTrue(exp_run.total == len(self.job_list))
- self.assertTrue(exp_run.completed == 4)
-
- def test_finish_current_run(self):
- exp_history = ExperimentHistoryDirector(ExperimentHistoryBuilder(EXPID)).build_current_experiment_history() # ExperimentHistory("tt00", BasicConfig, self.exp_db_manager_tt00, self.logging_tt00)
- # exp_history.initialize_database()
- exp_run = exp_history.finish_current_experiment_run()
- self.assertTrue(len(exp_run.modified) > 0)
- self.assertTrue(exp_run.finish > 0)
-
- def test_process_job_list_changes(self):
- exp_history = ExperimentHistoryDirector(ExperimentHistoryBuilder(EXPID)).build_current_experiment_history() # ExperimentHistory("tt00", BasicConfig, self.exp_db_manager_tt00, self.logging_tt00)
- # exp_history.initialize_database()
- exp_run = exp_history.process_job_list_changes_to_experiment_totals(self.job_list)
- self.assertTrue(exp_run.total == len(self.job_list))
- self.assertTrue(exp_run.completed == 4)
- self.assertTrue(exp_run.running == 1)
- self.assertTrue(exp_run.queuing == 1)
-
- def test_calculated_weights(self):
- exp_history = ExperimentHistoryDirector(ExperimentHistoryBuilder(EXPID)).build_current_experiment_history() # ExperimentHistory("tt00", BasicConfig, self.exp_db_manager_tt00, self.logging_tt00)
- # exp_history.initialize_database()
- job_data_dcs = exp_history.manager.get_all_last_job_data_dcs()
- calculated_weights = GeneralizedWrapperDistributionStrategy().get_calculated_weights_of_jobs_in_wrapper(job_data_dcs)
- sum_comp_weight = 0
- for job_name in calculated_weights:
- sum_comp_weight += calculated_weights[job_name]
- self.assertTrue(abs(sum_comp_weight - 1) <= 0.01)
-
- def test_distribute_energy_in_wrapper_1_to_1(self):
- exp_history = ExperimentHistoryDirector(ExperimentHistoryBuilder(EXPID)).build_current_experiment_history() # ExperimentHistory("tt00", BasicConfig, self.exp_db_manager_tt00, self.logging_tt00)
- # exp_history.initialize_database()
- ssh_output = ''' 17857525 COMPLETED 10 1 2021-10-13T15:51:16 2021-10-13T15:51:17 2021-10-13T15:52:47 2.41K
- 17857525.batch COMPLETED 10 1 2021-10-13T15:51:17 2021-10-13T15:51:17 2021-10-13T15:52:47 1.88K 6264K 6264K
- 17857525.extern COMPLETED 10 1 2021-10-13T15:51:17 2021-10-13T15:51:17 2021-10-13T15:52:47 1.66K 473K 68K
- 17857525.0 COMPLETED 10 1 2021-10-13T15:51:21 2021-10-13T15:51:21 2021-10-13T15:51:22 186 352K 312.30K
- 17857525.1 COMPLETED 10 1 2021-10-13T15:51:23 2021-10-13T15:51:23 2021-10-13T15:51:24 186 420K 306.70K
- 17857525.2 COMPLETED 10 1 2021-10-13T15:51:24 2021-10-13T15:51:24 2021-10-13T15:51:27 188 352K 325.80K
- 17857525.3 COMPLETED 10 1 2021-10-13T15:51:28 2021-10-13T15:51:28 2021-10-13T15:51:29 192 352K 341.90K
- '''
- slurm_monitor = SlurmMonitor(ssh_output)
- job_data_dcs = exp_history.manager.get_all_last_job_data_dcs()[:4] # Get me 4 jobs
- weights = StraightWrapperAssociationStrategy().get_calculated_weights_of_jobs_in_wrapper(job_data_dcs)
- info_handler = PlatformInformationHandler(StraightWrapperAssociationStrategy())
- job_data_dcs_with_data = info_handler.execute_distribution(job_data_dcs[0], job_data_dcs, slurm_monitor)
- self.assertTrue(job_data_dcs_with_data[0].energy == round(slurm_monitor.steps[0].energy + weights[job_data_dcs_with_data[0].job_name]*slurm_monitor.extern.energy, 2))
- self.assertTrue(job_data_dcs_with_data[0].MaxRSS == slurm_monitor.steps[0].MaxRSS)
- self.assertTrue(job_data_dcs_with_data[2].energy == round(slurm_monitor.steps[2].energy + weights[job_data_dcs_with_data[2].job_name]*slurm_monitor.extern.energy, 2))
- self.assertTrue(job_data_dcs_with_data[2].AveRSS == slurm_monitor.steps[2].AveRSS)
-
- def test_distribute_energy_in_wrapper_general_case(self):
- exp_history = ExperimentHistoryDirector(ExperimentHistoryBuilder(EXPID)).build_current_experiment_history() # ExperimentHistory("tt00", BasicConfig, self.exp_db_manager_tt00, self.logging_tt00)
- # exp_history.initialize_database()
- ssh_output = ''' 17857525 COMPLETED 10 1 2021-10-13T15:51:16 2021-10-13T15:51:17 2021-10-13T15:52:47 2.41K
- 17857525.batch COMPLETED 10 1 2021-10-13T15:51:17 2021-10-13T15:51:17 2021-10-13T15:52:47 1.88K 6264K 6264K
- 17857525.extern COMPLETED 10 1 2021-10-13T15:51:17 2021-10-13T15:51:17 2021-10-13T15:52:47 1.66K 473K 68K
- 17857525.0 COMPLETED 10 1 2021-10-13T15:51:21 2021-10-13T15:51:21 2021-10-13T15:51:22 186 352K 312.30K
- 17857525.1 COMPLETED 10 1 2021-10-13T15:51:23 2021-10-13T15:51:23 2021-10-13T15:51:24 186 420K 306.70K
- 17857525.2 COMPLETED 10 1 2021-10-13T15:51:24 2021-10-13T15:51:24 2021-10-13T15:51:27 188 352K 325.80K
- 17857525.3 COMPLETED 10 1 2021-10-13T15:51:28 2021-10-13T15:51:28 2021-10-13T15:51:29 192 352K 341.90K
- '''
- slurm_monitor = SlurmMonitor(ssh_output)
- job_data_dcs = exp_history.manager.get_all_last_job_data_dcs()[:5] # Get me 5 jobs
- weights = GeneralizedWrapperDistributionStrategy().get_calculated_weights_of_jobs_in_wrapper(job_data_dcs)
- # print(sum(weights[k] for k in weights))
- info_handler = PlatformInformationHandler(GeneralizedWrapperDistributionStrategy())
- job_data_dcs_with_data = info_handler.execute_distribution(job_data_dcs[0], job_data_dcs, slurm_monitor)
- self.assertTrue(job_data_dcs_with_data[0].energy == round(slurm_monitor.total_energy * weights[job_data_dcs_with_data[0].job_name], 2))
- self.assertTrue(job_data_dcs_with_data[1].energy == round(slurm_monitor.total_energy * weights[job_data_dcs_with_data[1].job_name], 2))
- self.assertTrue(job_data_dcs_with_data[2].energy == round(slurm_monitor.total_energy * weights[job_data_dcs_with_data[2].job_name], 2))
- self.assertTrue(job_data_dcs_with_data[3].energy == round(slurm_monitor.total_energy * weights[job_data_dcs_with_data[3].job_name], 2))
- self.assertTrue(job_data_dcs_with_data[4].energy == round(slurm_monitor.total_energy * weights[job_data_dcs_with_data[4].job_name], 2))
- sum_energy = sum(job.energy for job in job_data_dcs_with_data[:5]) # Last 1 is original job_data_dc
- self.assertTrue(abs(sum_energy - slurm_monitor.total_energy) <= 10)
-
- def test_process_status_changes(self):
- exp_history = ExperimentHistoryDirector(ExperimentHistoryBuilder(EXPID)).build_current_experiment_history() # ExperimentHistory("tt00", BasicConfig, self.exp_db_manager_tt00, self.logging_tt00)
- # exp_history.initialize_database()
- CHUNK_UNIT = "month"
- CHUNK_SIZE = 20
- CURRENT_CONFIG = "CURRENT CONFIG"
- current_experiment_run_dc = exp_history.manager.get_experiment_run_dc_with_max_id()
- exp_run = exp_history.process_status_changes(job_list=self.job_list, chunk_unit=CHUNK_UNIT, chunk_size=CHUNK_SIZE, current_config=CURRENT_CONFIG) # Generates new run
- self.assertTrue(current_experiment_run_dc.run_id != exp_run.run_id)
- self.assertTrue(exp_run.chunk_unit == CHUNK_UNIT)
- self.assertTrue(exp_run.metadata == CURRENT_CONFIG)
- self.assertTrue(exp_run.total == len(self.job_list))
- current_experiment_run_dc = exp_history.manager.get_experiment_run_dc_with_max_id()
- exp_run = exp_history.process_status_changes(job_list=self.job_list, chunk_unit=CHUNK_UNIT, chunk_size=CHUNK_SIZE, current_config=CURRENT_CONFIG) # Same run
- self.assertTrue(current_experiment_run_dc.run_id == exp_run.run_id)
- new_job_list = [
- job("a29z_20000101_fc2_1_POST", "2000-01-01 00:00:00", "POST", "FAILED", ""),
- job("a29z_20000101_fc1_1_CLEAN", "2000-01-01 00:00:00", "CLEAN", "FAILED", ""),
- job("a29z_20000101_fc3_1_POST", "2000-01-01 00:00:00", "POST", "RUNNING", ""),
- job("a29z_20000101_fc2_1_CLEAN", "2000-01-01 00:00:00", "CLEAN", "FAILED", ""),
- job("a29z_20000101_fc0_3_SIM", "2000-01-01 00:00:00", "SIM", "FAILED", ""),
- job("a29z_20000101_fc1_2_POST", "2000-01-01 00:00:00", "POST", "QUEUING", ""),
- ]
- current_experiment_run_dc = exp_history.manager.get_experiment_run_dc_with_max_id()
- exp_run = exp_history.process_status_changes(job_list=new_job_list, chunk_unit=CHUNK_UNIT, chunk_size=CHUNK_SIZE, current_config=CURRENT_CONFIG) # Generates new run
- self.assertTrue(current_experiment_run_dc.run_id != exp_run.run_id)
- self.assertTrue(exp_run.total == len(new_job_list))
- self.assertTrue(exp_run.failed == 4)
-
- def test_write_submit_time(self):
- exp_history = ExperimentHistoryDirector(ExperimentHistoryBuilder(EXPID)).build_current_experiment_history() # ExperimentHistory("tt00", BasicConfig, self.exp_db_manager_tt00, self.logging_tt00)
- # exp_history.initialize_database()
- JOB_NAME = "a29z_20000101_fc2_1_SIM"
- NCPUS = 128
- PLATFORM_NAME = "marenostrum4"
- JOB_ID = 101
- inserted_job_data_dc = exp_history.write_submit_time(JOB_NAME, time.time(), "SUBMITTED", NCPUS, "00:30", "debug", "20000101", "fc2", "SIM", 1, PLATFORM_NAME, JOB_ID, "bsc_es", 1, "")
- self.assertTrue(inserted_job_data_dc.job_name == JOB_NAME)
- self.assertTrue(inserted_job_data_dc.ncpus == NCPUS)
- self.assertTrue(inserted_job_data_dc.children == "")
- self.assertTrue(inserted_job_data_dc.energy == 0)
- self.assertTrue(inserted_job_data_dc.platform == PLATFORM_NAME)
- self.assertTrue(inserted_job_data_dc.job_id == JOB_ID)
- self.assertTrue(inserted_job_data_dc.qos == "debug")
-
-
- def test_write_start_time(self):
- exp_history = ExperimentHistoryDirector(ExperimentHistoryBuilder(EXPID)).build_current_experiment_history() # ExperimentHistory("tt00", BasicConfig, self.exp_db_manager_tt00, self.logging_tt00)
- # # exp_history.initialize_database()
- JOB_NAME = "a29z_20000101_fc2_1_SIM"
- NCPUS = 128
- PLATFORM_NAME = "marenostrum4"
- JOB_ID = 101
- inserted_job_data_dc_submit = exp_history.write_submit_time(JOB_NAME, time.time(), "SUBMITTED", NCPUS, "00:30", "debug", "20000101", "fc2", "SIM", 1, PLATFORM_NAME, JOB_ID, "bsc_es", 1, "")
- inserted_job_data_dc = exp_history.write_start_time(JOB_NAME, time.time(), "RUNNING", NCPUS, "00:30", "debug", "20000101", "fc2", "SIM", 1, PLATFORM_NAME, JOB_ID, "bsc_es", 1, "")
- self.assertTrue(inserted_job_data_dc.job_name == JOB_NAME)
- self.assertTrue(inserted_job_data_dc.ncpus == NCPUS)
- self.assertTrue(inserted_job_data_dc.children == "")
- self.assertTrue(inserted_job_data_dc.energy == 0)
- self.assertTrue(inserted_job_data_dc.platform == PLATFORM_NAME)
- self.assertTrue(inserted_job_data_dc.job_id == JOB_ID)
- self.assertTrue(inserted_job_data_dc.status == "RUNNING")
- self.assertTrue(inserted_job_data_dc.qos == "debug")
-
-
-
-class TestLogging(unittest.TestCase):
-
- def setUp(self):
- APIBasicConfig.read()
- message = "No Message"
- try:
- raise Exception("Setup test exception")
- except:
- message = traceback.format_exc()
- self.log = Logging("tt00", APIBasicConfig)
- self.exp_message = "Exception message"
- self.trace_message = message
-
- def test_build_message(self):
- message = self.log.build_message(self.exp_message, self.trace_message)
- # print(message)
- self.assertIsNotNone(message)
- self.assertTrue(len(message) > 0)
-
- def test_log(self):
- self.log.log(self.exp_message, self.trace_message)
-
-
-
-
-
-if __name__ == '__main__':
- unittest.main()
\ No newline at end of file
diff --git a/autosubmit_api/history/test_job_history.py b/autosubmit_api/history/test_job_history.py
deleted file mode 100644
index 7e70bb3fd7d6d991d301ad5d72bda53fc490b274..0000000000000000000000000000000000000000
--- a/autosubmit_api/history/test_job_history.py
+++ /dev/null
@@ -1,71 +0,0 @@
-import unittest
-import autosubmit_api.common.utils_for_testing as TestUtils
-from autosubmit_api.builders.experiment_history_builder import ExperimentHistoryDirector, ExperimentHistoryBuilder
-
-class TestJobHistory(unittest.TestCase):
-
- def setUp(self):
- self.basic_config = TestUtils.get_mock_basic_config()
-
- def test_get_job_history_gets_correct_number_of_cases(self):
- # Arrange
- sut = ExperimentHistoryDirector(ExperimentHistoryBuilder("a3tb")).build_reader_experiment_history(self.basic_config)
- # Act
- job_history = sut.get_historic_job_data("a3tb_19930501_fc01_3_SIM")
- # Assert
- self.assertEqual(len(job_history), 5) # 5 rows in database
-
- def test_get_job_history_gets_correct_queuing_times(self):
- # Arrange
- sut = ExperimentHistoryDirector(ExperimentHistoryBuilder("a3tb")).build_reader_experiment_history(self.basic_config)
- # Act
- job_history = sut.get_historic_job_data("a3tb_19930501_fc01_3_SIM")
- counter_to_queue_timedelta_str = {int(job["counter"]): job["queue_time"] for job in job_history}
- # Assert
- self.assertEqual(counter_to_queue_timedelta_str[18], "0:03:28")
- self.assertEqual(counter_to_queue_timedelta_str[19], "0:00:04")
- self.assertEqual(counter_to_queue_timedelta_str[20], "0:00:05")
- self.assertEqual(counter_to_queue_timedelta_str[24], "0:01:18")
- self.assertEqual(counter_to_queue_timedelta_str[25], "0:02:35")
-
- def test_get_job_history_gets_correct_running_times(self):
- # Arrange
- sut = ExperimentHistoryDirector(ExperimentHistoryBuilder("a3tb")).build_reader_experiment_history(self.basic_config)
- # Act
- job_history = sut.get_historic_job_data("a3tb_19930501_fc01_3_SIM")
- counter_to_run_timedelta_str = {int(job["counter"]): job["run_time"] for job in job_history}
- # Assert
- self.assertEqual(counter_to_run_timedelta_str[18], "0:00:54")
- self.assertEqual(counter_to_run_timedelta_str[19], "0:00:53")
- self.assertEqual(counter_to_run_timedelta_str[20], "0:00:52")
- self.assertEqual(counter_to_run_timedelta_str[24], "0:23:16")
- self.assertEqual(counter_to_run_timedelta_str[25], "0:07:58")
-
- def test_get_job_history_gets_correct_SYPD(self):
- # Arrange
- sut = ExperimentHistoryDirector(ExperimentHistoryBuilder("a3tb")).build_reader_experiment_history(self.basic_config)
- # Act
- job_history = sut.get_historic_job_data("a3tb_19930501_fc01_3_SIM")
- counter_to_SYPD = {int(job["counter"]): job["SYPD"] for job in job_history}
- # Assert
- self.assertEqual(counter_to_SYPD[18], None)
- self.assertEqual(counter_to_SYPD[19], None)
- self.assertEqual(counter_to_SYPD[20], None)
- self.assertEqual(counter_to_SYPD[24], round(86400*(1.0/12.0)/1396, 2))
- self.assertEqual(counter_to_SYPD[25], round(86400*(1.0/12.0)/478, 2))
-
- def test_get_job_history_gets_correct_ASYPD(self):
- # Arrange
- sut = ExperimentHistoryDirector(ExperimentHistoryBuilder("a3tb")).build_reader_experiment_history(self.basic_config)
- # Act
- job_history = sut.get_historic_job_data("a3tb_19930501_fc01_3_SIM")
- counter_to_ASYPD = {int(job["counter"]): job["ASYPD"] for job in job_history}
- # Assert
- self.assertEqual(counter_to_ASYPD[18], None)
- self.assertEqual(counter_to_ASYPD[19], None)
- self.assertEqual(counter_to_ASYPD[20], None)
- self.assertEqual(counter_to_ASYPD[24], round(86400*(1.0/12.0)/(1396 + 78), 2))
- self.assertEqual(counter_to_ASYPD[25], round(86400*(1.0/12.0)/(478 + 155), 2))
-
-if __name__ == "__main__":
- unittest.main()
\ No newline at end of file
diff --git a/autosubmit_api/history/test_strategies.py b/autosubmit_api/history/test_strategies.py
deleted file mode 100644
index 04d7fa870d50f54dc884df2020df489adbd1bf12..0000000000000000000000000000000000000000
--- a/autosubmit_api/history/test_strategies.py
+++ /dev/null
@@ -1,94 +0,0 @@
-#!/usr/bin/python
-
-# Copyright 2015-2020 Earth Sciences Department, BSC-CNS
-# This file is part of Autosubmit.
-
-# Autosubmit is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# Autosubmit is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with Autosubmit. If not, see .
-
-import unittest
-from collections import namedtuple
-from autosubmit_api.history.data_classes.job_data import JobData
-from autosubmit_api.history.strategies import StraightWrapperAssociationStrategy, GeneralizedWrapperDistributionStrategy, PlatformInformationHandler, TwoDimWrapperDistributionStrategy
-from autosubmit_api.history.platform_monitor.slurm_monitor import SlurmMonitor
-job_dc = namedtuple("Job", ["job_name", "date", "member", "status_str", "children", "children_list"])
-
-class Test2DWrapperDistributionStrategy(unittest.TestCase):
- def setUp(self):
- self.strategy = TwoDimWrapperDistributionStrategy()
- self.job_data_dcs_in_wrapper = [
- JobData(0, job_name="a29z_20000101_fc2_1_POSTR", status="COMPLETED", submit=10, start=100, finish=200, ncpus=100, energy=0, children="a29z_20000101_fc1_1_CLEAN, a29z_20000101_fc3_1_POST"),
- JobData(0, job_name="a29z_20000101_fc1_1_CLEAN", status="COMPLETED", submit=10, start=100, finish=200, ncpus=100, energy=0, children="a29z_20000101_fc2_1_CLEAN"),
- JobData(0, job_name="a29z_20000101_fc3_1_POST", status="COMPLETED", submit=10, start=100, finish=200, ncpus=100, energy=0, children="a29z_20000101_fc0_3_SIM"),
- JobData(0, job_name="a29z_20000101_fc2_1_CLEAN", status="COMPLETED", submit=10, start=100, finish=200, ncpus=100, energy=0, children=""),
- JobData(0, job_name="a29z_20000101_fc0_3_SIM", status="COMPLETED", submit=10, start=100, finish=200, ncpus=100, energy=0, children=""),
- JobData(0, job_name="a29z_20000101_fc1_2_POSTR1", status="COMPLETED", submit=10, start=100, finish=200, ncpus=100, energy=0, children="a29z_20000101_fc1_5_POST2"),
- JobData(0, job_name="a29z_20000101_fc1_5_POST2", status="COMPLETED", submit=10, start=100, finish=200, ncpus=100, energy=0, children="a29z_20000101_fc1_4_POST3"),
- JobData(0, job_name="a29z_20000101_fc1_4_POST3", status="COMPLETED", submit=10, start=100, finish=200, ncpus=100, energy=0, children="a29z_20000101_fc2_5_CLEAN4"),
- JobData(0, job_name="a29z_20000101_fc2_5_CLEAN4", status="COMPLETED", submit=10, start=100, finish=200, ncpus=100, energy=0, children="a29z_20000101_fc0_1_POST5"),
- JobData(0, job_name="a29z_20000101_fc0_1_POST5", status="COMPLETED", submit=10, start=100, finish=200, ncpus=100, energy=0, children=""),
- ]
-
- def test_get_all_children(self):
- children = self.strategy._get_all_children(self.job_data_dcs_in_wrapper)
- self.assertTrue(len(children) == 8)
-
- def test_get_roots(self):
- roots = self.strategy._get_roots(self.job_data_dcs_in_wrapper)
- self.assertTrue(len(roots) == 2)
-
- def test_get_level(self):
- roots = self.strategy._get_roots(self.job_data_dcs_in_wrapper)
- job_name_to_children_names = {job.job_name: job.children_list for job in self.job_data_dcs_in_wrapper}
- next_level = self.strategy.get_level(roots, job_name_to_children_names)
- self.assertTrue(len(next_level) == 3)
-
- def test_get_jobs_per_level(self):
- levels = self.strategy.get_jobs_per_level(self.job_data_dcs_in_wrapper)
- for level in levels:
- print([job.job_name for job in level])
- self.assertTrue(len(levels) == 5)
- self.assertTrue("a29z_20000101_fc0_1_POST5" in [job.job_name for job in levels[4]])
-
- def test_energy_distribution(self):
- ssh_output = ''' 17857525 COMPLETED 10 1 2021-10-13T15:51:16 2021-10-13T15:51:17 2021-10-13T15:52:47 2.62K
- 17857525.batch COMPLETED 10 1 2021-10-13T15:51:17 2021-10-13T15:51:17 2021-10-13T15:52:47 1.88K 6264K 6264K
- 17857525.extern COMPLETED 10 1 2021-10-13T15:51:17 2021-10-13T15:51:17 2021-10-13T15:52:47 1.66K 473K 68K
- 17857525.0 COMPLETED 10 1 2021-10-13T15:51:21 2021-10-13T15:51:21 2021-10-13T15:51:22 186 352K 312.30K
- 17857525.1 COMPLETED 10 1 2021-10-13T15:51:23 2021-10-13T15:51:23 2021-10-13T15:51:24 186 420K 306.70K
- 17857525.2 COMPLETED 10 1 2021-10-13T15:51:24 2021-10-13T15:51:24 2021-10-13T15:51:27 188 352K 325.80K
- 17857525.3 COMPLETED 10 1 2021-10-13T15:51:28 2021-10-13T15:51:28 2021-10-13T15:51:29 192 352K 341.90K
- 17857525.4 COMPLETED 10 1 2021-10-13T15:51:28 2021-10-13T15:51:28 2021-10-13T15:51:29 210 352K 341.90K
- '''
- slurm_monitor = SlurmMonitor(ssh_output)
- info_handler = PlatformInformationHandler(TwoDimWrapperDistributionStrategy())
- job_dcs = info_handler.execute_distribution(self.job_data_dcs_in_wrapper[0], self.job_data_dcs_in_wrapper, slurm_monitor)
- for job in job_dcs:
- print(("{0} -> {1} and {2} : ncpus {3} running {4}".format(job.job_name, job.energy, job.rowstatus, job.ncpus, job.running_time)))
- for level in info_handler.strategy.jobs_per_level:
- print([job.job_name for job in level])
- total_in_jobs = sum(job.energy for job in job_dcs[:-1]) # ignore last
- self.assertTrue(abs(total_in_jobs - slurm_monitor.total_energy) <= 10)
- self.assertTrue(abs(job_dcs[0].energy - 259) < 1)
- self.assertTrue(abs(job_dcs[1].energy - 259) < 1)
- self.assertTrue(abs(job_dcs[2].energy - 228) < 1)
- self.assertTrue(abs(job_dcs[3].energy - 228) < 1)
- self.assertTrue(abs(job_dcs[4].energy - 228) < 1)
- self.assertTrue(abs(job_dcs[5].energy - 228.67) < 1)
- self.assertTrue(abs(job_dcs[6].energy - 228.67) < 1)
- self.assertTrue(abs(job_dcs[7].energy - 228.67) < 1)
- self.assertTrue(abs(job_dcs[8].energy - 358) < 1)
- self.assertTrue(abs(job_dcs[9].energy - 376) < 1)
-
-if __name__ == '__main__':
- unittest.main()
\ No newline at end of file
diff --git a/autosubmit_api/history/test_utils.py b/autosubmit_api/history/test_utils.py
deleted file mode 100644
index 7abcc510b6a014f031571eb911a945d467deb50b..0000000000000000000000000000000000000000
--- a/autosubmit_api/history/test_utils.py
+++ /dev/null
@@ -1,32 +0,0 @@
-
-import unittest
-import history.utils as HUtils
-
-class TestUtils(unittest.TestCase):
- def setUp(self):
- pass
-
- def test_generate_arguments(self):
- arguments = {"status": 4, "last": 1, "rowtype": 2}
- statement, arg_values = HUtils.get_built_statement_from_kwargs("id", status=4, last=1, rowtype=2)
- print(statement)
- print(arg_values)
- self.assertTrue(statement.find("status") >= 0)
- self.assertTrue(statement.find("last") >= 0)
- self.assertTrue(statement.find("rowtype") >= 0)
- self.assertTrue(statement.find(" ORDER BY id") >= 0)
-
- def test_generate_arguments_kwargs(self):
- def inner_call(expid, **kwargs):
- return HUtils.get_built_statement_from_kwargs("created", **kwargs)
- arguments = {"status": 4, "last": 1, "rowtype": 2}
- answer, arg_values = inner_call("a28v", **arguments)
- print(answer)
- print(arg_values)
- self.assertTrue(answer.find("status") >= 0)
- self.assertTrue(answer.find("last") >= 0)
- self.assertTrue(answer.find("rowtype") >= 0)
- self.assertTrue(answer.find(" ORDER BY created") >= 0)
-
-if __name__ == "__main__":
- unittest.main()
\ No newline at end of file
diff --git a/tests/experiments/calculation_in_progress.lock b/autosubmit_api/persistance/__init__.py
similarity index 100%
rename from tests/experiments/calculation_in_progress.lock
rename to autosubmit_api/persistance/__init__.py
diff --git a/autosubmit_api/persistance/experiment.py b/autosubmit_api/persistance/experiment.py
new file mode 100644
index 0000000000000000000000000000000000000000..178c1b0e83421854bdaf763d62095fd99a1eac75
--- /dev/null
+++ b/autosubmit_api/persistance/experiment.py
@@ -0,0 +1,66 @@
+import os
+from autosubmit_api.config.basicConfig import APIBasicConfig
+
+
+class ExperimentPaths:
+ """
+ Helper class that builds related directories/files paths of an experiment
+ """
+
+ def __init__(self, expid: str) -> None:
+ self._expid = expid
+
+ @property
+ def expid(self):
+ return self._expid
+
+ @property
+ def exp_dir(self):
+ return os.path.join(APIBasicConfig.LOCAL_ROOT_DIR, self.expid)
+
+ @property
+ def pkl_dir(self):
+ return os.path.join(self.exp_dir, "pkl")
+
+ @property
+ def job_list_pkl(self):
+ filename = f"job_list_{self.expid}.pkl"
+ return os.path.join(self.pkl_dir, filename)
+
+ @property
+ def job_packages_db(self):
+ filename = f"job_packages_{self.expid}.db"
+ return os.path.join(self.pkl_dir, filename)
+
+ @property
+ def tmp_dir(self):
+ """
+ tmp dir
+ """
+ return os.path.join(self.exp_dir, APIBasicConfig.LOCAL_TMP_DIR)
+
+ @property
+ def tmp_log_dir(self):
+ """
+ tmp/LOG_{expid} dir
+ """
+ return os.path.join(self.tmp_dir, f"LOG_{self.expid}")
+
+ @property
+ def tmp_as_logs_dir(self):
+ """
+ tmp/ASLOGS dir
+ """
+ return os.path.join(self.tmp_dir, APIBasicConfig.LOCAL_ASLOG_DIR)
+
+ @property
+ def job_data_db(self):
+ return os.path.join(APIBasicConfig.JOBDATA_DIR, f"job_data_{self.expid}.db")
+
+ @property
+ def structure_db(self):
+ return os.path.join(APIBasicConfig.STRUCTURES_DIR, f"structure_{self.expid}.db")
+
+ @property
+ def graph_data_db(self):
+ return os.path.join(APIBasicConfig.GRAPHDATA_DIR, f"graph_data_{self.expid}.db")
diff --git a/autosubmit_api/views/v4.py b/autosubmit_api/views/v4.py
index 07db10cfd3995f788b4154fb8ef5b1a8936d38ae..f42b77e44fb02fe4f0f9182095865bb80dfdbd5d 100644
--- a/autosubmit_api/views/v4.py
+++ b/autosubmit_api/views/v4.py
@@ -165,8 +165,8 @@ class ExperimentView(MethodView):
# Get current run data from history
last_modified_timestamp = exp.created
- completed = exp.completed_jobs if exp.completed_jobs else 0
- total = exp.total_jobs if exp.total_jobs else 0
+ completed = 0
+ total = 0
submitted = 0
queuing = 0
running = 0
@@ -181,10 +181,6 @@ class ExperimentView(MethodView):
if (
current_run
and current_run.total > 0
- and (
- current_run.total == total
- or current_run.modified_timestamp > last_modified_timestamp
- )
):
completed = current_run.completed
total = current_run.total
diff --git a/autosubmit_api/workers/business/populate_times.py b/autosubmit_api/workers/business/populate_times.py
deleted file mode 100644
index 5c5e306d4cfb281f3f7a769dfd74775040e7de23..0000000000000000000000000000000000000000
--- a/autosubmit_api/workers/business/populate_times.py
+++ /dev/null
@@ -1,365 +0,0 @@
-import datetime
-import os
-import pwd
-import time
-import subprocess
-import traceback
-import socket
-import pickle
-from autosubmit_api.builders.configuration_facade_builder import AutosubmitConfigurationFacadeBuilder, ConfigurationFacadeDirector
-from autosubmit_api.components.experiment.pkl_organizer import PklOrganizer
-from autosubmit_api.components.jobs.utils import get_job_total_stats
-from autosubmit_api.config.config_common import AutosubmitConfigResolver
-from autosubmit_api.experiment import common_db_requests as DbRequests
-from autosubmit_api.config.basicConfig import APIBasicConfig
-from autosubmit_api.common.utils import Status
-from bscearth.utils.config_parser import ConfigParserFactory
-
-
-SAFE_TIME_LIMIT = 300
-
-
-def process_completed_times(time_condition=60):
- """
- Tests for completed jobs of all autosubmit experiments and updates their completion times data in job_times and experiment_times.
- :param time_condition: Time difference in seconds that qualifies a experiment as out of date.
- :type time_condition: Integer
- """
- try:
- t0 = time.time()
- DEBUG = False
- APIBasicConfig.read()
- path = APIBasicConfig.LOCAL_ROOT_DIR
- # Time test for data retrieval
- # All experiment from file system
- currentDirectories = subprocess.Popen(['ls', '-t', path],
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT) if (os.path.exists(path)) else None
- stdOut, _ = currentDirectories.communicate() if currentDirectories else (None, None)
- # Building connection to ecearth
-
- current_table = DbRequests.prepare_completed_times_db()
- # Build list of all folder in /esarchive/autosubmit which should be considered as experiments (although some might not be)
- # Pre process
- _preprocess_completed_times()
- experiments = stdOut.split() if stdOut else []
- counter = 0
-
- # Get current `details` from ecearth.db and convert to set for effcient contain test
- details_table_ids_set = set(DbRequests.get_exps_detailed_complete().keys())
- # Get current `experiments` from ecearth.db
- experiments_table = DbRequests.get_exps_base()
-
- for expid in experiments:
- # Experiment names should be 4 char long
- if (len(expid) != 4):
- counter += 1
- continue
- # Experiment names must correspond to an experiment that contains a .pkl file
- exp_str = expid.decode("UTF-8")
- full_path = os.path.join(path, exp_str, "pkl", "job_list_" + exp_str + ".pkl")
- timest = 0
- if os.path.exists(full_path):
- # get time of last modification
- timest = int(os.stat(full_path).st_mtime)
- else:
- counter += 1
- continue
- counter += 1
- experiments_table_exp_id = experiments_table.get(exp_str, None)
- if current_table.get(exp_str, None) is None:
- # Pkl exists but is not registered in the table
- # INSERT
- # print("Pkl of " + exp_str + " exists but not in the table: INSERT")
- current_id = _process_pkl_insert_times(exp_str, full_path, timest, APIBasicConfig, DEBUG)
- _process_details_insert_or_update(exp_str, experiments_table_exp_id, experiments_table_exp_id in details_table_ids_set)
- else:
- exp_id, created, modified, total_jobs, completed_jobs = current_table[exp_str]
- time_diff = int(timest - modified)
- # print("Pkl of " + exp_str + " exists")
- current_id = _process_pkl_insert_times(exp_str, full_path, timest, APIBasicConfig, DEBUG)
- if time_diff > time_condition:
- # Update table
- _process_pkl_update_times(exp_str, full_path, timest, APIBasicConfig, exp_id, DEBUG)
- _process_details_insert_or_update(exp_str, experiments_table_exp_id, experiments_table_exp_id in details_table_ids_set)
- DbRequests.update_experiment_times_only_modified(exp_id, timest)
- t1 = time.time()
- # Timer safeguard
- if (t1 - t0) > SAFE_TIME_LIMIT:
- raise Exception(
- "Time limit reached {0:06.2f} seconds on process_completed_times while processing {1}. Time spent on reading data {2:06.2f} seconds.".format((t1 - t0), exp_str, (t1 - t0)))
- except Exception as ex:
- print((traceback.format_exc()))
- print(str(ex))
-
-
-def _describe_experiment(experiment_id):
- user = ""
- created = ""
- model = ""
- branch = ""
- hpc = ""
- APIBasicConfig.read()
- exp_path = os.path.join(APIBasicConfig.LOCAL_ROOT_DIR, experiment_id)
- if not os.path.exists(exp_path):
- return user, created, model, branch, hpc
-
- user = os.stat(exp_path).st_uid
- try:
- user = pwd.getpwuid(user).pw_name
- except:
- pass
-
- created = datetime.datetime.fromtimestamp(
- os.path.getmtime(exp_path))
-
- try:
- as_conf = AutosubmitConfigResolver(
- experiment_id, APIBasicConfig, ConfigParserFactory())
- as_conf.reload()
-
- project_type = as_conf.get_project_type()
- if project_type != "none":
- if not as_conf.check_proj():
- return False
- if (as_conf.get_svn_project_url()):
- model = as_conf.get_svn_project_url()
- branch = as_conf.get_svn_project_url()
- else:
- model = as_conf.get_git_project_origin()
- branch = as_conf.get_git_project_branch()
- if model is "":
- model = "Not Found"
- if branch is "":
- branch = "Not Found"
- hpc = as_conf.get_platform()
- except:
- pass
- return user, created, model, branch, hpc
-
-
-def _process_details_insert_or_update(expid, exp_id, current_details):
- """
- Decides whether the experiment should be inserted or updated in the details table.
- :param expid: name of experiment e.g: a001
- :type expid: str
- :param exp_id: id of experiment e.g: 1
- :type exp_id: int
- :param current_details: True if it exp_id exists in details table, False otherwise
- :rtype: bool
- :result: True if successful, False otherwise
- :rtype: bool
- """
- result = False
- if exp_id:
- try:
- user, created, model, branch, hpc = _describe_experiment(expid)
- if current_details:
- # Update
- result = DbRequests._update_ecearth_details(exp_id, user, created, model, branch, hpc)
- else:
- # Insert
- _Id = DbRequests._insert_into_ecearth_details(exp_id, user, created, model, branch, hpc)
- result = True if _Id else False
- except Exception as exp:
- print(exp)
- return result
-
-
-def _preprocess_completed_times():
- """
- Preprocess table to get rid of possible conflicts
- :param current_table: table experiment_times from as_times.db
- """
- current_table = DbRequests.get_experiment_times_group()
- for name, _ids in list(current_table.items()):
- if len(_ids) > 1:
- print((str(name) + " has more than 1 register."))
- for i in range(0, len(_ids) - 1):
- _id = _ids[i]
- deleted_outdated = DbRequests.delete_experiment_data(_id)
-
-def _process_pkl_update_times(expid, path_pkl, timest_pkl, BasicConfig, exp_id, debug=False):
- """
- Updates register in job_times and experiment_times for the given experiment.
- :param expid: Experiment name
- :type expid: String
- :param path_pkl: path to the pkl file (DEPRECATED)
- :type path_pkl: String
- :param timest_pkl: Timestamp of the last modified date of the pkl file
- :type timest_pkl: Integer
- :param BasicConfig: Configuration of AS
- :type BasicConfig: Object
- :param exp_id: Id of experiment
- :type exp_id: Integer
- :param debug: Flag (testing purposes)
- :type debug: Boolean
- :return: Nothing
- """
- # debug = True
- try:
- found_in_pkl = list()
- BasicConfig.read()
-
- tmp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, BasicConfig.LOCAL_TMP_DIR)
- job_times_db = dict()
- total_jobs = 0
- completed_jobs = 0
- fd = None
- # t_start = time.time()
- # Get current detail from database
- experiment_times_detail = DbRequests.get_times_detail(exp_id)
- # t_seconds = time.time() - t_start
- must_update_header = False
-
- to_update = []
- to_create = []
-
- # Read PKL
- autosubmit_config_facade = ConfigurationFacadeDirector(
- AutosubmitConfigurationFacadeBuilder(expid)).build_autosubmit_configuration_facade()
- pkl_organizer = PklOrganizer(autosubmit_config_facade)
- for job_item in pkl_organizer.current_content:
- total_jobs += 1
- status_code = job_item.status
- job_name = job_item.name
- found_in_pkl.append(job_name)
- status_text = str(Status.VALUE_TO_KEY[status_code])
- if (status_code == Status.COMPLETED):
- completed_jobs += 1
- if (experiment_times_detail) and job_name in list(experiment_times_detail.keys()):
- # If job in pkl exists in database, retrieve data from database
- submit_time, start_time, finish_time, status_text_in_table, detail_id = experiment_times_detail[job_name]
- if (status_text_in_table != status_text):
- # If status has changed
- submit_time, start_time, finish_time, _ = get_job_total_stats(status_code, job_name, tmp_path)
- submit_ts = int(time.mktime(submit_time.timetuple())) if len(str(submit_time)) > 0 else 0
- start_ts = int(time.mktime(start_time.timetuple())) if len(str(start_time)) > 0 else 0
- finish_ts = int(time.mktime(finish_time.timetuple())) if len(str(finish_time)) > 0 else 0
- # UPDATE
- must_update_header = True
- to_update.append((int(timest_pkl),
- submit_ts,
- start_ts,
- finish_ts,
- status_text,
- detail_id))
-
- else:
- # Insert only if it is not WAITING nor READY
- if (status_code not in [Status.WAITING, Status.READY]):
- submit_time, start_time, finish_time, status_text = get_job_total_stats(status_code, job_name, tmp_path)
- must_update_header = True
- to_create.append((exp_id,
- job_name,
- int(timest_pkl),
- int(timest_pkl),
- int(time.mktime(submit_time.timetuple())) if len(str(submit_time)) > 0 else 0,
- int(time.mktime(start_time.timetuple())) if len(str(start_time)) > 0 else 0,
- int(time.mktime(finish_time.timetuple())) if len(str(finish_time)) > 0 else 0,
- status_text))
-
-
- # Update Many
- if len(to_update) > 0:
- DbRequests.update_many_job_times(to_update)
- # Create Many
- if len(to_create) > 0:
- DbRequests.create_many_job_times(to_create)
-
- if must_update_header == True:
- exp_id = DbRequests.update_experiment_times(exp_id, int(timest_pkl), completed_jobs, total_jobs, debug)
-
- # Reviewing for deletes:
- if len(found_in_pkl) > 0 and (experiment_times_detail):
- detail_list = []
- for key in experiment_times_detail:
- if key not in found_in_pkl:
- # Delete Row
- submit_time, start_time, finish_time, status_text_in_table, detail_id = experiment_times_detail[key]
- detail_list.append((detail_id,))
- if len(detail_list) > 0:
- DbRequests._delete_many_from_job_times_detail(detail_list)
-
- except (socket.error, EOFError):
- # print(str(expid) + "\t EOF Error")
- pass
- except Exception as ex:
- print(expid)
- print((traceback.format_exc()))
-
-
-def _process_pkl_insert_times(expid, path_pkl, timest_pkl, BasicConfig, debug=False):
- """
- Process Pkl contents and insert information into database if status of jobs is not WAITING (to save space).
- :param conn: Connection to database
- :type conn: Sqlite3 connection object
- :param expid: Experiment name
- :type expid: String
- :param path_pkl: Path to the pkl file
- :type path_pkl: String
- :param timest_pkl: Timestamp of the pkl modified date
- :type timest_pkl: Integer
- :param BasicConfig: Configuration data of AS
- :type BasicConfig: Object
- :param debug: Flag (proper name should be test)
- :type debug: Boolean
- """
- # BasicConfig.read()
- # path = BasicConfig.LOCAL_ROOT_DIR
- # db_file = os.path.join(path, DbRequests.DB_FILE_AS_TIMES)
- # Build tmp path to search for TOTAL_STATS files
- tmp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, BasicConfig.LOCAL_TMP_DIR)
- job_times = dict() # Key: job_name
- total_jobs = 0
- completed_jobs = 0
- status_code = Status.UNKNOWN
- status_text = str(Status.VALUE_TO_KEY[status_code])
- try:
- # Read PKL
- autosubmit_config_facade = ConfigurationFacadeDirector(
- AutosubmitConfigurationFacadeBuilder(expid)).build_autosubmit_configuration_facade()
- pkl_organizer = PklOrganizer(autosubmit_config_facade)
- for job_item in pkl_organizer.current_content:
- total_jobs += 1
- status_code = job_item.status
- job_name = job_item.name
- status_text = str(Status.VALUE_TO_KEY[status_code])
- if (status_code == Status.COMPLETED):
- completed_jobs += 1
- job_times[job_name] = status_code
- except:
- pass
-
- try:
- # Insert header
- current_id = DbRequests.insert_experiment_times_header(expid, int(timest_pkl), total_jobs, completed_jobs, debug)
- if(current_id > 0):
- # Insert detail
- to_insert_many = []
- for job_name in job_times:
- # Inserting detail. Do not insert WAITING or READY jobs.
- status_code = job_times[job_name]
- if (status_code not in [Status.WAITING, Status.READY]):
- submit_time, start_time, finish_time, status_text = get_job_total_stats(status_code, job_name, tmp_path)
- to_insert_many.append((current_id,
- job_name,
- int(timest_pkl),
- int(timest_pkl),
- int(time.mktime(submit_time.timetuple())) if len(
- str(submit_time)) > 0 else 0,
- int(time.mktime(start_time.timetuple())) if len(
- str(start_time)) > 0 else 0,
- int(time.mktime(finish_time.timetuple())) if len(
- str(finish_time)) > 0 else 0,
- status_text))
- if len(to_insert_many) > 0:
- DbRequests.create_many_job_times(to_insert_many)
- else:
- pass
-
- return current_id
- except Exception as ex:
- print(expid)
- print((traceback.format_exc()))
- return 0
diff --git a/autosubmit_api/workers/business/process_graph_drawings.py b/autosubmit_api/workers/business/process_graph_drawings.py
index 7a5177244baffcda2ae24929bf6fc4a96a9c2690..c6fa84cf120f8cb2e02b1e9c0063041d22c7d472 100644
--- a/autosubmit_api/workers/business/process_graph_drawings.py
+++ b/autosubmit_api/workers/business/process_graph_drawings.py
@@ -1,49 +1,78 @@
import time
import traceback
-from ...experiment import common_db_requests as DbRequests
-from ...common import utils as common_utils
-from ...database.db_jobdata import ExperimentGraphDrawing
-from ...builders.configuration_facade_builder import ConfigurationFacadeDirector, AutosubmitConfigurationFacadeBuilder
-from ...builders.joblist_loader_builder import JobListLoaderBuilder, JobListLoaderDirector, JobListHelperBuilder
+from autosubmit_api.database import tables
+from autosubmit_api.database.common import create_as_times_db_engine
+from autosubmit_api.common import utils as common_utils
+from autosubmit_api.database.db_jobdata import ExperimentGraphDrawing
+from autosubmit_api.builders.configuration_facade_builder import (
+ ConfigurationFacadeDirector,
+ AutosubmitConfigurationFacadeBuilder,
+)
+from autosubmit_api.builders.joblist_loader_builder import (
+ JobListLoaderBuilder,
+ JobListLoaderDirector,
+)
from typing import List, Any
+
def process_active_graphs():
- """
- Process the list of active experiments to generate the positioning of their graphs
- """
- try:
- currently_running = DbRequests.get_currently_running_experiments()
-
- for expid in currently_running:
-
- try:
- autosubmit_configuration_facade = ConfigurationFacadeDirector(AutosubmitConfigurationFacadeBuilder(expid)).build_autosubmit_configuration_facade()
- if common_utils.is_version_historical_ready(autosubmit_configuration_facade.get_autosubmit_version()):
- # job_count = currently_running.get(expid, 0)
- _process_graph(expid, autosubmit_configuration_facade.chunk_size)
- except Exception as exp:
- print((traceback.format_exc()))
- print(("Error while processing: {}".format(expid)))
-
- except Exception as exp:
- print((traceback.format_exc()))
- print(("Error while processing graph drawing: {}".format(exp)))
+ """
+ Process the list of active experiments to generate the positioning of their graphs
+ """
+ try:
+ with create_as_times_db_engine().connect() as conn:
+ query_result = conn.execute(
+ tables.experiment_status_table.select().where()
+ ).all()
+
+ active_experiments: List[str] = [exp.name for exp in query_result]
+
+ for expid in active_experiments:
+ try:
+ autosubmit_configuration_facade = ConfigurationFacadeDirector(
+ AutosubmitConfigurationFacadeBuilder(expid)
+ ).build_autosubmit_configuration_facade()
+ if common_utils.is_version_historical_ready(
+ autosubmit_configuration_facade.get_autosubmit_version()
+ ):
+ _process_graph(expid, autosubmit_configuration_facade.chunk_size)
+ except Exception as exp:
+ print((traceback.format_exc()))
+ print(("Error while processing: {}".format(expid)))
+
+ except Exception as exp:
+ print((traceback.format_exc()))
+ print(("Error while processing graph drawing: {}".format(exp)))
+
def _process_graph(expid, chunk_size):
- # type: (str, int) -> List[Any] | None
- result = None
- experimentGraphDrawing = ExperimentGraphDrawing(expid)
- locked = experimentGraphDrawing.locked
- # print("Start Processing {} with {} jobs".format(expid, job_count))
- if not locked:
- start_time = time.time()
- job_list_loader = JobListLoaderDirector(JobListLoaderBuilder(expid)).build_loaded_joblist_loader()
- current_data = experimentGraphDrawing.get_validated_data(job_list_loader.jobs)
- if not current_data:
- print(("Must update {}".format(expid)))
- result = experimentGraphDrawing.calculate_drawing(job_list_loader.jobs, independent=False, num_chunks=chunk_size, job_dictionary=job_list_loader.job_dictionary)
- print(("Time Spent in {}: {} seconds.".format(expid, int(time.time() - start_time))))
- else:
- print(("{} Locked".format(expid)))
-
- return result
\ No newline at end of file
+ # type: (str, int) -> List[Any] | None
+ result = None
+ experimentGraphDrawing = ExperimentGraphDrawing(expid)
+ locked = experimentGraphDrawing.locked
+ # print("Start Processing {} with {} jobs".format(expid, job_count))
+ if not locked:
+ start_time = time.time()
+ job_list_loader = JobListLoaderDirector(
+ JobListLoaderBuilder(expid)
+ ).build_loaded_joblist_loader()
+ current_data = experimentGraphDrawing.get_validated_data(job_list_loader.jobs)
+ if not current_data:
+ print(("Must update {}".format(expid)))
+ result = experimentGraphDrawing.calculate_drawing(
+ job_list_loader.jobs,
+ independent=False,
+ num_chunks=chunk_size,
+ job_dictionary=job_list_loader.job_dictionary,
+ )
+ print(
+ (
+ "Time Spent in {}: {} seconds.".format(
+ expid, int(time.time() - start_time)
+ )
+ )
+ )
+ else:
+ print(("{} Locked".format(expid)))
+
+ return result
diff --git a/autosubmit_api/workers/populate_details/test.py b/autosubmit_api/workers/populate_details/test.py
deleted file mode 100644
index 24fc30a64f7aebc5da308e883ae608af00dbae6e..0000000000000000000000000000000000000000
--- a/autosubmit_api/workers/populate_details/test.py
+++ /dev/null
@@ -1,53 +0,0 @@
-import unittest
-from autosubmit_api.workers.populate_details.populate import DetailsProcessor
-import autosubmit_api.common.utils_for_testing as TestUtils
-
-class TestPopulate(unittest.TestCase):
- def setUp(self):
- pass
-
- def test_retrieve_correct_experiments(self):
- # Arrange
- sut = DetailsProcessor(TestUtils.get_mock_basic_config())
- # Act
- experiments = sut._get_experiments()
- names = [experiment.name for experiment in experiments]
- # Assert
- self.assertIn("a28v", names)
- self.assertIn("a3tb", names)
-
-
- def test_get_details_from_experiment(self):
- # Arrange
- sut = DetailsProcessor(TestUtils.get_mock_basic_config())
- # Act
- details = sut._get_details_data_from_experiment("a28v")
- # Assert
- self.assertIsNotNone(details)
- self.assertEqual("wuruchi", details.owner)
- self.assertIsInstance(details.created, str)
- self.assertIsInstance(details.model, str)
- self.assertIsInstance(details.branch, str)
- self.assertEqual("marenostrum4", details.hpc)
-
-
- def test_get_all_experiment_details_equals_number_of_test_cases(self):
- # Arrange
- sut = DetailsProcessor(TestUtils.get_mock_basic_config())
- # Act
- processed_data = sut._get_all_details()
- # Assert
- self.assertEqual(len(processed_data), 2) # There are 2 cases in the test_cases folder
-
-
- def test_process_inserts_the_details(self):
- # Arrange
- sut = DetailsProcessor(TestUtils.get_mock_basic_config())
- # Act
- number_rows = sut.process()
- # Assert
- self.assertEqual(number_rows, 2)
-
-
-if __name__ == '__main__':
- unittest.main()
\ No newline at end of file
diff --git a/autosubmit_api/workers/populate_queue_run_times.py b/autosubmit_api/workers/populate_queue_run_times.py
deleted file mode 100644
index 0236886edc3e53b184b234fc07c82d96c2cd4353..0000000000000000000000000000000000000000
--- a/autosubmit_api/workers/populate_queue_run_times.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from autosubmit_api.bgtasks.bgtask import PopulateQueueRuntimes
-
-
-def main():
- """ Process and updates queuing and running times. """
- PopulateQueueRuntimes.run()
-
-
-if __name__ == "__main__":
- main()
diff --git a/autosubmit_api/workers/populate_running_experiments.py b/autosubmit_api/workers/populate_running_experiments.py
index c0fe4cecc2487ede16e9c6567defd0dbd8e5e059..935553965b68a50a6e314d3fd412574eccbf42e0 100644
--- a/autosubmit_api/workers/populate_running_experiments.py
+++ b/autosubmit_api/workers/populate_running_experiments.py
@@ -1,11 +1,11 @@
# import autosubmitAPIwu.experiment.common_requests as ExperimentUtils
-from autosubmit_api.bgtasks.bgtask import PopulateRunningExperiments
+from autosubmit_api.bgtasks.tasks.status_updater import StatusUpdater
def main():
"""
Updates STATUS of experiments.
"""
- PopulateRunningExperiments.run()
+ StatusUpdater.run()
if __name__ == "__main__":
diff --git a/autosubmit_api/workers/test.py b/autosubmit_api/workers/test.py
deleted file mode 100644
index cd2a57baa7b046b0a2bd1b5eb1bb485abf3b757b..0000000000000000000000000000000000000000
--- a/autosubmit_api/workers/test.py
+++ /dev/null
@@ -1,18 +0,0 @@
-import unittest
-
-import workers.business.process_graph_drawings as ProcessGraph
-from builders.configuration_facade_builder import ConfigurationFacadeDirector, AutosubmitConfigurationFacadeBuilder
-
-class TestGraphDraw(unittest.TestCase):
- def setUp(self):
- pass
-
- def test_graph_drawing_generator(self):
- EXPID = "a29z"
- autosubmit_configuration_facade = ConfigurationFacadeDirector(AutosubmitConfigurationFacadeBuilder(EXPID)).build_autosubmit_configuration_facade()
- result = ProcessGraph._process_graph(EXPID, autosubmit_configuration_facade.chunk_size)
- self.assertTrue(result == None)
-
-
-if __name__ == "__main__":
- unittest.main()
\ No newline at end of file
diff --git a/autosubmit_api/workers/verify_complete.py b/autosubmit_api/workers/verify_complete.py
deleted file mode 100644
index a8e85a4015ad811b73b53046f338641d76a8d236..0000000000000000000000000000000000000000
--- a/autosubmit_api/workers/verify_complete.py
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env python
-from autosubmit_api.bgtasks.bgtask import VerifyComplete
-
-
-def main():
- VerifyComplete.run()
-
-
-if __name__ == "__main__":
- main()
diff --git a/pytest.ini b/pytest.ini
new file mode 100644
index 0000000000000000000000000000000000000000..da75339225a307b2aa8beeb87be4a0a9755100fe
--- /dev/null
+++ b/pytest.ini
@@ -0,0 +1,10 @@
+[pytest]
+addopts =
+ --cov=autosubmit_api --cov-config=.coveragerc --cov-report=html
+testpaths =
+ tests/
+doctest_optionflags =
+ NORMALIZE_WHITESPACE
+ IGNORE_EXCEPTION_DETAIL
+ ELLIPSIS
+
diff --git a/setup.py b/setup.py
index 8a648a87eeff97d984603f621376f2e2e12c824e..8eb87b2f76389b0717447cd8068c1bcba08d5453 100644
--- a/setup.py
+++ b/setup.py
@@ -19,6 +19,40 @@ def get_authors():
return autosubmit_api.__author__
+install_requires = [
+ "Flask~=2.2.5",
+ "pyjwt~=2.8.0",
+ "requests~=2.28.1",
+ "flask_cors~=3.0.10",
+ "bscearth.utils~=0.5.2",
+ "pysqlite-binary",
+ "pydotplus~=2.0.2",
+ "portalocker~=2.6.0",
+ "networkx~=2.6.3",
+ "scipy~=1.7.3",
+ "paramiko~=2.12.0",
+ "python-dotenv",
+ "autosubmitconfigparser~=1.0.48",
+ "autosubmit>=3.13",
+ "Flask-APScheduler",
+ "gunicorn",
+ "pydantic~=2.5.2",
+ "SQLAlchemy~=2.0.23",
+ "python-cas>=1.6.0",
+ "Authlib>=1.3.0"
+]
+
+# Test dependencies
+test_requires = [
+ "pytest",
+ "pytest-cov"
+]
+
+extras_require = {
+ 'test': test_requires,
+ 'all': install_requires + test_requires
+}
+
setup(
name="autosubmit_api",
version=get_version(),
@@ -32,28 +66,8 @@ setup(
packages=find_packages(),
keywords=["autosubmit", "API"],
python_requires=">=3.8",
- install_requires=[
- "Flask~=2.2.5",
- "pyjwt~=2.8.0",
- "requests~=2.28.1",
- "flask_cors~=3.0.10",
- "bscearth.utils~=0.5.2",
- "pysqlite-binary",
- "pydotplus~=2.0.2",
- "portalocker~=2.6.0",
- "networkx~=2.6.3",
- "scipy~=1.7.3",
- "paramiko~=2.12.0",
- "python-dotenv",
- "autosubmitconfigparser~=1.0.48",
- "autosubmit>=3.13",
- "Flask-APScheduler",
- "gunicorn",
- "pydantic~=2.5.2",
- "SQLAlchemy~=2.0.23",
- "python-cas>=1.6.0",
- "Authlib>=1.3.0"
- ],
+ install_requires=install_requires,
+ extras_require=extras_require,
include_package_data=True,
package_data={"autosubmit-api": ["README", "VERSION", "LICENSE"]},
classifiers=[
@@ -61,7 +75,7 @@ setup(
"Intended Audience :: Developers",
"License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
"Operating System :: POSIX :: Linux",
- "Programming Language :: Python :: 3.7",
+ "Programming Language :: Python :: 3.8",
],
entry_points={
"console_scripts": [
diff --git a/tests/bgtasks/test_status_updater.py b/tests/bgtasks/test_status_updater.py
new file mode 100644
index 0000000000000000000000000000000000000000..6d2302956f97ffeb4eb5df736b4e3dfa6fbdece4
--- /dev/null
+++ b/tests/bgtasks/test_status_updater.py
@@ -0,0 +1,27 @@
+from autosubmit_api.bgtasks.tasks.status_updater import StatusUpdater
+from autosubmit_api.database import prepare_db, tables
+from autosubmit_api.database.common import (
+ create_autosubmit_db_engine,
+ create_as_times_db_engine,
+)
+from autosubmit_api.history.database_managers.database_models import RunningStatus
+
+class TestStatusUpdater:
+
+ def test_same_tables(self, fixture_mock_basic_config):
+ prepare_db()
+
+ StatusUpdater.run()
+
+ with create_autosubmit_db_engine().connect() as conn:
+ experiments = conn.execute(tables.experiment_table.select()).all()
+
+ with create_as_times_db_engine().connect() as conn:
+ exps_status = conn.execute(tables.experiment_status_table.select()).all()
+
+ assert len(experiments) == len(exps_status)
+ assert set([x.id for x in experiments]) == set([x.exp_id for x in exps_status])
+ assert set([x.name for x in experiments]) == set([x.name for x in exps_status])
+
+ for e_st in exps_status:
+ assert e_st.status in [RunningStatus.RUNNING, RunningStatus.NOT_RUNNING]
diff --git a/tests/conftest.py b/tests/conftest.py
index 8fa8be4f8f166fb246b74506667a36b30552843f..be699e00c6651b16daf191f6618656cbe26787b3 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,4 +1,4 @@
-# Conftest file for sharing fixtures
+# Conftest file for sharing fixtures
# Reference: https://docs.pytest.org/en/latest/reference/fixtures.html#conftest-py-sharing-fixtures-across-multiple-files
import os
@@ -7,6 +7,7 @@ import pytest
from autosubmitconfigparser.config.basicconfig import BasicConfig
from autosubmit_api.app import create_app
from autosubmit_api.config.basicConfig import APIBasicConfig
+from autosubmit_api import config
from tests.custom_utils import custom_return_value
FAKE_EXP_DIR = "./tests/experiments/"
@@ -15,20 +16,14 @@ FAKE_EXP_DIR = "./tests/experiments/"
#### FIXTURES ####
@pytest.fixture(autouse=True)
def fixture_disable_protection(monkeypatch: pytest.MonkeyPatch):
+ monkeypatch.setattr(config, "PROTECTION_LEVEL", "NONE")
monkeypatch.setenv("PROTECTION_LEVEL", "NONE")
+
@pytest.fixture
def fixture_mock_basic_config(monkeypatch: pytest.MonkeyPatch):
- # Patch APIBasicConfig parent BasicConfig
- monkeypatch.setattr(BasicConfig, "read", custom_return_value(None))
- monkeypatch.setattr(APIBasicConfig, "read", custom_return_value(None))
- monkeypatch.setattr(BasicConfig, "LOCAL_ROOT_DIR", FAKE_EXP_DIR)
- monkeypatch.setattr(BasicConfig, "DB_DIR", FAKE_EXP_DIR)
- monkeypatch.setattr(BasicConfig, "DB_FILE", "autosubmit.db")
- monkeypatch.setattr(
- BasicConfig, "DB_PATH", os.path.join(FAKE_EXP_DIR, "autosubmit.db")
- )
- monkeypatch.setattr(BasicConfig, "AS_TIMES_DB", "as_times.db")
+ # Get APIBasicConfig from file
+ monkeypatch.setenv("AUTOSUBMIT_CONFIGURATION", os.path.join(FAKE_EXP_DIR, ".autosubmitrc"))
yield APIBasicConfig
diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..9e2ed0c111ddb1a0a3e94324f2c2e1ca628c987b
--- /dev/null
+++ b/tests/docker/Dockerfile
@@ -0,0 +1,17 @@
+FROM python:3.8-slim-bullseye
+
+WORKDIR /
+
+RUN apt-get update && apt-get install -y git graphviz
+
+RUN git clone https://earth.bsc.es/gitlab/es/autosubmit_api.git
+
+WORKDIR /autosubmit_api/
+
+RUN git checkout v4.0.0b4
+
+RUN pip install .
+
+RUN pip install -U pytest pytest-cov
+
+RUN pytest --cov=autosubmit_api --cov-config=.coveragerc --cov-report=html tests/
\ No newline at end of file
diff --git a/tests/experiments/.autosubmitrc b/tests/experiments/.autosubmitrc
new file mode 100644
index 0000000000000000000000000000000000000000..da7bbe7c1b55fa565084366bcb41f5026c2cb561
--- /dev/null
+++ b/tests/experiments/.autosubmitrc
@@ -0,0 +1,21 @@
+[database]
+path = ./tests/experiments/
+filename = autosubmit.db
+
+[local]
+path = ./tests/experiments/
+
+[globallogs]
+path = ./tests/experiments/logs
+
+[historicdb]
+path = ./tests/experiments/metadata/data
+
+[structures]
+path = ./tests/experiments/metadata/structures
+
+[historiclog]
+path = ./tests/experiments/metadata/logs
+
+[graph]
+path = ./tests/experiments/metadata/graph
\ No newline at end of file
diff --git a/tests/experiments/as_times.db b/tests/experiments/as_times.db
index ef93733acc0d1b1e25f8f3577df6df68f498c66a..0f8be2ec756ee0c158c810bbb21ecc5baa5089d0 100644
Binary files a/tests/experiments/as_times.db and b/tests/experiments/as_times.db differ
diff --git a/tests/experiments/metadata/data/job_data_a007.db b/tests/experiments/metadata/data/job_data_a007.db
new file mode 100755
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/tests/experiments/metadata/data/job_data_a3tb.db b/tests/experiments/metadata/data/job_data_a3tb.db
new file mode 100644
index 0000000000000000000000000000000000000000..b5ead66dab09d063c5023a17bd17300211ea848f
Binary files /dev/null and b/tests/experiments/metadata/data/job_data_a3tb.db differ
diff --git a/tests/experiments/metadata/graph/graph_data_a003.db b/tests/experiments/metadata/graph/graph_data_a003.db
new file mode 100755
index 0000000000000000000000000000000000000000..ed0362b552850f04244080a7fe51840c7fdaf54c
Binary files /dev/null and b/tests/experiments/metadata/graph/graph_data_a003.db differ
diff --git a/tests/experiments/as_metadata/graph/graph_data_a003.db b/tests/experiments/metadata/graph/graph_data_a3tb.db
similarity index 100%
rename from tests/experiments/as_metadata/graph/graph_data_a003.db
rename to tests/experiments/metadata/graph/graph_data_a3tb.db
diff --git a/tests/experiments/metadata/structures/structure_a003.db b/tests/experiments/metadata/structures/structure_a003.db
new file mode 100644
index 0000000000000000000000000000000000000000..47b5ed9737213db2ab78234bdded4a6d587da00f
Binary files /dev/null and b/tests/experiments/metadata/structures/structure_a003.db differ
diff --git a/tests/experiments/metadata/structures/structure_a007.db b/tests/experiments/metadata/structures/structure_a007.db
new file mode 100644
index 0000000000000000000000000000000000000000..1db581bacb13626eed16874a91a27b31f468662b
Binary files /dev/null and b/tests/experiments/metadata/structures/structure_a007.db differ
diff --git a/tests/experiments/metadata/structures/structure_a3tb.db b/tests/experiments/metadata/structures/structure_a3tb.db
new file mode 100755
index 0000000000000000000000000000000000000000..45dbb0f57b16cd6af2db84180d89348307c2729c
Binary files /dev/null and b/tests/experiments/metadata/structures/structure_a3tb.db differ
diff --git a/tests/test_auth.py b/tests/test_auth.py
index ce59c9da09f466e619b7f82698882932699e7685..5fef2066abbd3e6810b817ed6a9bf6ee296f49a3 100644
--- a/tests/test_auth.py
+++ b/tests/test_auth.py
@@ -1,13 +1,19 @@
+import os
from uuid import uuid4
import pytest
from autosubmit_api.auth import ProtectionLevels, with_auth_token
from autosubmit_api import auth
from autosubmit_api.auth.utils import validate_client
from autosubmit_api.config.basicConfig import APIBasicConfig
+from autosubmit_api import config
from tests.custom_utils import custom_return_value, dummy_response
class TestCommonAuth:
+ def test_mock_env_protection_level(self):
+ assert os.environ.get("PROTECTION_LEVEL") == "NONE"
+ assert config.PROTECTION_LEVEL == "NONE"
+
def test_levels_enum(self):
assert ProtectionLevels.ALL > ProtectionLevels.WRITEONLY
assert ProtectionLevels.WRITEONLY > ProtectionLevels.NONE
diff --git a/tests/test_config.py b/tests/test_config.py
index e2e08d7e6ced5ea1d80249679d4dbf00c83f5d81..64b12455f13e1ce7742ece32dff1b4cbfeb3023c 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -6,10 +6,39 @@ from autosubmit_api.config.basicConfig import APIBasicConfig
from autosubmit_api.config.config_common import AutosubmitConfigResolver
from autosubmit_api.config.ymlConfigStrategy import ymlConfigStrategy
+from tests.conftest import FAKE_EXP_DIR
from tests.custom_utils import custom_return_value
+class TestBasicConfig:
+ def test_api_basic_config(self, fixture_mock_basic_config):
+ APIBasicConfig.read()
+
+ assert os.getenv("AUTOSUBMIT_CONFIGURATION") == os.path.join(
+ FAKE_EXP_DIR, ".autosubmitrc"
+ )
+ assert APIBasicConfig.LOCAL_ROOT_DIR == FAKE_EXP_DIR
+ assert APIBasicConfig.DB_FILE == "autosubmit.db"
+ assert APIBasicConfig.DB_PATH == os.path.join(
+ FAKE_EXP_DIR, APIBasicConfig.DB_FILE
+ )
+ assert APIBasicConfig.AS_TIMES_DB == "as_times.db"
+ assert APIBasicConfig.JOBDATA_DIR == os.path.join(
+ FAKE_EXP_DIR, "metadata", "data"
+ )
+ assert APIBasicConfig.GLOBAL_LOG_DIR == os.path.join(FAKE_EXP_DIR, "logs")
+ assert APIBasicConfig.STRUCTURES_DIR == os.path.join(
+ FAKE_EXP_DIR, "metadata", "structures"
+ )
+ assert APIBasicConfig.HISTORICAL_LOG_DIR == os.path.join(
+ FAKE_EXP_DIR, "metadata", "logs"
+ )
+
+ assert APIBasicConfig.GRAPHDATA_DIR == os.path.join(
+ FAKE_EXP_DIR, "metadata", "graph"
+ )
+
class TestConfigResolver:
def test_simple_init(self, monkeypatch: pytest.MonkeyPatch):
# Conf test decision
diff --git a/tests/test_endpoints_v3.py b/tests/test_endpoints_v3.py
index d410ec94c326d82d226102ba3c5e18b692816fe1..4d5b036720c8f69b3c91a60264214d8d1a8cf698 100644
--- a/tests/test_endpoints_v3.py
+++ b/tests/test_endpoints_v3.py
@@ -89,6 +89,7 @@ class TestExpInfo:
expid = "a003"
response = fixture_client.get(self.endpoint.format(expid=expid))
resp_obj: dict = response.get_json()
+ assert resp_obj["error_message"] == ""
assert resp_obj["error"] == False
assert resp_obj["expid"] == expid
assert resp_obj["total_jobs"] == 8
@@ -97,10 +98,11 @@ class TestExpInfo:
expid = "a3tb"
response = fixture_client.get(self.endpoint.format(expid=expid))
resp_obj: dict = response.get_json()
+ assert resp_obj["error_message"] == ""
assert resp_obj["error"] == False
assert resp_obj["expid"] == expid
assert resp_obj["total_jobs"] == 55
- assert resp_obj["completed_jobs"] == 24
+ assert resp_obj["completed_jobs"] == 28
class TestPerformance:
@@ -113,12 +115,14 @@ class TestPerformance:
expid = "a007"
response = fixture_client.get(self.endpoint.format(expid=expid))
resp_obj: dict = response.get_json()
+ assert resp_obj["error_message"] == ""
assert resp_obj["error"] == False
assert resp_obj["Parallelization"] == 8
expid = "a3tb"
response = fixture_client.get(self.endpoint.format(expid=expid))
resp_obj: dict = response.get_json()
+ assert resp_obj["error_message"] == ""
assert resp_obj["error"] == False
assert resp_obj["Parallelization"] == 768
@@ -129,6 +133,7 @@ class TestPerformance:
expid = "a003"
response = fixture_client.get(self.endpoint.format(expid=expid))
resp_obj: dict = response.get_json()
+ assert resp_obj["error_message"] == ""
assert resp_obj["error"] == False
assert resp_obj["Parallelization"] == 16
@@ -145,6 +150,7 @@ class TestTree:
)
resp_obj: dict = response.get_json()
+ assert resp_obj["error_message"] == ""
assert resp_obj["error"] == False
assert resp_obj["total"] == 8
assert resp_obj["total"] == len(resp_obj["jobs"])
@@ -160,6 +166,7 @@ class TestTree:
)
resp_obj: dict = response.get_json()
+ assert resp_obj["error_message"] == ""
assert resp_obj["error"] == False
assert resp_obj["total"] == 55
assert resp_obj["total"] == len(resp_obj["jobs"])
@@ -179,6 +186,7 @@ class TestRunsList:
response = fixture_client.get(self.endpoint.format(expid=expid))
resp_obj: dict = response.get_json()
+ assert resp_obj["error_message"] == ""
assert resp_obj["error"] == False
assert isinstance(resp_obj["runs"], list)
@@ -192,6 +200,7 @@ class TestRunDetail:
response = fixture_client.get(self.endpoint.format(expid=expid, runId=2))
resp_obj: dict = response.get_json()
+ assert resp_obj["error_message"] == ""
assert resp_obj["error"] == False
assert resp_obj["total"] == 8
@@ -204,6 +213,7 @@ class TestQuick:
response = fixture_client.get(self.endpoint.format(expid=expid))
resp_obj: dict = response.get_json()
+ assert resp_obj["error_message"] == ""
assert resp_obj["error"] == False
assert resp_obj["total"] == len(resp_obj["tree_view"])
assert resp_obj["total"] == len(resp_obj["view_data"])
@@ -220,7 +230,7 @@ class TestGraph:
query_string={"loggedUser": random_user},
)
resp_obj: dict = response.get_json()
-
+ assert resp_obj["error_message"] == ""
assert resp_obj["error"] == False
assert resp_obj["total_jobs"] == len(resp_obj["nodes"])
@@ -234,7 +244,7 @@ class TestGraph:
query_string={"loggedUser": random_user},
)
resp_obj: dict = response.get_json()
-
+ assert resp_obj["error_message"] == ""
assert resp_obj["error"] == False
assert resp_obj["total_jobs"] == len(resp_obj["nodes"])
@@ -246,7 +256,7 @@ class TestGraph:
query_string={"loggedUser": random_user},
)
resp_obj: dict = response.get_json()
-
+ assert resp_obj["error_message"] == ""
assert resp_obj["error"] == False
assert resp_obj["total_jobs"] == len(resp_obj["nodes"])
@@ -258,7 +268,45 @@ class TestGraph:
query_string={"loggedUser": random_user},
)
resp_obj: dict = response.get_json()
+ assert resp_obj["error_message"] == ""
+ assert resp_obj["error"] == False
+ assert resp_obj["total_jobs"] == len(resp_obj["nodes"])
+
+ def test_graph_standard_none_retro3(self, fixture_client: FlaskClient):
+ expid = "a3tb"
+ random_user = str(uuid4())
+ response = fixture_client.get(
+ self.endpoint.format(expid=expid, graph_type="standard", grouped="none"),
+ query_string={"loggedUser": random_user},
+ )
+ resp_obj: dict = response.get_json()
+ assert resp_obj["error_message"] == ""
+ assert resp_obj["error"] == False
+ assert resp_obj["total_jobs"] == len(resp_obj["nodes"])
+
+ def test_graph_standard_datemember_retro3(self, fixture_client: FlaskClient):
+ expid = "a3tb"
+ random_user = str(uuid4())
+ response = fixture_client.get(
+ self.endpoint.format(
+ expid=expid, graph_type="standard", grouped="date-member"
+ ),
+ query_string={"loggedUser": random_user},
+ )
+ resp_obj: dict = response.get_json()
+ assert resp_obj["error_message"] == ""
+ assert resp_obj["error"] == False
+ assert resp_obj["total_jobs"] == len(resp_obj["nodes"])
+ def test_graph_standard_status_retro3(self, fixture_client: FlaskClient):
+ expid = "a3tb"
+ random_user = str(uuid4())
+ response = fixture_client.get(
+ self.endpoint.format(expid=expid, graph_type="standard", grouped="status"),
+ query_string={"loggedUser": random_user},
+ )
+ resp_obj: dict = response.get_json()
+ assert resp_obj["error_message"] == ""
assert resp_obj["error"] == False
assert resp_obj["total_jobs"] == len(resp_obj["nodes"])
@@ -271,6 +319,7 @@ class TestExpCount:
response = fixture_client.get(self.endpoint.format(expid=expid))
resp_obj: dict = response.get_json()
+ assert resp_obj["error_message"] == ""
assert resp_obj["error"] == False
assert resp_obj["total"] == sum(
[resp_obj["counters"][key] for key in resp_obj["counters"]]
@@ -284,6 +333,7 @@ class TestExpCount:
response = fixture_client.get(self.endpoint.format(expid=expid))
resp_obj: dict = response.get_json()
+ assert resp_obj["error_message"] == ""
assert resp_obj["error"] == False
assert resp_obj["total"] == sum(
[resp_obj["counters"][key] for key in resp_obj["counters"]]
@@ -308,6 +358,7 @@ class TestSummary:
)
resp_obj: dict = response.get_json()
+ assert resp_obj["error_message"] == ""
assert resp_obj["error"] == False
assert resp_obj["n_sim"] > 0
@@ -322,6 +373,7 @@ class TestStatistics:
)
resp_obj: dict = response.get_json()
+ assert resp_obj["error_message"] == ""
assert resp_obj["error"] == False
assert resp_obj["Statistics"]["Period"]["From"] == "None"
@@ -334,21 +386,19 @@ class TestCurrentConfig:
response = fixture_client.get(self.endpoint.format(expid=expid))
resp_obj: dict = response.get_json()
+ assert resp_obj["error_message"] == ""
assert resp_obj["error"] == False
assert (
resp_obj["configuration_filesystem"]["CONFIG"]["AUTOSUBMIT_VERSION"]
== "4.0.95"
)
- assert (
- resp_obj["configuration_current_run"]["CONFIG"]["AUTOSUBMIT_VERSION"]
- == "4.0.101"
- )
def test_retrocomp_v3_conf_files(self, fixture_client: FlaskClient):
expid = "a3tb"
response = fixture_client.get(self.endpoint.format(expid=expid))
resp_obj: dict = response.get_json()
+ assert resp_obj["error_message"] == ""
assert resp_obj["error"] == False
assert (
resp_obj["configuration_filesystem"]["conf"]["config"]["AUTOSUBMIT_VERSION"]
@@ -364,6 +414,7 @@ class TestPklInfo:
response = fixture_client.get(self.endpoint.format(expid=expid, timestamp=0))
resp_obj: dict = response.get_json()
+ assert resp_obj["error_message"] == ""
assert resp_obj["error"] == False
assert len(resp_obj["pkl_content"]) == 8
@@ -379,6 +430,7 @@ class TestPklTreeInfo:
response = fixture_client.get(self.endpoint.format(expid=expid, timestamp=0))
resp_obj: dict = response.get_json()
+ assert resp_obj["error_message"] == ""
assert resp_obj["error"] == False
assert len(resp_obj["pkl_content"]) == 8
@@ -394,5 +446,6 @@ class TestExpRunLog:
response = fixture_client.get(self.endpoint.format(expid=expid))
resp_obj: dict = response.get_json()
+ assert resp_obj["error_message"] == ""
assert resp_obj["error"] == False
assert resp_obj["found"] == True
diff --git a/tests/test_graph.py b/tests/test_graph.py
new file mode 100644
index 0000000000000000000000000000000000000000..0dc5beda06a9f91b2944b9fd402bf0ce305f8f7a
--- /dev/null
+++ b/tests/test_graph.py
@@ -0,0 +1,75 @@
+import os
+
+from sqlalchemy import create_engine
+from autosubmit_api.builders.configuration_facade_builder import (
+ AutosubmitConfigurationFacadeBuilder,
+ ConfigurationFacadeDirector,
+)
+from autosubmit_api.builders.joblist_loader_builder import (
+ JobListLoaderBuilder,
+ JobListLoaderDirector,
+)
+from autosubmit_api.database import tables
+from autosubmit_api.database.db_jobdata import ExperimentGraphDrawing
+from autosubmit_api.monitor.monitor import Monitor
+from autosubmit_api.persistance.experiment import ExperimentPaths
+
+
+class TestPopulateDB:
+
+ def test_monitor_dot(self, fixture_mock_basic_config):
+ expid = "a003"
+ job_list_loader = JobListLoaderDirector(
+ JobListLoaderBuilder(expid)
+ ).build_loaded_joblist_loader()
+
+ monitor = Monitor()
+ graph = monitor.create_tree_list(
+ expid,
+ job_list_loader.jobs,
+ None,
+ dict(),
+ False,
+ job_list_loader.job_dictionary,
+ )
+ assert graph
+
+ result = graph.create("dot", format="plain")
+ assert result and len(result) > 0
+
+ def test_process_graph(self, fixture_mock_basic_config):
+ expid = "a003"
+ experimentGraphDrawing = ExperimentGraphDrawing(expid)
+ job_list_loader = JobListLoaderDirector(
+ JobListLoaderBuilder(expid)
+ ).build_loaded_joblist_loader()
+
+ autosubmit_configuration_facade = ConfigurationFacadeDirector(
+ AutosubmitConfigurationFacadeBuilder(expid)
+ ).build_autosubmit_configuration_facade()
+
+ exp_paths = ExperimentPaths(expid)
+ with create_engine(
+ f"sqlite:///{ os.path.abspath(exp_paths.graph_data_db)}"
+ ).connect() as conn:
+ conn.execute(tables.graph_data_table.delete())
+ conn.commit()
+
+ experimentGraphDrawing.calculate_drawing(
+ allJobs=job_list_loader.jobs,
+ independent=False,
+ num_chunks=autosubmit_configuration_facade.chunk_size,
+ job_dictionary=job_list_loader.job_dictionary,
+ )
+
+ assert (
+ experimentGraphDrawing.coordinates
+ and len(experimentGraphDrawing.coordinates) == 8
+ )
+
+ rows = conn.execute(tables.graph_data_table.select()).all()
+
+ assert len(rows) == 8
+ for job in rows:
+ job_name: str = job.job_name
+ assert job_name.startswith(expid)