From 9a94496c19faf870e79cd5cd7cb06bbd74276a07 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Wed, 3 Apr 2024 11:16:30 +0200 Subject: [PATCH 1/3] #73 Close connections for as_times.db --- .../experiment/common_db_requests.py | 68 ++++++++----------- 1 file changed, 30 insertions(+), 38 deletions(-) diff --git a/autosubmit_api/experiment/common_db_requests.py b/autosubmit_api/experiment/common_db_requests.py index 801e7e4..f1252ce 100644 --- a/autosubmit_api/experiment/common_db_requests.py +++ b/autosubmit_api/experiment/common_db_requests.py @@ -16,15 +16,13 @@ DB_FILES_STATUS = os.path.join(APIBasicConfig.LOCAL_ROOT_DIR, "as_metadata", "te def insert_archive_status(status, alatency, abandwidth, clatency, cbandwidth, rtime): try: - conn = create_connection(DB_FILES_STATUS) - sql = ''' INSERT INTO archive_status(status, avg_latency, avg_bandwidth, current_latency, current_bandwidth, response_time, modified ) VALUES(?,?,?,?,?,?,?)''' - # print(row_content) - cur = conn.cursor() - cur.execute(sql, (int(status), alatency, abandwidth, clatency, - cbandwidth, rtime, datetime.today().strftime('%Y-%m-%d-%H:%M:%S'))) - # print(cur) - conn.commit() - return cur.lastrowid + with create_connection(DB_FILES_STATUS) as conn: + sql = ''' INSERT INTO archive_status(status, avg_latency, avg_bandwidth, current_latency, current_bandwidth, response_time, modified ) VALUES(?,?,?,?,?,?,?)''' + cur = conn.cursor() + cur.execute(sql, (int(status), alatency, abandwidth, clatency, + cbandwidth, rtime, datetime.today().strftime('%Y-%m-%d-%H:%M:%S'))) + conn.commit() + return cur.lastrowid except Exception as exp: print((traceback.format_exc())) print(("Error on Insert : " + str(exp))) @@ -37,14 +35,13 @@ def get_last_read_archive_status(): :rtype: 7-tuple """ try: - conn = create_connection(DB_FILES_STATUS) - sql = "SELECT status, avg_latency, avg_bandwidth, current_latency, current_bandwidth, response_time, modified FROM archive_status order by rowid DESC LIMIT 1" - cur = conn.cursor() - cur.execute(sql) - rows = cur.fetchall() - status, alatency, abandwidth, clatency, cbandwidth, rtime, date = rows[0] - return (status, alatency, abandwidth, clatency, cbandwidth, rtime, date) - # print(rows) + with create_connection(DB_FILES_STATUS) as conn: + sql = "SELECT status, avg_latency, avg_bandwidth, current_latency, current_bandwidth, response_time, modified FROM archive_status order by rowid DESC LIMIT 1" + cur = conn.cursor() + cur.execute(sql) + rows = cur.fetchall() + status, alatency, abandwidth, clatency, cbandwidth, rtime, date = rows[0] + return (status, alatency, abandwidth, clatency, cbandwidth, rtime, date) except Exception as exp: print((traceback.format_exc())) print(("Error on Get Last : " + str(exp))) @@ -102,14 +99,14 @@ def _get_exp_status(): :rtype: 4-tuple (int, str, str, int) """ try: - conn = create_connection(os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB)) - conn.text_factory = str - cur = conn.cursor() - cur.execute( - "SELECT exp_id, name, status, seconds_diff FROM experiment_status") - rows = cur.fetchall() - return rows - except Exception as exp: + with create_connection(os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB)) as conn: + conn.text_factory = str + cur = conn.cursor() + cur.execute( + "SELECT exp_id, name, status, seconds_diff FROM experiment_status") + rows = cur.fetchall() + return rows + except Exception: print((traceback.format_exc())) return dict() @@ -121,19 +118,14 @@ def _get_specific_exp_status(expid): :rtype: 4-tuple (int, str, str, int) """ try: - # print("Honk") - conn = create_connection(os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB)) - cur = conn.cursor() - cur.execute( - "SELECT exp_id, name, status, seconds_diff FROM experiment_status WHERE name=?", (expid,)) - row = cur.fetchone() - if row == None: - return (0, expid, "NOT RUNNING", 0) - # print(row) - return row + with create_connection(os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB)) as conn: + cur = conn.cursor() + cur.execute( + "SELECT exp_id, name, status, seconds_diff FROM experiment_status WHERE name=?", (expid,)) + row = cur.fetchone() + if row == None: + return (0, expid, "NOT RUNNING", 0) + return row except Exception as exp: print((traceback.format_exc())) return (0, expid, "NOT RUNNING", 0) - - -# UPDATES -- GitLab From c6bc33e2fc9f3d2637b445f8e13da7c917a43aa4 Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Thu, 4 Apr 2024 10:56:44 +0200 Subject: [PATCH 2/3] refactor common_db_requests --- .../experiment/common_db_requests.py | 125 ++++++++---------- 1 file changed, 56 insertions(+), 69 deletions(-) diff --git a/autosubmit_api/experiment/common_db_requests.py b/autosubmit_api/experiment/common_db_requests.py index f1252ce..187cd45 100644 --- a/autosubmit_api/experiment/common_db_requests.py +++ b/autosubmit_api/experiment/common_db_requests.py @@ -2,25 +2,52 @@ import os import traceback import sqlite3 from datetime import datetime - +from autosubmit_api.logger import logger from autosubmit_api.config.basicConfig import APIBasicConfig +from autosubmit_api.database import tables +from autosubmit_api.database.common import create_as_times_db_engine + APIBasicConfig.read() +DB_FILES_STATUS = os.path.join( + APIBasicConfig.LOCAL_ROOT_DIR, "as_metadata", "test", APIBasicConfig.FILE_STATUS_DB +) # "/esarchive/autosubmit/as_metadata/test/status.db" + -DB_FILE_AS_TIMES = os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB) # "/esarchive/autosubmit/as_times.db" -DB_FILES_STATUS = os.path.join(APIBasicConfig.LOCAL_ROOT_DIR, "as_metadata", "test", APIBasicConfig.FILE_STATUS_DB) # "/esarchive/autosubmit/as_metadata/test/status.db" -# PATH_DB_DATA = "/esarchive/autosubmit/as_metadata/data/" +# STATUS ARCHIVE # Might be removed soon -# STATUS ARCHIVE +def create_connection(db_file): + # type: (str) -> sqlite3.Connection + """ + Create a database connection to the SQLite database specified by db_file. + :param db_file: database file name + :return: Connection object or None + """ + try: + conn = sqlite3.connect(db_file) + return conn + except Exception as exc: + logger.error(exc) + def insert_archive_status(status, alatency, abandwidth, clatency, cbandwidth, rtime): try: with create_connection(DB_FILES_STATUS) as conn: - sql = ''' INSERT INTO archive_status(status, avg_latency, avg_bandwidth, current_latency, current_bandwidth, response_time, modified ) VALUES(?,?,?,?,?,?,?)''' + sql = """ INSERT INTO archive_status(status, avg_latency, avg_bandwidth, current_latency, current_bandwidth, response_time, modified ) VALUES(?,?,?,?,?,?,?)""" cur = conn.cursor() - cur.execute(sql, (int(status), alatency, abandwidth, clatency, - cbandwidth, rtime, datetime.today().strftime('%Y-%m-%d-%H:%M:%S'))) + cur.execute( + sql, + ( + int(status), + alatency, + abandwidth, + clatency, + cbandwidth, + rtime, + datetime.today().strftime("%Y-%m-%d-%H:%M:%S"), + ), + ) conn.commit() return cur.lastrowid except Exception as exp: @@ -47,35 +74,24 @@ def get_last_read_archive_status(): print(("Error on Get Last : " + str(exp))) return (False, None, None, None, None, None, None) -# INSERTIONS - -def create_connection(db_file): - # type: (str) -> sqlite3.Connection - """ - Create a database connection to the SQLite database specified by db_file. - :param db_file: database file name - :return: Connection object or None - """ - try: - conn = sqlite3.connect(db_file) - return conn - except Exception as exp: - print(exp) - # SELECTS + def get_experiment_status(): """ Gets table experiment_status as dictionary conn is expected to reference as_times.db """ - # conn = create_connection(DB_FILE_AS_TIMES) experiment_status = dict() - current_table = _get_exp_status() - for item in current_table: - exp_id, name, status, seconds_diff = item - experiment_status[name] = status + try: + with create_as_times_db_engine().connect() as conn: + cursor = conn.execute(tables.experiment_status_table.select()) + for row in cursor: + experiment_status[row.name] = row.status + except Exception as exc: + logger.error(f"Exception while reading experiment_status: {exc}") + logger.error(traceback.format_exc()) return experiment_status @@ -87,45 +103,16 @@ def get_specific_experiment_status(expid): :return: name of experiment and status :rtype: 2-tuple (name, status) """ - exp_id, name, status, seconds_diff = _get_specific_exp_status(expid) - print(("{} {} {} {}".format(exp_id, name, status, seconds_diff))) - return (name, status) - - -def _get_exp_status(): - """ - Get all registers from experiment_status.\n - :return: row content: exp_id, name, status, seconds_diff - :rtype: 4-tuple (int, str, str, int) - """ try: - with create_connection(os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB)) as conn: - conn.text_factory = str - cur = conn.cursor() - cur.execute( - "SELECT exp_id, name, status, seconds_diff FROM experiment_status") - rows = cur.fetchall() - return rows - except Exception: - print((traceback.format_exc())) - return dict() - - -def _get_specific_exp_status(expid): - """ - Get all registers from experiment_status.\n - :return: row content: exp_id, name, status, seconds_diff - :rtype: 4-tuple (int, str, str, int) - """ - try: - with create_connection(os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB)) as conn: - cur = conn.cursor() - cur.execute( - "SELECT exp_id, name, status, seconds_diff FROM experiment_status WHERE name=?", (expid,)) - row = cur.fetchone() - if row == None: - return (0, expid, "NOT RUNNING", 0) - return row - except Exception as exp: - print((traceback.format_exc())) - return (0, expid, "NOT RUNNING", 0) + with create_as_times_db_engine().connect() as conn: + row = conn.execute( + tables.experiment_status_table.select().where( + tables.experiment_status_table.c.name == expid + ) + ).one_or_none() + except Exception as exc: + logger.error(f"Exception while reading experiment_status for {expid}: {exc}") + logger.error(traceback.format_exc()) + if row: + return (row.name, row.status) + return (expid, "NOT RUNNING") -- GitLab From 9c242dc5146a5b31fe8d48fbc12e23654f150283 Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Thu, 4 Apr 2024 15:09:01 +0200 Subject: [PATCH 3/3] close more connections #73 --- autosubmit_api/autosubmit_legacy/job/job_list.py | 6 ------ autosubmit_api/database/db_common.py | 1 + autosubmit_api/database/db_structure.py | 16 ++++++++-------- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/autosubmit_api/autosubmit_legacy/job/job_list.py b/autosubmit_api/autosubmit_legacy/job/job_list.py index c9ad0cb..766a906 100644 --- a/autosubmit_api/autosubmit_legacy/job/job_list.py +++ b/autosubmit_api/autosubmit_legacy/job/job_list.py @@ -37,7 +37,6 @@ from autosubmit_api.components.jobs import utils as JUtils from autosubmit_api.monitor.monitor import Monitor from autosubmit_api.common.utils import Status from bscearth.utils.date import date2str, parse_date -from autosubmit_api.experiment import common_db_requests as DbRequests from autosubmit_api.autosubmit_legacy.job.job_package_persistence import JobPackagePersistence # from autosubmit_legacy.job.tree import Tree from autosubmit_api.database import db_structure as DbStructure @@ -590,11 +589,6 @@ class JobList: path_local_root = basic_config.LOCAL_ROOT_DIR path_structure = basic_config.STRUCTURES_DIR db_file = os.path.join(path_local_root, basic_config.DB_FILE) - conn = DbRequests.create_connection(db_file) - # job_data = None - # Job information from worker database - # job_times = dict() # REMOVED: DbRequests.get_times_detail_by_expid(conn, expid) - conn.close() # Job information from job historic data # print("Get current job data structure...") experiment_history = ExperimentHistoryDirector(ExperimentHistoryBuilder(expid)).build_reader_experiment_history() diff --git a/autosubmit_api/database/db_common.py b/autosubmit_api/database/db_common.py index 961bba6..40443b2 100644 --- a/autosubmit_api/database/db_common.py +++ b/autosubmit_api/database/db_common.py @@ -382,6 +382,7 @@ def get_experiment_by_id(expid): cursor.execute(query) headers = get_headers_sqlite(cursor) row = cursor.fetchone() + close_conn(conn, cursor) if row is not None: obj = map_row_result_to_dict_sqlite(row, headers) result['id'] = obj["id"] diff --git a/autosubmit_api/database/db_structure.py b/autosubmit_api/database/db_structure.py index 0866488..06ad129 100644 --- a/autosubmit_api/database/db_structure.py +++ b/autosubmit_api/database/db_structure.py @@ -24,7 +24,6 @@ def get_structure(expid, structures_path): os.open(db_structure_path, os.O_WRONLY | os.O_CREAT, 0o777) # open(db_structure_path, "w") # print(db_structure_path) - conn = create_connection(db_structure_path) create_table_query = textwrap.dedent( '''CREATE TABLE IF NOT EXISTS experiment_structure ( @@ -32,7 +31,8 @@ def get_structure(expid, structures_path): e_to text NOT NULL, UNIQUE(e_from,e_to) );''') - create_table(conn, create_table_query) + with create_connection(db_structure_path) as conn: + create_table(conn, create_table_query) current_table = _get_exp_structure(db_structure_path) # print("Current table: ") # print(current_table) @@ -92,12 +92,12 @@ def _get_exp_structure(path): :rtype: 4-tuple (int, str, str, int) """ try: - conn = create_connection(path) - conn.text_factory = str - cur = conn.cursor() - cur.execute( - "SELECT e_from, e_to FROM experiment_structure") - rows = cur.fetchall() + with create_connection(path) as conn: + conn.text_factory = str + cur = conn.cursor() + cur.execute( + "SELECT e_from, e_to FROM experiment_structure") + rows = cur.fetchall() return rows except Exception as exp: print((traceback.format_exc())) -- GitLab