diff --git a/autosubmit_api/database/adapters/experiment_run.py b/autosubmit_api/database/adapters/experiment_run.py index 541fc7d7f29fa5bf35b78b868c8d1f075c749a00..aaed9485af7b7edc2ed22caf86f7ec9e27ddf1d2 100644 --- a/autosubmit_api/database/adapters/experiment_run.py +++ b/autosubmit_api/database/adapters/experiment_run.py @@ -15,14 +15,14 @@ class ExperimentRunDbAdapter: schema=expid, ) - def get_last_run(self) -> Optional[Dict[str,str]]: + def get_last_run(self) -> Optional[Dict[str, str]]: """ Gets last run of the experiment """ with self.table_manager.get_connection() as conn: row = conn.execute( select(self.table_manager.table) - .order_by(tables.ExperimentRunTable.run_id.desc()) + .order_by(tables.experiment_run_table.c.run_id.desc()) .limit(1) ).one_or_none() diff --git a/autosubmit_api/database/tables.py b/autosubmit_api/database/tables.py index 38dd5f4196d42492491cb755fd08fc00378fb2a8..c28b56e9157dc8661248740af562ee836bdd23ff 100644 --- a/autosubmit_api/database/tables.py +++ b/autosubmit_api/database/tables.py @@ -5,11 +5,15 @@ from autosubmit.database.tables import ( ExperimentTable, experiment_run_table, JobDataTable, + ExperimentStructureTable, + table_change_schema, ExperimentStatusTable, JobPackageTable, WrapperJobPackageTable, ) +table_change_schema = table_change_schema + ## SQLAlchemy ORM tables class DetailsTable(BaseTable): """ @@ -59,3 +63,6 @@ wrapper_job_package_table: Table = WrapperJobPackageTable.__table__ # Job Data TABLES job_data_table: Table = JobDataTable.__table__ experiment_run_table: Table = experiment_run_table + +# Structure TABLES +experiment_structure_table: Table = ExperimentStructureTable.__table__ diff --git a/tests/conftest.py b/tests/conftest.py index be699e00c6651b16daf191f6618656cbe26787b3..1728ee8bbe46c668f0c0751d098e31a08bffe927 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,31 +2,55 @@ # Reference: https://docs.pytest.org/en/latest/reference/fixtures.html#conftest-py-sharing-fixtures-across-multiple-files import os +import tempfile +from typing import Tuple from flask import Flask import pytest -from autosubmitconfigparser.config.basicconfig import BasicConfig from autosubmit_api.app import create_app from autosubmit_api.config.basicConfig import APIBasicConfig from autosubmit_api import config -from tests.custom_utils import custom_return_value +from tests import utils +from sqlalchemy import Engine, create_engine FAKE_EXP_DIR = "./tests/experiments/" +DEFAULT_DATABASE_CONN_URL = ( + "postgresql://postgres:mysecretpassword@localhost:5432/autosubmit_test" +) + + +# FIXTURES #### + +# Config fixtures -#### FIXTURES #### @pytest.fixture(autouse=True) def fixture_disable_protection(monkeypatch: pytest.MonkeyPatch): + """ + This fixture disables the protection level for all the tests. + + Autouse is set, so, no need to put this fixture in the test function. + """ monkeypatch.setattr(config, "PROTECTION_LEVEL", "NONE") monkeypatch.setenv("PROTECTION_LEVEL", "NONE") -@pytest.fixture -def fixture_mock_basic_config(monkeypatch: pytest.MonkeyPatch): - # Get APIBasicConfig from file - monkeypatch.setenv("AUTOSUBMIT_CONFIGURATION", os.path.join(FAKE_EXP_DIR, ".autosubmitrc")) +@pytest.fixture( + params=[ + pytest.param("fixture_sqlite", marks=pytest.mark.sqlite), + pytest.param("fixture_pg", marks=pytest.mark.pg), + ] +) +def fixture_mock_basic_config(request: pytest.FixtureRequest): + """ + Sets a mock basic config for the tests. + """ + request.getfixturevalue(request.param) yield APIBasicConfig +# Flask app fixtures + + @pytest.fixture def fixture_app(fixture_mock_basic_config): app = create_app() @@ -46,3 +70,169 @@ def fixture_client(fixture_app: Flask): @pytest.fixture def fixture_runner(fixture_app: Flask): return fixture_app.test_cli_runner() + + +# Fixtures sqlite + + +@pytest.fixture(scope="session") +def fixture_temp_dir_copy(): + """ + Fixture that copies the contents of the FAKE_EXP_DIR to a temporary directory with rsync + """ + with tempfile.TemporaryDirectory() as tempdir: + # Copy all files recursively + os.system(f"rsync -r {FAKE_EXP_DIR} {tempdir}") + yield tempdir + + +@pytest.fixture(scope="session") +def fixture_gen_rc_sqlite(fixture_temp_dir_copy: str): + """ + Fixture that generates a .autosubmitrc file in the temporary directory + """ + rc_file = os.path.join(fixture_temp_dir_copy, ".autosubmitrc") + with open(rc_file, "w") as f: + f.write( + "\n".join( + [ + "[database]", + f"path = {fixture_temp_dir_copy}", + "filename = autosubmit.db", + "backend = sqlite", + "[local]", + f"path = {fixture_temp_dir_copy}", + "[globallogs]", + f"path = {fixture_temp_dir_copy}/logs", + "[historicdb]", + f"path = {fixture_temp_dir_copy}/metadata/data", + "[structures]", + f"path = {fixture_temp_dir_copy}/metadata/structures", + "[historiclog]", + f"path = {fixture_temp_dir_copy}/metadata/logs", + "[graph]", + f"path = {fixture_temp_dir_copy}/metadata/graph", + ] + ) + ) + yield fixture_temp_dir_copy + + +@pytest.fixture +def fixture_sqlite(fixture_gen_rc_sqlite: str, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv( + "AUTOSUBMIT_CONFIGURATION", os.path.join(fixture_gen_rc_sqlite, ".autosubmitrc") + ) + yield fixture_gen_rc_sqlite + + +# Fixtures Postgres + + +@pytest.fixture(scope="session") +def fixture_temp_dir_copy_exclude_db(): + """ + Fixture that copies the contents of the FAKE_EXP_DIR to a temporary directory with rsync + and exclues .db files + """ + with tempfile.TemporaryDirectory() as tempdir: + # Copy all files recursively excluding .db files + os.system(f"rsync -r --exclude '*.db' {FAKE_EXP_DIR} {tempdir}") + yield tempdir + + +@pytest.fixture(scope="session") +def fixture_gen_rc_pg(fixture_temp_dir_copy_exclude_db: str): + """ + Fixture that generates a .autosubmitrc file in the temporary directory + """ + rc_file = os.path.join(fixture_temp_dir_copy_exclude_db, ".autosubmitrc") + conn_url = os.environ.get("PYTEST_DATABASE_CONN_URL", DEFAULT_DATABASE_CONN_URL) + with open(rc_file, "w") as f: + f.write( + "\n".join( + [ + "[database]", + f"path = {fixture_temp_dir_copy_exclude_db}", + "backend = postgres", + f"conn_url = {conn_url}", + "[local]", + f"path = {fixture_temp_dir_copy_exclude_db}", + "[globallogs]", + f"path = {fixture_temp_dir_copy_exclude_db}/logs", + "[historicdb]", + f"path = {fixture_temp_dir_copy_exclude_db}/metadata/data", + "[structures]", + f"path = {fixture_temp_dir_copy_exclude_db}/metadata/structures", + "[historiclog]", + f"path = {fixture_temp_dir_copy_exclude_db}/metadata/logs", + "[graph]", + f"path = {fixture_temp_dir_copy_exclude_db}/metadata/graph", + ] + ) + ) + yield fixture_temp_dir_copy_exclude_db + + +@pytest.fixture +def fixture_pg_db(fixture_gen_rc_pg: str): + """ + This fixture cleans and setup a PostgreSQL database for testing purposes. + """ + conn_url = os.environ.get("PYTEST_DATABASE_CONN_URL", DEFAULT_DATABASE_CONN_URL) + engine = create_engine(conn_url) + + with engine.connect() as conn: + utils.setup_pg_db(conn) + conn.commit() + + yield (fixture_gen_rc_pg, engine) + + with engine.connect() as conn: + utils.setup_pg_db(conn) + conn.commit() + + +@pytest.fixture +def fixture_pg_db_copy_all(fixture_pg_db: Tuple[str, Engine]): + """ + This fixture recursively search all the .db files in the FAKE_EXP_DIR and copies them to the test database + """ + engine = fixture_pg_db[1] + # Get .db files absolute paths from the FAKE_EXP_DIR recursively + all_files = [] + for root, dirs, files in os.walk(FAKE_EXP_DIR): + for filepath in files: + if filepath.endswith(".db"): + all_files.append(os.path.join(root, filepath)) + + for filepath in all_files: + # Infer which type of DB is this + if "metadata/structures" in filepath: + utils.copy_structure_db(filepath, engine) + elif "metadata/data" in filepath: + utils.copy_job_data_db(filepath, engine) + elif "metadata/graph" in filepath: + utils.copy_graph_data_db(filepath, engine) + elif "autosubmit.db" in filepath: + utils.copy_autosubmit_db(filepath, engine) + elif "as_times.db" in filepath: + utils.copy_as_times_db(filepath, engine) + elif "pkl/job_packages" in filepath: + utils.copy_job_packages_db(filepath, engine) + + yield fixture_pg_db + + +@pytest.fixture +def fixture_pg( + fixture_pg_db_copy_all: Tuple[str, Engine], monkeypatch: pytest.MonkeyPatch +): + """ + This fixture cleans and setup a PostgreSQL database for testing purposes. + """ + monkeypatch.setenv( + "AUTOSUBMIT_CONFIGURATION", + os.path.join(fixture_pg_db_copy_all[0], ".autosubmitrc"), + ) + yield fixture_pg_db_copy_all[0] diff --git a/tests/custom_utils.py b/tests/custom_utils.py deleted file mode 100644 index 9148a98a887c26f71c409f8ef94e2dd10ee99ab3..0000000000000000000000000000000000000000 --- a/tests/custom_utils.py +++ /dev/null @@ -1,12 +0,0 @@ -from http import HTTPStatus - - -def dummy_response(*args, **kwargs): - return "Hello World!", HTTPStatus.OK - - -def custom_return_value(value=None): - def blank_func(*args, **kwargs): - return value - - return blank_func diff --git a/tests/experiments/.autosubmitrc b/tests/experiments/.autosubmitrc deleted file mode 100644 index 4b894ee2624733659136dda281f83c3666dfb9d0..0000000000000000000000000000000000000000 --- a/tests/experiments/.autosubmitrc +++ /dev/null @@ -1,22 +0,0 @@ -[database] -path = ./tests/experiments/ -filename = autosubmit.db -backend = sqlite - -[local] -path = ./tests/experiments/ - -[globallogs] -path = ./tests/experiments/logs - -[historicdb] -path = ./tests/experiments/metadata/data - -[structures] -path = ./tests/experiments/metadata/structures - -[historiclog] -path = ./tests/experiments/metadata/logs - -[graph] -path = ./tests/experiments/metadata/graph \ No newline at end of file diff --git a/tests/experiments/metadata/data/job_data_a007.db b/tests/experiments/metadata/data/job_data_a007.db index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..7fcd6487e0efd128f66e1889e268c268079f8b1b 100755 Binary files a/tests/experiments/metadata/data/job_data_a007.db and b/tests/experiments/metadata/data/job_data_a007.db differ diff --git a/tests/test_auth.py b/tests/test_auth.py index 5fef2066abbd3e6810b817ed6a9bf6ee296f49a3..9bfc5eed2d9b506b3e007a1b31cd881c7398fbb2 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -6,7 +6,7 @@ from autosubmit_api import auth from autosubmit_api.auth.utils import validate_client from autosubmit_api.config.basicConfig import APIBasicConfig from autosubmit_api import config -from tests.custom_utils import custom_return_value, dummy_response +from tests.utils import custom_return_value, dummy_response class TestCommonAuth: diff --git a/tests/test_config.py b/tests/test_config.py index 64b12455f13e1ce7742ece32dff1b4cbfeb3023c..748da2bee66cc2970cbd066f525db58d89e86e8e 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -6,39 +6,10 @@ from autosubmit_api.config.basicConfig import APIBasicConfig from autosubmit_api.config.config_common import AutosubmitConfigResolver from autosubmit_api.config.ymlConfigStrategy import ymlConfigStrategy -from tests.conftest import FAKE_EXP_DIR -from tests.custom_utils import custom_return_value +from tests.utils import custom_return_value -class TestBasicConfig: - def test_api_basic_config(self, fixture_mock_basic_config): - APIBasicConfig.read() - - assert os.getenv("AUTOSUBMIT_CONFIGURATION") == os.path.join( - FAKE_EXP_DIR, ".autosubmitrc" - ) - assert APIBasicConfig.LOCAL_ROOT_DIR == FAKE_EXP_DIR - assert APIBasicConfig.DB_FILE == "autosubmit.db" - assert APIBasicConfig.DB_PATH == os.path.join( - FAKE_EXP_DIR, APIBasicConfig.DB_FILE - ) - assert APIBasicConfig.AS_TIMES_DB == "as_times.db" - assert APIBasicConfig.JOBDATA_DIR == os.path.join( - FAKE_EXP_DIR, "metadata", "data" - ) - assert APIBasicConfig.GLOBAL_LOG_DIR == os.path.join(FAKE_EXP_DIR, "logs") - assert APIBasicConfig.STRUCTURES_DIR == os.path.join( - FAKE_EXP_DIR, "metadata", "structures" - ) - assert APIBasicConfig.HISTORICAL_LOG_DIR == os.path.join( - FAKE_EXP_DIR, "metadata", "logs" - ) - - assert APIBasicConfig.GRAPHDATA_DIR == os.path.join( - FAKE_EXP_DIR, "metadata", "graph" - ) - class TestConfigResolver: def test_simple_init(self, monkeypatch: pytest.MonkeyPatch): # Conf test decision @@ -61,7 +32,7 @@ class TestConfigResolver: class TestYMLConfigStrategy: def test_exclusive(self, fixture_mock_basic_config): wrapper = ymlConfigStrategy("a007", fixture_mock_basic_config) - assert True == wrapper.get_exclusive(JobSection.SIM) + assert True is wrapper.get_exclusive(JobSection.SIM) wrapper = ymlConfigStrategy("a003", fixture_mock_basic_config) - assert False == wrapper.get_exclusive(JobSection.SIM) + assert False is wrapper.get_exclusive(JobSection.SIM) diff --git a/tests/test_database.py b/tests/test_database.py index 518523b3705220b6a9d8bcb41fe485f215c7b72b..632c05337d37cac13ce12aafd8c4c6cf121bd48a 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -13,7 +13,7 @@ def count_pid_lsof(pid): class TestDatabase: - def test_open_files(self, fixture_mock_basic_config): + def test_open_files(self, fixture_sqlite): current_pid = os.getpid() counter = count_pid_lsof(current_pid) diff --git a/tests/test_endpoints_v3.py b/tests/test_endpoints_v3.py index e38c75bce6afbf34e71df2700fca5355efe84bba..429fe0e6b74a0d585db74ed243171986f8663458 100644 --- a/tests/test_endpoints_v3.py +++ b/tests/test_endpoints_v3.py @@ -14,7 +14,6 @@ class TestLogin: def test_not_allowed_client( self, fixture_client: FlaskClient, - fixture_mock_basic_config: APIBasicConfig, monkeypatch: pytest.MonkeyPatch, ): monkeypatch.setattr(APIBasicConfig, "ALLOWED_CLIENTS", []) @@ -28,7 +27,6 @@ class TestLogin: def test_redirect( self, fixture_client: FlaskClient, - fixture_mock_basic_config: APIBasicConfig, monkeypatch: pytest.MonkeyPatch, ): random_referer = str(f"https://${str(uuid4())}/") diff --git a/tests/test_endpoints_v4.py b/tests/test_endpoints_v4.py index dc83894f9c90849e7358a6ae11efc414c0f1695f..6e7fb38d1d36c6b412ca8224f60353592b24c334 100644 --- a/tests/test_endpoints_v4.py +++ b/tests/test_endpoints_v4.py @@ -7,7 +7,7 @@ import jwt import pytest from autosubmit_api import config from autosubmit_api.views.v4 import PAGINATION_LIMIT_DEFAULT, ExperimentJobsViewOptEnum -from tests.custom_utils import custom_return_value +from tests.utils import custom_return_value class TestCASV2Login: diff --git a/tests/test_fixtures.py b/tests/test_fixtures.py new file mode 100644 index 0000000000000000000000000000000000000000..75809d4773d3d701839c8b0df4584f1d1ee5c20b --- /dev/null +++ b/tests/test_fixtures.py @@ -0,0 +1,114 @@ +import os +from typing import Tuple + +import pytest +from sqlalchemy import Engine, select + +from autosubmit_api.config.basicConfig import APIBasicConfig +from autosubmit_api.database import tables +from tests.utils import get_schema_names + + +class TestSQLiteFixtures: + def test_fixture_temp_dir_copy(self, fixture_temp_dir_copy: str): + """ + Test if all the files are copied from FAKEDIR to the temporary directory + """ + FILES_SHOULD_EXIST = [ + "a003/conf/minimal.yml", + "metadata/data/job_data_a007.db", + ] + for file in FILES_SHOULD_EXIST: + assert os.path.exists(os.path.join(fixture_temp_dir_copy, file)) + + def test_fixture_gen_rc_sqlite(self, fixture_gen_rc_sqlite: str): + """ + Test if the .autosubmitrc file is generated and the environment variable is set + """ + rc_file = os.path.join(fixture_gen_rc_sqlite, ".autosubmitrc") + + # File should exist + assert os.path.exists(rc_file) + + with open(rc_file, "r") as f: + content = f.read() + assert "[database]" in content + assert f"path = {fixture_gen_rc_sqlite}" in content + assert "filename = autosubmit.db" in content + assert "backend = sqlite" in content + + @pytest.mark.skip(reason="TODO: Fix this test") + def test_mock_basic_config( + self, fixture_mock_basic_config: APIBasicConfig, fixture_gen_rc_sqlite: str + ): + rc_file = os.path.join(fixture_gen_rc_sqlite, ".autosubmitrc") + # Environment variable should be set and should point to the .autosubmitrc file + assert "AUTOSUBMIT_CONFIGURATION" in os.environ and os.path.exists( + os.environ["AUTOSUBMIT_CONFIGURATION"] + ) + assert os.environ["AUTOSUBMIT_CONFIGURATION"] == rc_file + + # Reading the configuration file + APIBasicConfig.read() + assert APIBasicConfig.GRAPHDATA_DIR == f"{fixture_gen_rc_sqlite}/metadata/graph" + assert APIBasicConfig.LOCAL_ROOT_DIR == fixture_gen_rc_sqlite + assert APIBasicConfig.DATABASE_BACKEND == "sqlite" + assert APIBasicConfig.DB_DIR == fixture_gen_rc_sqlite + assert APIBasicConfig.DB_FILE == "autosubmit.db" + + +class TestPostgresFixtures: + def test_fixture_temp_dir_copy_exclude_db( + self, fixture_temp_dir_copy_exclude_db: str + ): + """ + Test if all the files are copied from FAKEDIR to the temporary directory except .db files + """ + FILES_SHOULD_EXIST = [ + "a003/conf/minimal.yml", + ] + FILES_SHOULD_EXCLUDED = ["metadata/data/job_data_a007.db"] + for file in FILES_SHOULD_EXIST: + assert os.path.exists(os.path.join(fixture_temp_dir_copy_exclude_db, file)) + + for file in FILES_SHOULD_EXCLUDED: + assert not os.path.exists( + os.path.join(fixture_temp_dir_copy_exclude_db, file) + ) + + def test_fixture_gen_rc_postgres(self, fixture_gen_rc_pg: str): + """ + Test if the .autosubmitrc file is generated and the environment variable is set + """ + rc_file = os.path.join(fixture_gen_rc_pg, ".autosubmitrc") + + # File should exist + assert os.path.exists(rc_file) + + with open(rc_file, "r") as f: + content = f.read() + assert "[database]" in content + assert "backend = postgres" in content + assert "postgresql://" in content + assert fixture_gen_rc_pg in content + + def test_fixture_pg_db(self, fixture_pg_db: Tuple[str, Engine]): + engine = fixture_pg_db[1] + + # Check if the public schema exists and is the only one + with engine.connect() as conn: + schema_names = get_schema_names(conn) + assert schema_names == ["public"] + + def test_fixture_pg_db_copy_all(self, fixture_pg_db_copy_all: Tuple[str, Engine]): + engine = fixture_pg_db_copy_all[1] + + # Check if the experiment and details tables are copied + with engine.connect() as conn: + exp_rows = conn.execute(select(tables.ExperimentTable)).all() + details_rows = conn.execute(select(tables.DetailsTable)).all() + + assert len(exp_rows) > 0 + assert len(details_rows) > 0 + + # TODO: Check if the other tables are copied diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..186a17e198f1d97ac4c1441cc574d2521b689625 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,193 @@ +from http import HTTPStatus +import re +from typing import List +from sqlalchemy import Connection, Engine, create_engine, insert, select, text + +from autosubmit_api.database import tables +from sqlalchemy.schema import CreateSchema, CreateTable + + +def dummy_response(*args, **kwargs): + return "Hello World!", HTTPStatus.OK + + +def custom_return_value(value=None): + def blank_func(*args, **kwargs): + return value + + return blank_func + + +def get_schema_names(conn: Connection) -> List[str]: + """ + Get all schema names that are not from the system + """ + results = conn.execute( + text( + "SELECT schema_name FROM information_schema.schemata WHERE schema_name NOT LIKE 'pg_%' AND schema_name != 'information_schema'" + ) + ).all() + return [res[0] for res in results] + + +def setup_pg_db(conn: Connection): + """ + Resets database by dropping all schemas except the system ones and restoring the public schema + """ + # Get all schema names that are not from the system + schema_names = get_schema_names(conn) + + # Drop all schemas + for schema_name in schema_names: + conn.execute(text(f'DROP SCHEMA IF EXISTS "{schema_name}" CASCADE')) + + # Restore default public schema + conn.execute(text("CREATE SCHEMA public")) + conn.execute(text("GRANT ALL ON SCHEMA public TO public")) + conn.execute(text("GRANT ALL ON SCHEMA public TO postgres")) + + +def copy_structure_db(filepath: str, engine: Engine): + """ + This function copies the content of the FAKE_EXP_DIR/metadata/structures to the Postgres database + """ + # Get the xxxx from structure_xxxx.db with regex + match = re.search(r"structure_(\w+)\.db", filepath) + expid = match.group(1) + + # Get SQLite source data + source_as_db = create_engine(f"sqlite:///{filepath}") + with source_as_db.connect() as source_conn: + structures_rows = source_conn.execute( + select(tables.ExperimentStructureTable) + ).all() + + # Copy data to the Postgres database + with engine.connect() as conn: + conn.execute(CreateSchema(expid, if_not_exists=True)) + target_table = tables.table_change_schema( + expid, tables.ExperimentStructureTable + ) + conn.execute(CreateTable(target_table, if_not_exists=True)) + if len(structures_rows) > 0: + conn.execute( + insert(target_table), [row._mapping for row in structures_rows] + ) + conn.commit() + + +def copy_job_data_db(filepath: str, engine: Engine): + """ + This function copies the content of the FAKE_EXP_DIR/metadata/data to the Postgres database + """ + # Get the xxxx from job_data_xxxx.db with regex + match = re.search(r"job_data_(\w+)\.db", filepath) + expid = match.group(1) + # Get SQLite source data + source_as_db = create_engine(f"sqlite:///{filepath}") + with source_as_db.connect() as source_conn: + job_data_rows = source_conn.execute(select(tables.JobDataTable)).all() + exprun_rows = source_conn.execute(select(tables.experiment_run_table)).all() + + # Copy data to the Postgres database + with engine.connect() as conn: + conn.execute(CreateSchema(expid, if_not_exists=True)) + # Job data + target_table = tables.table_change_schema(expid, tables.JobDataTable) + conn.execute(CreateTable(target_table, if_not_exists=True)) + if len(job_data_rows) > 0: + conn.execute(insert(target_table),[row._mapping for row in job_data_rows]) + # Experiment run + target_table = tables.table_change_schema(expid, tables.experiment_run_table) + conn.execute(CreateTable(target_table, if_not_exists=True)) + if len(exprun_rows) > 0: + conn.execute(insert(target_table),[row._mapping for row in exprun_rows]) + conn.commit() + + +def copy_graph_data_db(filepath: str, engine: Engine): + """ + This function copies the content of the FAKE_EXP_DIR/metadata/graph to the Postgres database + """ + # Get the xxxx from graph_xxxx.db with regex + match = re.search(r"graph_data_(\w+)\.db", filepath) + expid = match.group(1) + + # Get SQLite source data + source_as_db = create_engine(f"sqlite:///{filepath}") + with source_as_db.connect() as source_conn: + graph_rows = source_conn.execute(select(tables.GraphDataTable)).all() + + # Copy data to the Postgres database + with engine.connect() as conn: + conn.execute(CreateSchema(expid, if_not_exists=True)) + target_table = tables.table_change_schema(expid, tables.GraphDataTable) + conn.execute(CreateTable(target_table, if_not_exists=True)) + if len(graph_rows) > 0: + conn.execute(insert(target_table),[row._mapping for row in graph_rows]) + conn.commit() + + +def copy_autosubmit_db(filepath: str, engine: Engine): + """ + This function copies the content of the FAKE_EXP_DIR/autosubmit.db to the Postgres database + """ + # Get SQLite source data + source_as_db = create_engine(f"sqlite:///{filepath}") + with source_as_db.connect() as source_conn: + exp_rows = source_conn.execute(select(tables.ExperimentTable)).all() + details_rows = source_conn.execute(select(tables.DetailsTable)).all() + + # Copy data to the Postgres database + with engine.connect() as conn: + conn.execute(CreateTable(tables.ExperimentTable.__table__, if_not_exists=True)) + conn.execute(insert(tables.ExperimentTable),[row._mapping for row in exp_rows]) + conn.execute(CreateTable(tables.DetailsTable.__table__, if_not_exists=True)) + conn.execute(insert(tables.DetailsTable),[row._mapping for row in details_rows]) + conn.commit() + + +def copy_as_times_db(filepath: str, engine: Engine): + """ + This function copies the content of the FAKE_EXP_DIR/as_times.db to the Postgres database + """ + # Get SQLite source data + source_as_db = create_engine(f"sqlite:///{filepath}") + with source_as_db.connect() as source_conn: + as_times_rows = source_conn.execute(select(tables.ExperimentStatusTable)).all() + + # Copy data to the Postgres database + with engine.connect() as conn: + conn.execute(CreateTable(tables.ExperimentStatusTable.__table__, if_not_exists=True)) + conn.execute(insert(tables.ExperimentStatusTable),[row._mapping for row in as_times_rows]) + conn.commit() + + +def copy_job_packages_db(filepath: str, engine: Engine): + """ + This function copies the content of the FAKE_EXP_DIR/pkl/job_packages to the Postgres database + """ + # Get the xxxx from job_packages_xxxx.db with regex + match = re.search(r"job_packages_(\w+)\.db", filepath) + expid = match.group(1) + + # Get SQLite source data + source_as_db = create_engine(f"sqlite:///{filepath}") + with source_as_db.connect() as source_conn: + job_packages_rows = source_conn.execute(select(tables.JobPackageTable)).all() + wrapper_job_packages_rows = source_conn.execute(select(tables.WrapperJobPackageTable)).all() + + # Copy data to the Postgres database + with engine.connect() as conn: + conn.execute(CreateSchema(expid, if_not_exists=True)) + # Job packages + target_table = tables.table_change_schema(expid, tables.JobPackageTable) + conn.execute(CreateTable(target_table, if_not_exists=True)) + if len(job_packages_rows) > 0: + conn.execute(insert(target_table),[row._mapping for row in job_packages_rows]) + # Wrapper job packages + target_table = tables.table_change_schema(expid, tables.WrapperJobPackageTable) + conn.execute(CreateTable(target_table, if_not_exists=True)) + if len(wrapper_job_packages_rows) > 0: + conn.execute(insert(target_table),[row._mapping for row in wrapper_job_packages_rows]) + conn.commit()