diff --git a/autosubmit/database/db_jobdata.py b/autosubmit/database/db_jobdata.py index 56183771b39c63989a767c0f75f592c7d320f599..376e1ba67047f1c878ff09f301997b371515e341 100644 --- a/autosubmit/database/db_jobdata.py +++ b/autosubmit/database/db_jobdata.py @@ -417,8 +417,10 @@ class JobDataStructure(MainDataBase): platform TEXT NOT NULL, job_id INTEGER NOT NULL, extra_data TEXT NOT NULL, - UNIQUE(counter,job_name) - );''') + UNIQUE(counter,job_name), + INDEX + ); + CREATE INDEX IF NOT EXISTS ID_JOB_NAME ON job_data(job_name);''') if not os.path.exists(self.database_path): open(self.database_path, "w") self.conn = self.create_connection(self.database_path) diff --git a/autosubmit/database/db_structure.py b/autosubmit/database/db_structure.py index 78b91c20575df3d2eeca6fc24f03259670343fe2..e4a996b32300c47a466b758bb138106f6d1338c5 100644 --- a/autosubmit/database/db_structure.py +++ b/autosubmit/database/db_structure.py @@ -27,6 +27,7 @@ import traceback import sqlite3 import copy from datetime import datetime +from bscearth.utils.date import Log # from networkx import DiGraph #DB_FILE_AS_TIMES = "/esarchive/autosubmit/as_times.db" @@ -77,7 +78,8 @@ def get_structure(exp_id, structures_path): raise Exception("Structures folder not found " + str(structures_path)) except Exception as exp: - print(traceback.format_exc()) + Log.warning("Get structure error: {0}".format(str(exp))) + Log.debug(traceback.format_exc()) def create_connection(db_file): @@ -103,7 +105,7 @@ def create_table(conn, create_table_sql): c = conn.cursor() c.execute(create_table_sql) except Exception as e: - print(e) + Log.warning("Create table error {0}".format(str(e))) def _get_exp_structure(path): @@ -121,7 +123,8 @@ def _get_exp_structure(path): rows = cur.fetchall() return rows except Exception as exp: - print(traceback.format_exc()) + Log.warning("Get structure error {0}".format(str(exp))) + Log.debug(traceback.format_exc()) return dict() @@ -129,27 +132,30 @@ def save_structure(graph, exp_id, structures_path): """ Saves structure if path is valid """ - #pkl_path = os.path.join(exp_path, exp_id, "pkl") + conn = None + # Path to structures folder exists if os.path.exists(structures_path): + # Path to structure file db_structure_path = os.path.join( structures_path, "structure_" + exp_id + ".db") - # with open(db_structure_path, "w"): - conn = None + # Path to structure file exists -> delete if os.path.exists(db_structure_path): conn = create_connection(db_structure_path) _delete_table_content(conn) else: - open(db_structure_path, "w") + # Path to structure file does not edxist -> Initialize structure + get_structure(exp_id, structures_path) conn = create_connection(db_structure_path) if conn: + # Save structure for u, v in graph.edges(): # save _create_edge(conn, u, v) - #print("Created edge " + str(u) + str(v)) conn.commit() else: - # pkl folder not found - raise Exception("pkl folder not found " + str(structures_path)) + # Structure Folder not found + raise Exception( + "Structures folder not found {0}".format(structures_path)) def _create_edge(conn, u, v): @@ -162,7 +168,8 @@ def _create_edge(conn, u, v): cur.execute(sql, (u, v)) # return cur.lastrowid except sqlite3.Error as e: - print("Error on Insert : " + str(type(e).__name__)) + Log.debug(traceback.format_exc()) + Log.warning("Error on Insert : {0}".format(str(type(e).__name__))) def _delete_table_content(conn): @@ -175,5 +182,5 @@ def _delete_table_content(conn): cur.execute(sql) conn.commit() except sqlite3.Error as e: - print(traceback.format_exc()) - print("Error on Delete : " + str(type(e).__name__)) + Log.debug(traceback.format_exc()) + Log.warning("Error on Delete : {0}".format(str(type(e).__name__))) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index a03ced461d426a72037b013d5284e93280d9d7bf..64b41c32a06e49265cf3defde1cc0a9a16cb5c7d 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -1262,8 +1262,6 @@ class JobList: child.parents.remove(job) if structure_valid == False: # Structure does not exist or it is not be updated, attempt to create it. - # print("Current: ") - # print(current_structure) Log.info("Updating structure persistence...") self.graph = transitive_reduction(self.graph) if self.graph: @@ -1277,7 +1275,7 @@ class JobList: DbStructure.save_structure( self.graph, self.expid, self._config.STRUCTURES_DIR) except Exception as exp: - Log.warning(exp) + Log.debug(exp) pass for job in self._job_list: