diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index b51dd91bfea63a00f918b65467043893758115cc..13155b32b0696314408f1395022d73356bec3817 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -4180,12 +4180,7 @@ class Autosubmit: tmp_backup_path = os.path.join(BasicConfig.JOBDATA_DIR, "job_data_{0}_tmp.sql".format(expid)) command = "sqlite3 {0} .dump > {1} ".format(database_path, tmp_backup_path) Log.debug("Backing up jobs_data...") - if os.path.exists(database_path) and os.path.getsize(database_path) > 0: # Check if database has any data. - subprocess.call(command, shell=True) - if os.path.exists(tmp_backup_path) and os.path.exists(backup_path): - if os.path.getsize(tmp_backup_path) >= os.path.getsize(backup_path): # Check if all data is still there. - command = "mv {0} {1} ".format(tmp_backup_path, backup_path) - subprocess.call(command, shell=True) + out = subprocess.call(command, shell=True) Log.debug("Jobs_data database backup completed.") except BaseException as e: Log.debug("Jobs_data database backup failed.") @@ -4212,15 +4207,16 @@ class Autosubmit: try: if os.path.exists(database_path): result = os.popen("mv {0} {1}".format(database_path, corrupted_db_path)).read() - time.sleep(10) + time.sleep(1) Log.info("Original database moved.") try: exp_history = ExperimentHistory(expid, jobdata_dir_path=BasicConfig.JOBDATA_DIR, historiclog_dir_path=BasicConfig.HISTORICAL_LOG_DIR) - exp_history.initialize_database() Log.info("Restoring from sql") result = os.popen(bash_command).read() - except Exception as e: + exp_history.initialize_database() + + except: Log.warning("It was not possible to restore the jobs_data.db file... , a new blank db will be created") result = os.popen("rm {0}".format(database_path)).read() diff --git a/autosubmit/history/experiment_history.py b/autosubmit/history/experiment_history.py index 10600b36cd4efdc61c717d1efc945488243ad108..dfd4450aba741333c371be5d236f3dfec70e372b 100644 --- a/autosubmit/history/experiment_history.py +++ b/autosubmit/history/experiment_history.py @@ -191,16 +191,24 @@ class ExperimentHistory: def process_status_changes(self, job_list=None, chunk_unit="NA", chunk_size=0, current_config="",create=False): """ Detect status differences between job_list and current job_data rows, and update. Creates a new run if necessary. """ try: - current_experiment_run_dc = self.manager.get_experiment_run_dc_with_max_id() - update_these_changes = self._get_built_list_of_changes(job_list) - should_create_new_run = self.should_we_create_a_new_run(job_list, len(update_these_changes), current_experiment_run_dc, chunk_unit, chunk_size,create) - if len(update_these_changes) > 0 and should_create_new_run is False: - self.manager.update_many_job_data_change_status(update_these_changes) - if should_create_new_run: - return self.create_new_experiment_run(chunk_unit, chunk_size, current_config, job_list) - return self.update_counts_on_experiment_run_dc(current_experiment_run_dc, job_list) + try: + current_experiment_run_dc = self.manager.get_experiment_run_dc_with_max_id() + except: + current_experiment_run_dc = 0 + try: + update_these_changes = self._get_built_list_of_changes(job_list) + except: + # May be a new experiment run, so we don't have any changes to update ( could happen due job_data issues from 3.14.0) + update_these_changes = [] + #("no runs") + should_create_new_run = self.should_we_create_a_new_run(job_list, len(update_these_changes), current_experiment_run_dc, chunk_unit, chunk_size,create) + if len(update_these_changes) > 0 and should_create_new_run == False: + self.manager.update_many_job_data_change_status(update_these_changes) + if should_create_new_run: + return self.create_new_experiment_run(chunk_unit, chunk_size, current_config, job_list) + return self.update_counts_on_experiment_run_dc(current_experiment_run_dc, job_list) except Exception as exp: - self._log.log(str(exp), traceback.format_exc()) + self._log.log(str(exp), traceback.format_exc()) def _get_built_list_of_changes(self, job_list): """ Return: List of (current timestamp, current datetime str, status, rowstatus, id in job_data). One tuple per change. """