diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index fccf961ea0da216f763662facedb7295884c9fda..8fa0db1e3741eda9cac2e63e2b24985d0325c1a5 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -1872,8 +1872,6 @@ class Autosubmit: as_conf.set_new_project(platform[0], platform[2]) if as_conf.get_migrate_host_to(platform[0]) is not None: as_conf.set_new_host(platform[0], as_conf.get_migrate_host_to(platform[0])) - - return False else: if not Autosubmit.archive(experiment_id,False,False): @@ -2420,7 +2418,6 @@ class Autosubmit: Log.set_file(os.path.join(BasicConfig.LOCAL_ROOT_DIR, "ASlogs", 'archive_{0}.log'.format(expid))) exp_folder = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid) - if clean: # Cleaning to reduce file size. version = get_autosubmit_version(expid) @@ -2468,36 +2465,42 @@ class Autosubmit: try: shutil.rmtree(exp_folder) except Exception as e: - Log.critical("Can not remove experiments folder: {0}".format(e)) - Autosubmit.unarchive(expid,compress) - return False + Log.warning("Can not fully remove experiments folder: {0}".format(e)) + if os.stat(exp_folder): + try: + tmp_folder = os.path.join(BasicConfig.LOCAL_ROOT_DIR, "tmp") + tmp_expid = os.path.join(tmp_folder,expid+"_to_delete") + os.rename(exp_folder,tmp_expid) + Log.warning("Experiment folder renamed to: {0}".format(exp_folder+"_to_delete ")) + except Exception as e: + Log.critical("Can not remove or rename experiments folder: {0}".format(e)) + Autosubmit.unarchive(expid,compress,True) + return False Log.result("Experiment archived successfully") return True @staticmethod - def unarchive(experiment_id,compress=True): + def unarchive(experiment_id, compress=True, overwrite=False): """ Unarchives an experiment: uncompress folder from tar.gz and moves to experiments root folder :param experiment_id: experiment identifier :type experiment_id: str + :type compress: boolean + :type overwrite: boolean """ BasicConfig.read() Log.set_file(os.path.join(BasicConfig.LOCAL_ROOT_DIR, "ASlogs", 'unarchive_{0}.log'.format(experiment_id))) exp_folder = os.path.join(BasicConfig.LOCAL_ROOT_DIR, experiment_id) - if os.path.exists(exp_folder): - Log.error("Experiment {0} is not archived", experiment_id) - return False # Searching by year. We will store it on database year = datetime.datetime.today().year archive_path = None if compress: - compress_type="r:gz" + compress_type = "r:gz" output_pathfile = '{0}.tar.gz'.format(experiment_id) - else: compress_type="r:" output_pathfile='{0}.tar'.format(experiment_id) @@ -2508,19 +2511,20 @@ class Autosubmit: year -= 1 if year == 2000: - Log.critical("Experiment can not be located on archive") + Log.error("Experiment {0} is not archived", experiment_id) return False Log.info("Experiment located in {0} archive", year) # Creating tar file Log.info("Unpacking tar file ... ") try: - os.mkdir(exp_folder) + if not os.stat(exp_folder): + os.mkdir(exp_folder) with tarfile.open(os.path.join(archive_path), compress_type) as tar: tar.extractall(exp_folder) tar.close() except Exception as e: - os.rmdir(exp_folder) + shutil.rmtree(exp_folder,ignore_errors=True) Log.critical("Can not extract tar file: {0}".format(e)) return False diff --git a/autosubmit/job/job_packager.py b/autosubmit/job/job_packager.py index 5a7603150e1c0ffa997905111e944cecf050ed38..206be38cca6d2de39904961b3cfac45d033eaabb 100644 --- a/autosubmit/job/job_packager.py +++ b/autosubmit/job/job_packager.py @@ -212,18 +212,18 @@ class JobPackager(object): self.max_jobs -= len(jobs_list) if job.status is Status.READY: packages.append(JobPackageVertical(jobs_list)) - else: - package = JobPackageVertical(jobs_list, potential_dependency) - packages.append(package) - # Possible need of "if self.remote_dependencies here" - remote_dependencies_dict['name_to_id'][potential_dependency] = -1 - remote_dependencies_dict['dependencies'][package.name] = potential_dependency - if self.remote_dependencies: - # Sending last item in list of packaged - child = job_vertical_packager.get_wrappable_child(jobs_list[-1]) - if child is not None: - section_list.insert(section_list.index(job) + 1, child) - potential_dependency = packages[-1].name + else: + if self.remote_dependencies: + # Sending last item in list of packaged + child = job_vertical_packager.get_wrappable_child(jobs_list[-1]) + if child is not None: + section_list.insert(section_list.index(job) + 1, child) + potential_dependency = packages[-1].name + package = JobPackageVertical(jobs_list, potential_dependency) + packages.append(package) + # Possible need of "if self.remote_dependencies here" + remote_dependencies_dict['name_to_id'][potential_dependency] = -1 + remote_dependencies_dict['dependencies'][package.name] = potential_dependency else: break return packages, remote_dependencies_dict diff --git a/autosubmit/platforms/paramiko_submitter.py b/autosubmit/platforms/paramiko_submitter.py index 46196dbc67f9ec49a36e20d4b9bc76888026eda8..494fcee08fd5669080f563478a8455e36f8779bd 100644 --- a/autosubmit/platforms/paramiko_submitter.py +++ b/autosubmit/platforms/paramiko_submitter.py @@ -42,6 +42,9 @@ class ParamikoSubmitter(Submitter): Class to manage the experiments platform """ + def load_platforms_migrate(self, asconf, retries=5): + pass #Add all info related to migrate + def load_platforms(self, asconf, retries=5): """ Create all the platforms object that will be used by the experiment