From 5d6603a5ff5b86217b89fc727a592f2acb42f35c Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 22 Mar 2023 15:41:08 +0100 Subject: [PATCH 1/6] migrate rework --- autosubmit/autosubmit.py | 620 ++++++++++++++++++--------------------- 1 file changed, 291 insertions(+), 329 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 8122146d5..21e80864c 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -2795,205 +2795,144 @@ class Autosubmit: 7000, str(e)) return True - @staticmethod - def migrate(experiment_id, offer, pickup, only_remote): + def get_platforms_grouped_by_dir(platforms): """ - Migrates experiment files from current to other user. - It takes mapping information for new user from config files. - - :param experiment_id: experiment identifier: - :param pickup: - :param offer: - :param only_remote: + Groups the platforms by the directory where the data is stored + :param platforms: list of platforms + :return: dictionary with the directory as key and a list of platforms as value """ - - if offer: - as_conf = AutosubmitConfig( - experiment_id, BasicConfig, YAMLParserFactory()) - as_conf.check_conf_files(True) - pkl_dir = os.path.join( - BasicConfig.LOCAL_ROOT_DIR, experiment_id, 'pkl') - job_list = Autosubmit.load_job_list( - experiment_id, as_conf, notransitive=True, monitor=True) - Log.debug("Job list restored from {0} files", pkl_dir) - error = False - platforms_to_test = set() - submitter = Autosubmit._get_submitter(as_conf) - submitter.load_platforms(as_conf) - if submitter.platforms is None: - raise AutosubmitCritical("No platforms configured!!!", 7014) - platforms = submitter.platforms - for job in job_list.get_job_list(): - job.submitter = submitter - if job.platform_name is None: - job.platform_name = as_conf.get_platform() - platforms_to_test.add(platforms[job.platform_name]) - # establish the connection to all platforms on use - Autosubmit.restore_platforms(platforms_to_test) - Log.info('Migrating experiment {0}'.format(experiment_id)) - Autosubmit._check_ownership(experiment_id, raise_error=True) - if submitter.platforms is None: - return False - Log.info("Checking remote platforms") - platforms = [x for x in submitter.platforms if x not in [ - 'local', 'LOCAL']] - already_moved = set() - backup_files = [] - backup_conf = [] - error = False - err_message = 'Invalid Configuration:' - for platform in platforms: - # Checks - Log.info( - "Checking [{0}] from platforms configuration...", platform) - if as_conf.get_migrate_user_to(platform) == '': - err_message += "\nInvalid USER_TO target [ USER == USER_TO in [{0}] ]".format( - platform) - error = True - elif not as_conf.get_migrate_duplicate(platform) and as_conf.get_migrate_user_to( - platform) == as_conf.get_current_user(platform): - err_message += "\nInvalid USER_TO target [ USER == USER_TO in ({0}) ] while parameter SAME_USER is false (or unset)".format( - platform) - error = True - p = submitter.platforms[platform] - if p.temp_dir is None: - err_message += "\nInvalid TEMP_DIR, Parameter must be present even if empty in [{0}]".format( - platform) - error = True - elif p.temp_dir != "": - if not p.check_tmp_exists(): - err_message += "\nTEMP_DIR {0}, does not exists in [{1}]".format( - p.temp_dir, platform) - error = True - if error: - raise AutosubmitCritical(err_message, 7014) - for platform in platforms: - if as_conf.get_migrate_project_to(platform) != '': - Log.info("Project in platform configuration file successfully updated to {0}", - as_conf.get_current_project(platform)) - as_conf.get_current_project(platform) - backup_conf.append([platform, as_conf.get_current_user( - platform), as_conf.get_current_project(platform)]) - as_conf.set_new_user( - platform, as_conf.get_migrate_user_to(platform)) - - as_conf.set_new_project( - platform, as_conf.get_migrate_project_to(platform)) - as_conf.get_current_project(platform) - as_conf.get_current_user(platform) - else: - Log.result( - "[OPTIONAL] PROJECT_TO directive not found. The directive PROJECT will remain unchanged") - backup_conf.append( - [platform, as_conf.get_current_user(platform), None]) - as_conf.set_new_user( - platform, as_conf.get_migrate_user_to(platform)) - as_conf.get_current_project(platform) - as_conf.get_current_user(platform) - - if as_conf.get_migrate_host_to(platform) != "none" and len(as_conf.get_migrate_host_to(platform)) > 0: - Log.result( - "Host in platform configuration file successfully updated to {0}", - as_conf.get_migrate_host_to(platform)) - as_conf.set_new_host( - platform, as_conf.get_migrate_host_to(platform)) + platforms_by_dir = defaultdict([]) + for platform in platforms: + platforms_by_dir[platform.root_dir].append(platform) + return platforms_by_dir + @staticmethod + def check_migrate_config(as_conf,platforms): + """ + Checks if the configuration file has the necessary information to migrate the data + :param as_conf: Autosubmit configuration file + :param platforms: list of platforms + :return: platforms to migrate + """ + error = False + platforms_to_migrate = list() + platforms_with_missconfiguration = list() + platforms_by_dir = Autosubmit.get_platforms_grouped_by_dir(platforms) + for platform_dir in platforms_by_dir: + platform_dir_error = True + for platform in platform_dir: + Log.info("Checking [{0}] from platforms configuration...", platform) + if as_conf.platforms[platform.name].get("USER",None) is None or \ + as_conf.platforms[platform.name].get("USER_TO",as_conf.platforms[platform.name].get("USER","")) == as_conf.platforms[platform.name].get("USER","") and not as_conf.platforms[platform.name].get("SAME_USER",False) or \ + as_conf.platforms[platform.name].get("PROJECT",None) is None or\ + as_conf.platforms[platform.name].get("PROJECT_TO",None) is None or \ + as_conf.platforms[platform].get("TEMP_DIR", "") == "": + Log.debug(f"Values: USER: {as_conf.platforms[platform.name].get('USER',None)}" + f" USER_TO: {as_conf.platforms[platform.name].get('USER_TO',None)}" + f" PROJECT: {as_conf.platforms[platform.name].get('PROJECT',None)}" + f" PROJECT_TO: {as_conf.platforms[platform.name].get('PROJECT_TO',None)}" + f" TEMP_DIR: {as_conf.platforms[platform].get('TEMP_DIR', '')}") + Log.debug(f"Invalid configuration for platform [{platform.name}]\nTrying next platform...") else: - Log.result( - "[OPTIONAL] HOST_TO directive not found. The directive HOST will remain unchanged") - p = submitter.platforms[platform] - if p.temp_dir not in already_moved: - if p.root_dir != p.temp_dir and len(p.temp_dir) > 0: - already_moved.add(p.temp_dir) - # find /home/bsc32/bsc32070/dummy3 -type l -lname '/*' -printf ' ln -sf "$(realpath -s --relative-to="%p" $(readlink "%p")")" \n' > script.sh - # command = "find " + p.root_dir + " -type l -lname \'/*\' -printf 'var=\"$(realpath -s --relative-to=\"%p\" \"$(readlink \"%p\")\")\" && var=${var:3} && ln -sf $var \"%p\" \\n'" - Log.info( - "Converting the absolute symlinks into relatives on platform {0} ", platform) - command = "find " + p.root_dir + \ - " -type l -lname \'/*\' -printf 'var=\"$(realpath -s --relative-to=\"%p\" \"$(readlink \"%p\")\")\" && var=${var:3} && ln -sf $var \"%p\" \\n' " - try: - p.send_command(command, True) - if p.get_ssh_output().startswith("var="): - convertLinkPath = os.path.join( - BasicConfig.LOCAL_ROOT_DIR, experiment_id, BasicConfig.LOCAL_TMP_DIR, - 'convertLink.sh') - with open(convertLinkPath, 'w') as convertLinkFile: - convertLinkFile.write(p.get_ssh_output()) - p.send_file("convertLink.sh") - convertLinkPathRemote = os.path.join( - p.remote_log_dir, "convertLink.sh") - command = "chmod +x " + convertLinkPathRemote + " && " + \ - convertLinkPathRemote + " && rm " + convertLinkPathRemote - p.send_command(command, True) - else: - Log.result("No links found in {0} for [{1}] ".format( - p.root_dir, platform)) - - except IOError: - Log.debug( - "The platform {0} does not contain absolute symlinks", platform) - except BaseException: - Log.printlog( - "Absolute symlinks failed to convert, check user in platform.yml", 3000) - error = True - break - try: - Log.info( - "Moving remote files/dirs on {0}", platform) - p.send_command("chmod 777 -R " + p.root_dir) - if not p.move_file(p.root_dir, os.path.join(p.temp_dir, experiment_id), False): - Log.result("No data found in {0} for [{1}]\n".format( - p.root_dir, platform)) - except IOError as e: - Log.printlog("The files/dirs on {0} cannot be moved to {1}.".format(p.root_dir, - os.path.join(p.temp_dir, - experiment_id), - 6012)) - error = True - break - except Exception as e: - Log.printlog("Trace: {2}\nThe files/dirs on {0} cannot be moved to {1}.".format( - p.root_dir, os.path.join(p.temp_dir, experiment_id), str(e)), 6012) - error = True - break - backup_files.append(platform) - Log.result( - "Files/dirs on {0} have been successfully offered", platform) - if error: - as_conf = AutosubmitConfig( - experiment_id, BasicConfig, YAMLParserFactory()) - as_conf.check_conf_files(False) - for platform in backup_files: - p = submitter.platforms[platform] - p.move_file(os.path.join( - p.temp_dir, experiment_id), p.root_dir, True) - for platform in backup_conf: - as_conf.set_new_user(platform[0], platform[1]) - if platform[2] is not None and len(str(platform[2])) > 0: - as_conf.set_new_project(platform[0], platform[2]) - if as_conf.get_migrate_host_to(platform[0]) != "none" and len( - as_conf.get_migrate_host_to(platform[0])) > 0: - as_conf.set_new_host( - platform[0], as_conf.get_migrate_host_to(platform[0])) - raise AutosubmitCritical( - "The experiment cannot be offered, changes are reverted", 7014) - else: + Log.info("Valid configuration for platform [{0}]".format(platform.name)) + Log.result(f"Using platform: [{platform.name}] to migrate [{platform.root_dir}] data") + platform_dir_error = False + platforms_to_migrate.append(platform) + break + if platform_dir_error: + error = True + platform_names = [p.name+", " for p in platforms_by_dir[platform_dir]] + platform_names = platform_names[:-2] + platforms_with_missconfiguration += f"{platform_dir}: {platform_names}\n" + if error: + raise AutosubmitCritical(f"Invalid migrate configuration for platforms: {platforms_with_missconfiguration}", 7014) + else: + return platforms_to_migrate + @staticmethod + def migrate_offer(experiment_id,only_remote): + as_conf = AutosubmitConfig( + experiment_id, BasicConfig, YAMLParserFactory()) + as_conf.check_conf_files(True) + pkl_dir = os.path.join(BasicConfig.LOCAL_ROOT_DIR, experiment_id, 'pkl') + job_list = Autosubmit.load_job_list(experiment_id, as_conf, notransitive=True, monitor=True) + Log.debug("Job list restored from {0} files", pkl_dir) + platforms_to_test = set() + submitter = Autosubmit._get_submitter(as_conf) + submitter.load_platforms(as_conf) + if submitter.platforms is None: + raise AutosubmitCritical("No platforms configured!!!", 7014) + platforms = submitter.platforms + for job in job_list.get_job_list(): + job.submitter = submitter + if job.platform_name is None: + job.platform_name = as_conf.get_platform() + platforms_to_test.add(platforms[job.platform_name]) + # establish the connection to all platforms on use + Autosubmit.restore_platforms(platforms_to_test) + Log.info('Migrating experiment {0}'.format(experiment_id)) + Autosubmit._check_ownership(experiment_id, raise_error=True) + if submitter.platforms is None: + return False + Log.info("Checking remote platforms") + platforms = [x for x in submitter.platforms if x not in [ + 'local', 'LOCAL']] + backup_files = [] + backup_conf = [] + # Checks and annotates the platforms to migrate ( one per directory if they share it ) + platforms_to_migrate = Autosubmit.check_migrate_config(as_conf, platforms) + platforms_without_issues = list() + for platform in platforms_to_migrate: + p = submitter.platforms[platform] + if p.root_dir != p.temp_dir and len(p.temp_dir) > 0: + # find /home/bsc32/bsc32070/dummy3 -type l -lname '/*' -printf ' ln -sf "$(realpath -s --relative-to="%p" $(readlink "%p")")" \n' > script.sh + # command = "find " + p.root_dir + " -type l -lname \'/*\' -printf 'var=\"$(realpath -s --relative-to=\"%p\" \"$(readlink \"%p\")\")\" && var=${var:3} && ln -sf $var \"%p\" \\n'" + Log.info(f"Converting the absolute symlinks into relatives on platform [{platform.name}] ") + command = f"find {p.root_dir} -type l -lname \'/*\' -printf 'var=\"$(realpath -s --relative-to=\"%p\" \"$(readlink \"%p\")\")\" && var=${{var:3}} && ln -sf $var \"%p\" \\n' " try: - if not only_remote: - if not Autosubmit.archive(experiment_id, True, True): - for platform in backup_files: - p = submitter.platforms[platform] - p.move_file(os.path.join( - p.temp_dir, experiment_id), p.root_dir, True) - for platform in backup_conf: - as_conf.set_new_user(platform[0], platform[1]) - if platform[2] is not None and len(str(platform[2])) > 0: - as_conf.set_new_project( - platform[0], platform[2]) - raise AutosubmitCritical( - "The experiment cannot be offered, changes are reverted", 7014) - Log.result("The experiment has been successfully offered.") - except Exception as e: + p.send_command(command, True) + if p.get_ssh_output().startswith("var="): + convertLinkPath = os.path.join( + BasicConfig.LOCAL_ROOT_DIR, experiment_id, BasicConfig.LOCAL_TMP_DIR, + 'convertLink.sh') + with open(convertLinkPath, 'w') as convertLinkFile: + convertLinkFile.write(p.get_ssh_output()) + p.send_file("convertLink.sh") + convertLinkPathRemote = os.path.join( + p.remote_log_dir, "convertLink.sh") + command = "chmod +x " + convertLinkPathRemote + " && " + \ + convertLinkPathRemote + " && rm " + convertLinkPathRemote + p.send_command(command, True) + Log.result(f"Absolute symlinks converted on platform [{platform.name}]") + else: + Log.result(f"No absolute symlinks found in [{p.root_dir}] for platform [{platform.name}]") + except IOError: + Log.result(f"No absolute symlinks found in [{p.root_dir}] for platform [{platform.name}]") + except BaseException as e: + Log.printlog(f"Absolute symlinks failed to convert due to [{str(e)}] on platform [{platform.name}]", + 7014) + break + # If there are no errors in the conversion of the absolute symlinks, then move the files of this platform + try: + Log.info(f"Moving remote files/dirs on platform [{platform.name}] to [{p.temp_dir}]") + p.send_command(f"chmod 777 -R {p.root_dir}") + if not p.move_file(p.root_dir, os.path.join(p.temp_dir, experiment_id), False): + Log.result(f"No data found in [{p.root_dir}] for platform [{platform.name}]") + else: + Log.result( + f"Remote files/dirs on platform [{platform.name}] have been successfully moved to [{p.temp_dir}]") + except BaseException as e: + Log.printlog( + f"Cant move files/dirs on platform [{platform.name}] to [{p.temp_dir}] due to [{str(e)}]", + 6000) + break + platforms_without_issues.append(platform) + Log.result(f"Platform [{platform.name}] has been successfully migrated") + + # At this point, all remote platforms has been migrated. + + try: + if not only_remote: + if not Autosubmit.archive(experiment_id, True, True): for platform in backup_files: p = submitter.platforms[platform] p.move_file(os.path.join( @@ -3001,151 +2940,174 @@ class Autosubmit: for platform in backup_conf: as_conf.set_new_user(platform[0], platform[1]) if platform[2] is not None and len(str(platform[2])) > 0: - as_conf.set_new_project(platform[0], platform[2]) - raise AutosubmitCritical( - "The experiment cannot be offered, changes are reverted", 7014, str(e)) - elif pickup: - Log.info('Migrating experiment {0}'.format(experiment_id)) - Log.info("Moving local files/dirs") - if not only_remote: - if not Autosubmit.unarchive(experiment_id, True): - raise AutosubmitCritical( - "The experiment cannot be picked up", 7012) - Log.info("Local files/dirs have been successfully picked up") - else: - exp_path = os.path.join( - BasicConfig.LOCAL_ROOT_DIR, experiment_id) - if not os.path.exists(exp_path): + as_conf.set_new_project( + platform[0], platform[2]) raise AutosubmitCritical( - "Experiment seems to be archived, no action is performed", 7012) - - as_conf = AutosubmitConfig( - experiment_id, BasicConfig, YAMLParserFactory()) - as_conf.check_conf_files(False) - pkl_dir = os.path.join( - BasicConfig.LOCAL_ROOT_DIR, experiment_id, 'pkl') - job_list = Autosubmit.load_job_list( - experiment_id, as_conf, notransitive=True, monitor=True) - Log.debug("Job list restored from {0} files", pkl_dir) - error = False - platforms_to_test = set() - submitter = Autosubmit._get_submitter(as_conf) - submitter.load_platforms(as_conf) - if submitter.platforms is None: - raise AutosubmitCritical("No platforms configured!!!", 7014) - platforms = submitter.platforms - for job in job_list.get_job_list(): - job.submitter = submitter - if job.platform_name is None: - job.platform_name = as_conf.get_platform() - platforms_to_test.add(platforms[job.platform_name]) - - Log.info("Checking remote platforms") - platforms = [x for x in submitter.platforms if x not in [ - 'local', 'LOCAL']] - already_moved = set() - backup_files = [] - # establish the connection to all platforms on use - try: - Autosubmit.restore_platforms(platforms_to_test) - except AutosubmitCritical as e: + "The experiment cannot be offered, changes are reverted", 7014) + Log.result("The experiment has been successfully offered.") + except Exception as e: + # todo put the IO error code + raise AutosubmitCritical(f"[LOCAL] Error offering the experiment: {str(e)}\n" + f"Please, try again", 7000) + @staticmethod + def migrate_pickup(experiment_id, only_remote): + Log.info('Migrating experiment {0}'.format(experiment_id)) + Log.info("Moving local files/dirs") + if not only_remote: + if not Autosubmit.unarchive(experiment_id, True): raise AutosubmitCritical( - e.message + "\nInvalid Remote Platform configuration, recover them manually or:\n 1) Configure platform.yml with the correct info\n 2) autosubmit expid -p --onlyremote", - 7014, e.trace) - except Exception as e: + "The experiment cannot be picked up", 7012) + Log.info("Local files/dirs have been successfully picked up") + else: + exp_path = os.path.join( + BasicConfig.LOCAL_ROOT_DIR, experiment_id) + if not os.path.exists(exp_path): raise AutosubmitCritical( - "Invalid Remote Platform configuration, recover them manually or:\n 1) Configure platform.yml with the correct info\n 2) autosubmit expid -p --onlyremote", - 7014, str(e)) - for platform in platforms: - p = submitter.platforms[platform] - if p.temp_dir is not None and p.temp_dir not in already_moved: - if p.root_dir != p.temp_dir and len(p.temp_dir) > 0: - already_moved.add(p.temp_dir) - Log.info( - "Copying remote files/dirs on {0}", platform) - Log.info("Copying from {0} to {1}", os.path.join( - p.temp_dir, experiment_id), p.root_dir) - finished = False - limit = 150 - rsync_retries = 0 - try: - # Avoid infinite loop unrealistic upper limit, only for rsync failure - while not finished and rsync_retries < limit: - finished = False - pipeline_broke = False - Log.info( - "Rsync launched {0} times. Can take up to 150 retrials or until all data is transferred".format( - rsync_retries + 1)) + "Experiment seems to be archived, no action is performed", 7012) + + as_conf = AutosubmitConfig( + experiment_id, BasicConfig, YAMLParserFactory()) + as_conf.check_conf_files(False) + pkl_dir = os.path.join( + BasicConfig.LOCAL_ROOT_DIR, experiment_id, 'pkl') + job_list = Autosubmit.load_job_list( + experiment_id, as_conf, notransitive=True, monitor=True) + Log.debug("Job list restored from {0} files", pkl_dir) + error = False + platforms_to_test = set() + submitter = Autosubmit._get_submitter(as_conf) + submitter.load_platforms(as_conf) + if submitter.platforms is None: + raise AutosubmitCritical("No platforms configured!!!", 7014) + platforms = submitter.platforms + for job in job_list.get_job_list(): + job.submitter = submitter + if job.platform_name is None: + job.platform_name = as_conf.get_platform() + platforms_to_test.add(platforms[job.platform_name]) + + Log.info("Checking remote platforms") + platforms = [x for x in submitter.platforms if x not in [ + 'local', 'LOCAL']] + already_moved = set() + backup_files = [] + # establish the connection to all platforms on use + try: + Autosubmit.restore_platforms(platforms_to_test) + except AutosubmitCritical as e: + raise AutosubmitCritical( + e.message + "\nInvalid Remote Platform configuration, recover them manually or:\n 1) Configure platform.yml with the correct info\n 2) autosubmit expid -p --onlyremote", + 7014, e.trace) + except Exception as e: + raise AutosubmitCritical( + "Invalid Remote Platform configuration, recover them manually or:\n 1) Configure platform.yml with the correct info\n 2) autosubmit expid -p --onlyremote", + 7014, str(e)) + for platform in platforms: + p = submitter.platforms[platform] + if p.temp_dir is not None and p.temp_dir not in already_moved: + if p.root_dir != p.temp_dir and len(p.temp_dir) > 0: + already_moved.add(p.temp_dir) + Log.info( + "Copying remote files/dirs on {0}", platform) + Log.info("Copying from {0} to {1}", os.path.join( + p.temp_dir, experiment_id), p.root_dir) + finished = False + limit = 150 + rsync_retries = 0 + try: + # Avoid infinite loop unrealistic upper limit, only for rsync failure + while not finished and rsync_retries < limit: + finished = False + pipeline_broke = False + Log.info( + "Rsync launched {0} times. Can take up to 150 retrials or until all data is transferred".format( + rsync_retries + 1)) + try: + p.send_command( + "rsync --timeout=3600 --bwlimit=20000 -aq --remove-source-files " + os.path.join( + p.temp_dir, experiment_id) + " " + p.root_dir[:-5]) + except BaseException as e: + Log.debug("{0}".format(str(e))) + rsync_retries += 1 try: - p.send_command( - "rsync --timeout=3600 --bwlimit=20000 -aq --remove-source-files " + os.path.join( - p.temp_dir, experiment_id) + " " + p.root_dir[:-5]) - except BaseException as e: - Log.debug("{0}".format(str(e))) - rsync_retries += 1 - try: - if p.get_ssh_output_err() == "": - finished = True - elif p.get_ssh_output_err().lower().find("no such file or directory") == -1: - finished = True - else: - finished = False - except Exception as e: - finished = False - pipeline_broke = True - if not pipeline_broke: - if p.get_ssh_output_err().lower().find("no such file or directory") == -1: + if p.get_ssh_output_err() == "": finished = True - elif p.get_ssh_output_err().lower().find( - "warning: rsync") != -1 or p.get_ssh_output_err().lower().find( - "closed") != -1 or p.get_ssh_output_err().lower().find( - "broken pipe") != -1 or p.get_ssh_output_err().lower().find( - "directory has vanished") != -1: - rsync_retries += 1 - finished = False - elif p.get_ssh_output_err() == "": + elif p.get_ssh_output_err().lower().find("no such file or directory") == -1: finished = True else: - error = True finished = False - break - p.send_command( - "find {0} -depth -type d -empty -delete".format( - os.path.join(p.temp_dir, experiment_id))) - Log.result( - "Empty dirs on {0} have been successfully deleted".format(p.temp_dir)) - if finished: - p.send_command("chmod 755 -R " + p.root_dir) - Log.result( - "Files/dirs on {0} have been successfully picked up", platform) - # p.send_command( - # "find {0} -depth -type d -empty -delete".format(os.path.join(p.temp_dir, experiment_id))) - Log.result( - "Empty dirs on {0} have been successfully deleted".format(p.temp_dir)) - else: - Log.printlog("The files/dirs on {0} cannot be copied to {1}.".format( - os.path.join(p.temp_dir, experiment_id), p.root_dir), 6012) - error = True - break - - except IOError as e: - raise AutosubmitError( - "I/O Issues", 6016, e.message) - except BaseException as e: + except Exception as e: + finished = False + pipeline_broke = True + if not pipeline_broke: + if p.get_ssh_output_err().lower().find("no such file or directory") == -1: + finished = True + elif p.get_ssh_output_err().lower().find( + "warning: rsync") != -1 or p.get_ssh_output_err().lower().find( + "closed") != -1 or p.get_ssh_output_err().lower().find( + "broken pipe") != -1 or p.get_ssh_output_err().lower().find( + "directory has vanished") != -1: + rsync_retries += 1 + finished = False + elif p.get_ssh_output_err() == "": + finished = True + else: + error = True + finished = False + break + p.send_command( + "find {0} -depth -type d -empty -delete".format( + os.path.join(p.temp_dir, experiment_id))) + Log.result( + "Empty dirs on {0} have been successfully deleted".format(p.temp_dir)) + if finished: + p.send_command("chmod 755 -R " + p.root_dir) + Log.result( + "Files/dirs on {0} have been successfully picked up", platform) + # p.send_command( + # "find {0} -depth -type d -empty -delete".format(os.path.join(p.temp_dir, experiment_id))) + Log.result( + "Empty dirs on {0} have been successfully deleted".format(p.temp_dir)) + else: + Log.printlog("The files/dirs on {0} cannot be copied to {1}.".format( + os.path.join(p.temp_dir, experiment_id), p.root_dir), 6012) error = True - Log.printlog("The files/dirs on {0} cannot be copied to {1}.\nTRACE:{2}".format( - os.path.join(p.temp_dir, experiment_id), p.root_dir, str(e)), 6012) break - else: - Log.result( - "Files/dirs on {0} have been successfully picked up", platform) - if error: - raise AutosubmitCritical( - "Unable to pickup all platforms, the non-moved files are on the TEMP_DIR\n You can try again with autosubmit {0} -p --onlyremote".format( - experiment_id), 7012) - else: - Log.result("The experiment has been successfully picked up.") - return True + + except IOError as e: + raise AutosubmitError( + "I/O Issues", 6016, e.message) + except BaseException as e: + error = True + Log.printlog("The files/dirs on {0} cannot be copied to {1}.\nTRACE:{2}".format( + os.path.join(p.temp_dir, experiment_id), p.root_dir, str(e)), 6012) + break + else: + Log.result( + "Files/dirs on {0} have been successfully picked up", platform) + if error: + raise AutosubmitCritical( + "Unable to pickup all platforms, the non-moved files are on the TEMP_DIR\n You can try again with autosubmit {0} -p --onlyremote".format( + experiment_id), 7012) + else: + Log.result("The experiment has been successfully picked up.") + return True + @staticmethod + def migrate(experiment_id, offer, pickup, only_remote): + """ + Migrates experiment files from current to other user. + It takes mapping information for new user from config files. + + :param experiment_id: experiment identifier: + :param pickup: + :param offer: + :param only_remote: + """ + + if offer: + Autosubmit.migrate_offer(experiment_id, only_remote) + elif pickup: + Autosubmit.migrate_pickup(experiment_id, only_remote) @staticmethod def check(experiment_id, notransitive=False): -- GitLab From ef9744ca48e235b9ca4c8a85a5b247855f715aa9 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 22 Mar 2023 15:41:44 +0100 Subject: [PATCH 2/6] migrate rework --- autosubmit/autosubmit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 21e80864c..b6a2c36e3 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -2929,7 +2929,7 @@ class Autosubmit: Log.result(f"Platform [{platform.name}] has been successfully migrated") # At this point, all remote platforms has been migrated. - + # TODO set user_to and project_to to the correct values try: if not only_remote: if not Autosubmit.archive(experiment_id, True, True): -- GitLab From a7cd6fe5036718a9cdc8f8303c5f1c479f81c69d Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 22 Mar 2023 16:23:30 +0100 Subject: [PATCH 3/6] new autoconfigparser version to solve a bug --- requeriments.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requeriments.txt b/requeriments.txt index f612dafd5..b97597236 100644 --- a/requeriments.txt +++ b/requeriments.txt @@ -1,4 +1,4 @@ -autosubmitconfigparser==1.0.22 +autosubmitconfigparser==1.0.23 paramiko>=2.9.2 bcrypt>=3.2 PyNaCl>=1.5.0 diff --git a/setup.py b/setup.py index aada781fa..c37bd9d2e 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ setup( url='http://www.bsc.es/projects/earthscience/autosubmit/', download_url='https://earth.bsc.es/wiki/doku.php?id=tools:autosubmit', keywords=['climate', 'weather', 'workflow', 'HPC'], - install_requires=['autosubmitconfigparser==1.0.22','bcrypt>=3.2','packaging>19','six>=1.10.0','configobj>=5.0.6','argparse>=1.4.0','python-dateutil>=2.8.2','matplotlib<3.6','numpy<1.22','py3dotplus>=1.1.0','pyparsing>=3.0.7','paramiko>=2.9.2','mock>=4.0.3','portalocker>=2.3.2','networkx==2.6.3','requests>=2.27.1','bscearth.utils>=0.5.2','cryptography>=36.0.1','setuptools>=60.8.2','xlib>=0.21','pip>=22.0.3','ruamel.yaml','pythondialog','pytest','nose','coverage','PyNaCl>=1.4.0','Pygments'], + install_requires=['autosubmitconfigparser==1.0.23','bcrypt>=3.2','packaging>19','six>=1.10.0','configobj>=5.0.6','argparse>=1.4.0','python-dateutil>=2.8.2','matplotlib<3.6','numpy<1.22','py3dotplus>=1.1.0','pyparsing>=3.0.7','paramiko>=2.9.2','mock>=4.0.3','portalocker>=2.3.2','networkx==2.6.3','requests>=2.27.1','bscearth.utils>=0.5.2','cryptography>=36.0.1','setuptools>=60.8.2','xlib>=0.21','pip>=22.0.3','ruamel.yaml','pythondialog','pytest','nose','coverage','PyNaCl>=1.4.0','Pygments'], classifiers=[ "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.9", -- GitLab From 3f875e3b596931973a6707155cba681e1f5163d8 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 23 Mar 2023 11:45:52 +0100 Subject: [PATCH 4/6] migrate --- autosubmit/autosubmit.py | 40 ++++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index b6a2c36e3..ff88c8b6c 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -2848,8 +2848,19 @@ class Autosubmit: raise AutosubmitCritical(f"Invalid migrate configuration for platforms: {platforms_with_missconfiguration}", 7014) else: return platforms_to_migrate + @staticmethod - def migrate_offer(experiment_id,only_remote): + def migrate_offer_local(experiment_id): + try: + if not Autosubmit.archive(experiment_id, True, True): + raise AutosubmitCritical(f"Error archiving the experiment", 7014) + Log.result("The experiment has been successfully offered.") + except Exception as e: + # todo put the IO error code + raise AutosubmitCritical(f"[LOCAL] Error offering the experiment: {str(e)}\n" + f"Please, try again", 7000) + @staticmethod + def migrate_offer_remote(): as_conf = AutosubmitConfig( experiment_id, BasicConfig, YAMLParserFactory()) as_conf.check_conf_files(True) @@ -2876,8 +2887,6 @@ class Autosubmit: Log.info("Checking remote platforms") platforms = [x for x in submitter.platforms if x not in [ 'local', 'LOCAL']] - backup_files = [] - backup_conf = [] # Checks and annotates the platforms to migrate ( one per directory if they share it ) platforms_to_migrate = Autosubmit.check_migrate_config(as_conf, platforms) platforms_without_issues = list() @@ -2930,25 +2939,12 @@ class Autosubmit: # At this point, all remote platforms has been migrated. # TODO set user_to and project_to to the correct values - try: - if not only_remote: - if not Autosubmit.archive(experiment_id, True, True): - for platform in backup_files: - p = submitter.platforms[platform] - p.move_file(os.path.join( - p.temp_dir, experiment_id), p.root_dir, True) - for platform in backup_conf: - as_conf.set_new_user(platform[0], platform[1]) - if platform[2] is not None and len(str(platform[2])) > 0: - as_conf.set_new_project( - platform[0], platform[2]) - raise AutosubmitCritical( - "The experiment cannot be offered, changes are reverted", 7014) - Log.result("The experiment has been successfully offered.") - except Exception as e: - # todo put the IO error code - raise AutosubmitCritical(f"[LOCAL] Error offering the experiment: {str(e)}\n" - f"Please, try again", 7000) + @staticmethod + def migrate_offer(experiment_id,only_remote): + Autosubmit.migrate_offer_remote() + if not only_remote: + Autosubmit.migrate_offer_local(experiment_id) + @staticmethod def migrate_pickup(experiment_id, only_remote): Log.info('Migrating experiment {0}'.format(experiment_id)) -- GitLab From 06b4db03ee133adcb787f771a27b60ca37eb2eee Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 20 Apr 2023 09:40:48 +0200 Subject: [PATCH 5/6] migrate improves --- autosubmit/autosubmit.py | 60 +++++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index ff88c8b6c..ce5d691bb 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -2802,12 +2802,12 @@ class Autosubmit: :param platforms: list of platforms :return: dictionary with the directory as key and a list of platforms as value """ - platforms_by_dir = defaultdict([]) + platforms_by_dir = defaultdict(list) for platform in platforms: platforms_by_dir[platform.root_dir].append(platform) return platforms_by_dir @staticmethod - def check_migrate_config(as_conf,platforms): + def check_migrate_config(as_conf,platforms,new_platform_data): """ Checks if the configuration file has the necessary information to migrate the data :param as_conf: Autosubmit configuration file @@ -2818,20 +2818,16 @@ class Autosubmit: platforms_to_migrate = list() platforms_with_missconfiguration = list() platforms_by_dir = Autosubmit.get_platforms_grouped_by_dir(platforms) - for platform_dir in platforms_by_dir: + for platform_dir,platforms_list in platforms_by_dir.items(): platform_dir_error = True - for platform in platform_dir: - Log.info("Checking [{0}] from platforms configuration...", platform) - if as_conf.platforms[platform.name].get("USER",None) is None or \ - as_conf.platforms[platform.name].get("USER_TO",as_conf.platforms[platform.name].get("USER","")) == as_conf.platforms[platform.name].get("USER","") and not as_conf.platforms[platform.name].get("SAME_USER",False) or \ - as_conf.platforms[platform.name].get("PROJECT",None) is None or\ - as_conf.platforms[platform.name].get("PROJECT_TO",None) is None or \ - as_conf.platforms[platform].get("TEMP_DIR", "") == "": - Log.debug(f"Values: USER: {as_conf.platforms[platform.name].get('USER',None)}" - f" USER_TO: {as_conf.platforms[platform.name].get('USER_TO',None)}" - f" PROJECT: {as_conf.platforms[platform.name].get('PROJECT',None)}" - f" PROJECT_TO: {as_conf.platforms[platform.name].get('PROJECT_TO',None)}" - f" TEMP_DIR: {as_conf.platforms[platform].get('TEMP_DIR', '')}") + for platform in platforms_list: + Log.info(f"Checking [{platform.name}] from platforms configuration...") + if as_conf.platforms_data[platform.name].get("USER", None) == new_platform_data[platform.name].get("USER", None) and not as_conf.platforms_data[platform.name].get("SAME_USER",False): + Log.debug(f"Values: USER: {as_conf.platforms_data[platform.name].get('USER',None)}" + f" USER_TO: {new_platform_data[platform.name].get('USER_TO',None)}" + f" PROJECT: {as_conf.platforms_data[platform.name].get('PROJECT',None)}" + f" PROJECT_TO: {new_platform_data[platform.name].get('PROJECT_TO',None)}" + f" TEMP_DIR: {as_conf.platforms_data[platform].get('TEMP_DIR', '')}") Log.debug(f"Invalid configuration for platform [{platform.name}]\nTrying next platform...") else: Log.info("Valid configuration for platform [{0}]".format(platform.name)) @@ -2860,10 +2856,29 @@ class Autosubmit: raise AutosubmitCritical(f"[LOCAL] Error offering the experiment: {str(e)}\n" f"Please, try again", 7000) @staticmethod - def migrate_offer_remote(): - as_conf = AutosubmitConfig( - experiment_id, BasicConfig, YAMLParserFactory()) - as_conf.check_conf_files(True) + def migrate_offer_remote(experiment_id): + # Init the configuration + as_conf = AutosubmitConfig(experiment_id, BasicConfig, YAMLParserFactory()) + as_conf.check_conf_files(False) + # Load migrate + #Find migrate file + new_platform_data = as_conf.platforms_data + migrate_file = as_conf.experiment_data.get("AS_MIGRATE", None) + if migrate_file is None: + raise AutosubmitCritical("No migrate information found\nPlease add a key named AS_MIGRATE with the path to the file", 7014) + # expand home if needed + migrate_file = Path(os.path.expanduser(migrate_file)) + # If does not exist, raise error + if not migrate_file.exists(): + raise AutosubmitCritical(f"File {migrate_file} does not exist", 7014) + # Merge platform keys with migrate keys that should be the old credentials + # Migrate file consist of: + # platform_name: must match the platform name in the platforms configuration file, must have the old user + # USER: user + # PROJECT: project + # Host ( optional ) : host of the machine if using alias + # TEMP_DIR: temp dir for current platform, because can be different for each of them + as_conf.experiment_data = as_conf.load_config_file(as_conf.experiment_data,migrate_file) pkl_dir = os.path.join(BasicConfig.LOCAL_ROOT_DIR, experiment_id, 'pkl') job_list = Autosubmit.load_job_list(experiment_id, as_conf, notransitive=True, monitor=True) Log.debug("Job list restored from {0} files", pkl_dir) @@ -2885,10 +2900,9 @@ class Autosubmit: if submitter.platforms is None: return False Log.info("Checking remote platforms") - platforms = [x for x in submitter.platforms if x not in [ - 'local', 'LOCAL']] + #[x for x in submitter.platforms if x not in ['local', 'LOCAL']] # Checks and annotates the platforms to migrate ( one per directory if they share it ) - platforms_to_migrate = Autosubmit.check_migrate_config(as_conf, platforms) + platforms_to_migrate = Autosubmit.check_migrate_config(as_conf,platforms_to_test,new_platform_data) platforms_without_issues = list() for platform in platforms_to_migrate: p = submitter.platforms[platform] @@ -2941,7 +2955,7 @@ class Autosubmit: # TODO set user_to and project_to to the correct values @staticmethod def migrate_offer(experiment_id,only_remote): - Autosubmit.migrate_offer_remote() + Autosubmit.migrate_offer_remote(experiment_id) if not only_remote: Autosubmit.migrate_offer_local(experiment_id) -- GitLab From eed18eb7e8fa2e529b34f87051eb317852017569 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 21 Apr 2023 09:23:28 +0200 Subject: [PATCH 6/6] migrate improves --- autosubmit/autosubmit.py | 78 ++++++++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 26 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index ce5d691bb..6fc58eb58 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -2816,18 +2816,21 @@ class Autosubmit: """ error = False platforms_to_migrate = list() - platforms_with_missconfiguration = list() + platforms_with_missconfiguration = "" platforms_by_dir = Autosubmit.get_platforms_grouped_by_dir(platforms) for platform_dir,platforms_list in platforms_by_dir.items(): platform_dir_error = True for platform in platforms_list: + if platform.name.upper() == "LOCAL": + platform_dir_error = False + continue Log.info(f"Checking [{platform.name}] from platforms configuration...") if as_conf.platforms_data[platform.name].get("USER", None) == new_platform_data[platform.name].get("USER", None) and not as_conf.platforms_data[platform.name].get("SAME_USER",False): - Log.debug(f"Values: USER: {as_conf.platforms_data[platform.name].get('USER',None)}" - f" USER_TO: {new_platform_data[platform.name].get('USER_TO',None)}" - f" PROJECT: {as_conf.platforms_data[platform.name].get('PROJECT',None)}" - f" PROJECT_TO: {new_platform_data[platform.name].get('PROJECT_TO',None)}" - f" TEMP_DIR: {as_conf.platforms_data[platform].get('TEMP_DIR', '')}") + Log.debug(f"Values: USER: {as_conf.platforms_data[platform.name].get('USER',None)}\n" + f" USER_TO: {new_platform_data[platform.name].get('USER',None)}\n" + f" PROJECT: {as_conf.platforms_data[platform.name].get('PROJECT',None)}\n" + f" PROJECT_TO: {new_platform_data[platform.name].get('PROJECT',None)}\n" + f" TEMP_DIR: {as_conf.platforms_data[platform.name].get('TEMP_DIR', '')}\n") Log.debug(f"Invalid configuration for platform [{platform.name}]\nTrying next platform...") else: Log.info("Valid configuration for platform [{0}]".format(platform.name)) @@ -2841,7 +2844,7 @@ class Autosubmit: platform_names = platform_names[:-2] platforms_with_missconfiguration += f"{platform_dir}: {platform_names}\n" if error: - raise AutosubmitCritical(f"Invalid migrate configuration for platforms: {platforms_with_missconfiguration}", 7014) + raise AutosubmitCritical(f"Invalid migrate configuration for platforms: {platforms_with_missconfiguration} ", 7014) else: return platforms_to_migrate @@ -2856,13 +2859,33 @@ class Autosubmit: raise AutosubmitCritical(f"[LOCAL] Error offering the experiment: {str(e)}\n" f"Please, try again", 7000) @staticmethod + def get_migrate_info(as_conf): + migrate_file = as_conf.experiment_data.get("AS_MIGRATE", None) + if migrate_file is None: + raise AutosubmitCritical( + "No migrate information found\nPlease add a key named AS_MIGRATE with the path to the file", 7014) + + # expand home if needed + migrate_file = Path(os.path.expanduser(migrate_file)) + # If does not exist, raise error + if not migrate_file.exists(): + raise AutosubmitCritical(f"File {migrate_file} does not exist", 7014) + # Merge platform keys with migrate keys that should be the old credentials + # Migrate file consist of: + # platform_name: must match the platform name in the platforms configuration file, must have the old user + # USER: user + # PROJECT: project + # Host ( optional ) : host of the machine if using alias + # TEMP_DIR: temp dir for current platform, because can be different for each of them + return as_conf.load_config_file(as_conf.experiment_data, migrate_file) + @staticmethod def migrate_offer_remote(experiment_id): # Init the configuration as_conf = AutosubmitConfig(experiment_id, BasicConfig, YAMLParserFactory()) as_conf.check_conf_files(False) # Load migrate #Find migrate file - new_platform_data = as_conf.platforms_data + new_platform_data = copy.deepcopy(as_conf.platforms_data) migrate_file = as_conf.experiment_data.get("AS_MIGRATE", None) if migrate_file is None: raise AutosubmitCritical("No migrate information found\nPlease add a key named AS_MIGRATE with the path to the file", 7014) @@ -2878,7 +2901,7 @@ class Autosubmit: # PROJECT: project # Host ( optional ) : host of the machine if using alias # TEMP_DIR: temp dir for current platform, because can be different for each of them - as_conf.experiment_data = as_conf.load_config_file(as_conf.experiment_data,migrate_file) + as_conf.experiment_data = Autosubmit.get_migrate_info(as_conf,migrate_file) pkl_dir = os.path.join(BasicConfig.LOCAL_ROOT_DIR, experiment_id, 'pkl') job_list = Autosubmit.load_job_list(experiment_id, as_conf, notransitive=True, monitor=True) Log.debug("Job list restored from {0} files", pkl_dir) @@ -2905,10 +2928,8 @@ class Autosubmit: platforms_to_migrate = Autosubmit.check_migrate_config(as_conf,platforms_to_test,new_platform_data) platforms_without_issues = list() for platform in platforms_to_migrate: - p = submitter.platforms[platform] + p = submitter.platforms[platform.name] if p.root_dir != p.temp_dir and len(p.temp_dir) > 0: - # find /home/bsc32/bsc32070/dummy3 -type l -lname '/*' -printf ' ln -sf "$(realpath -s --relative-to="%p" $(readlink "%p")")" \n' > script.sh - # command = "find " + p.root_dir + " -type l -lname \'/*\' -printf 'var=\"$(realpath -s --relative-to=\"%p\" \"$(readlink \"%p\")\")\" && var=${var:3} && ln -sf $var \"%p\" \\n'" Log.info(f"Converting the absolute symlinks into relatives on platform [{platform.name}] ") command = f"find {p.root_dir} -type l -lname \'/*\' -printf 'var=\"$(realpath -s --relative-to=\"%p\" \"$(readlink \"%p\")\")\" && var=${{var:3}} && ln -sf $var \"%p\" \\n' " try: @@ -2961,20 +2982,20 @@ class Autosubmit: @staticmethod def migrate_pickup(experiment_id, only_remote): - Log.info('Migrating experiment {0}'.format(experiment_id)) - Log.info("Moving local files/dirs") + Log.info(f'Migrating experiment {experiment_id}') if not only_remote: - if not Autosubmit.unarchive(experiment_id, True): - raise AutosubmitCritical( - "The experiment cannot be picked up", 7012) + Log.info("Moving local files/dirs") + if not Autosubmit.unarchive(experiment_id, True, False): + if not Path(os.path.join(BasicConfig.LOCAL_ROOT_DIR, experiment_id)).exists(): + raise AutosubmitCritical( + "The experiment cannot be picked up", 7012) Log.info("Local files/dirs have been successfully picked up") else: exp_path = os.path.join( BasicConfig.LOCAL_ROOT_DIR, experiment_id) if not os.path.exists(exp_path): raise AutosubmitCritical( - "Experiment seems to be archived, no action is performed", 7012) - + "Experiment seems to be archived, no action is performed\nHint: Try to pickup without the remote flag", 7012) as_conf = AutosubmitConfig( experiment_id, BasicConfig, YAMLParserFactory()) as_conf.check_conf_files(False) @@ -2990,11 +3011,14 @@ class Autosubmit: if submitter.platforms is None: raise AutosubmitCritical("No platforms configured!!!", 7014) platforms = submitter.platforms + job_sections_check = set() for job in job_list.get_job_list(): - job.submitter = submitter - if job.platform_name is None: - job.platform_name = as_conf.get_platform() - platforms_to_test.add(platforms[job.platform_name]) + if job.section not in job_sections_check: + job_sections_check.add(job.section) + job.submitter = submitter + if job.platform_name is None: + job.platform_name = as_conf.get_platform() + platforms_to_test.add(platforms[job.platform_name]) Log.info("Checking remote platforms") platforms = [x for x in submitter.platforms if x not in [ @@ -4233,7 +4257,7 @@ class Autosubmit: return True @staticmethod - def unarchive(experiment_id, uncompressed=True): + def unarchive(experiment_id, uncompressed=True, show_err_log = True): """ Unarchives an experiment: uncompress folder from tar.gz and moves to experiment root folder @@ -4262,7 +4286,8 @@ class Autosubmit: year -= 1 if year == 2000: - Log.error("Experiment {0} is not archived", experiment_id) + if show_err_log: + Log.error("Experiment {0} is not archived", experiment_id) return False Log.info("Experiment located in {0} archive", year) @@ -4276,7 +4301,8 @@ class Autosubmit: tar.close() except Exception as e: shutil.rmtree(exp_folder, ignore_errors=True) - Log.printlog("Can not extract tar file: {0}".format(str(e)), 6012) + if show_err_log: + Log.printlog("Can not extract tar file: {0}".format(str(e)), 6012) return False Log.info("Unpacking finished") -- GitLab