diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 5530ba3f50c92383756f7f485c5ddb959e81f4e1..bf434b4efce80b194216351654b2fd8586822ccf 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -50,6 +50,7 @@ import random import signal import datetime import portalocker +import pwd from pkg_resources import require, resource_listdir, resource_exists, resource_string from distutils.util import strtobool @@ -214,6 +215,13 @@ class Autosubmit: subparser.add_argument('--hide', action='store_true', default=False, help='hides plot window') + # Migrate + subparser = subparsers.add_parser('migrate', description="Migrate experiments from current user to another") + subparser.add_argument('expid', help='experiment identifier') + group = subparser.add_mutually_exclusive_group(required=True) + group.add_argument('-o', '--offer', action="store_true", default=False, help='Offer experiment') + group.add_argument('-p', '--pickup', action="store_true", default=False, help='Pick-up released experiment') + # Check subparser = subparsers.add_parser('check', description="check configuration for specified experiment") subparser.add_argument('expid', help='experiment identifier') @@ -340,6 +348,8 @@ class Autosubmit: return Autosubmit.recovery(args.expid, args.noplot, args.save, args.all, args.hide) elif args.command == 'check': return Autosubmit.check(args.expid) + elif args.command == 'migrate': + return Autosubmit.migrate(args.expid, args.offer, args.pickup) elif args.command == 'create': return Autosubmit.create(args.expid, args.noplot, args.hide, args.output) elif args.command == 'configure': @@ -499,7 +509,8 @@ class Autosubmit: Log.debug("Creating temporal directory...") exp_id_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id) - os.mkdir(os.path.join(exp_id_path, "tmp"), 0o775) + os.mkdir(os.path.join(exp_id_path, "tmp")) + os.chmod(os.path.join(exp_id_path, "tmp"), 0o775) Log.debug("Creating pkl directory...") os.mkdir(os.path.join(exp_id_path, "pkl")) @@ -1052,6 +1063,105 @@ class Autosubmit: return True + @staticmethod + def migrate(experiment_id, offer, pickup): + """ + Migrates experiment files from current to other user. + It takes mapping information for new user from config files. + + :param experiment_id: experiment identifier: + :param pickup: + :param offer: + """ + log_file = os.path.join(BasicConfig.LOCAL_ROOT_DIR, "ASlogs", 'migrate_{0}.log'.format(experiment_id)) + Log.set_file(log_file) + + if offer: + Log.info('Migrating experiment {0}'.format(experiment_id)) + as_conf = AutosubmitConfig(experiment_id, BasicConfig, ConfigParserFactory()) + if not as_conf.check_conf_files(): + Log.critical('Can not proceed with invalid configuration') + return False + + submitter = Autosubmit._get_submitter(as_conf) + submitter.load_platforms(as_conf) + if submitter.platforms is None: + return False + + Log.info("Checking remote platforms") + platforms = filter(lambda x: x not in ['local', 'LOCAL'], submitter.platforms) + for platform in platforms: + Log.info("Updating {0} platform configuration with target user", platform) + if not as_conf.get_migrate_user_to(platform): + Log.critical("Missing target user in platforms configuration file") + return False + + as_conf.set_new_user(platform, as_conf.get_migrate_user_to(platform)) + Log.info("User in platform configuration file successfully updated to {0}", + as_conf.get_migrate_user_to(platform)) + + if as_conf.get_migrate_project_to(platform): + Log.info("Updating {0} platform configuration with target project", platform) + as_conf.set_new_project(platform, as_conf.get_migrate_project_to(platform)) + Log.info("Project in platform configuration file successfully updated to {0}", + as_conf.get_migrate_user_to(platform)) + else: + Log.warning("Project in platforms configuration file remains unchanged") + + Log.info("Moving remote files/dirs on {0}", platform) + p = submitter.platforms[platform] + Log.info("Moving from {0} to {1}", os.path.join(p.root_dir), + os.path.join(p.temp_dir, experiment_id)) + if not p.move_file(os.path.join(p.root_dir), os.path.join(p.temp_dir, experiment_id)): + Log.critical("The files/dirs on {0} cannot be moved to {1}.", p.root_dir, + os.path.join(p.temp_dir, experiment_id)) + return False + + Log.result("Files/dirs on {0} have been successfully offered", platform) + + Log.info("Moving local files/dirs") + if not Autosubmit.archive(experiment_id, False): + Log.critical("The experiment cannot be offered") + return False + + Log.result("The experiment has been successfully offered.") + + elif pickup: + Log.info('Migrating experiment {0}'.format(experiment_id)) + Log.info("Moving local files/dirs") + if not Autosubmit.unarchive(experiment_id): + Log.critical("The experiment cannot be picked up") + return False + Log.info("Local files/dirs have been sucessfully picked up") + as_conf = AutosubmitConfig(experiment_id, BasicConfig, ConfigParserFactory()) + if not as_conf.check_conf_files(): + Log.critical('Can not proceed with invalid configuration') + return False + + Log.info("Checking remote platforms") + submitter = Autosubmit._get_submitter(as_conf) + submitter.load_platforms(as_conf) + if submitter.platforms is None: + return False + + platforms = filter(lambda x: x not in ['local', 'LOCAL'], submitter.platforms) + for platform in platforms: + Log.info("Copying remote files/dirs on {0}", platform) + p = submitter.platforms[platform] + Log.info("Copying from {0} to {1}", os.path.join(p.temp_dir, experiment_id), + os.path.join(p.root_dir)) + if not p.send_command("cp -r " + os.path.join(p.temp_dir, experiment_id) + " " + + os.path.join(p.root_dir)): + Log.critical("The files/dirs on {0} cannot be copied to {1}.", + os.path.join(p.temp_dir, experiment_id), p.root_dir) + return False + + Log.result("Files/dirs on {0} have been successfully picked up", platform) + + Log.result("The experiment has been successfully picked up.") + + return True + @staticmethod def check(experiment_id): """ @@ -1444,11 +1554,13 @@ class Autosubmit: return True @staticmethod - def archive(expid): + def archive(expid, clean=True): """ Archives an experiment: call clean (if experiment is of version 3 or later), compress folder to tar.gz and moves to year's folder + :param clean: + :return: :param expid: experiment identifier :type expid: str """ @@ -1459,14 +1571,15 @@ class Autosubmit: Log.warning("Does an experiment with the given id exist?") return 1 - Log.set_file(os.path.join(BasicConfig.LOCAL_ROOT_DIR, "ASlogs", 'archive{0}.log'.format(expid))) + Log.set_file(os.path.join(BasicConfig.LOCAL_ROOT_DIR, "ASlogs", 'archive_{0}.log'.format(expid))) exp_folder = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid) - # Cleaning to reduce file size. - version = get_autosubmit_version(expid) - if version is not None and version.startswith('3') and not Autosubmit.clean(expid, True, True, True, False): - Log.critical("Can not archive project. Clean not successful") - return False + if clean: + # Cleaning to reduce file size. + version = get_autosubmit_version(expid) + if version is not None and version.startswith('3') and not Autosubmit.clean(expid, True, True, True, False): + Log.critical("Can not archive project. Clean not successful") + return False # Getting year of last completed. If not, year of expid folder year = None @@ -1491,6 +1604,7 @@ class Autosubmit: with tarfile.open(os.path.join(year_path, '{0}.tar.gz'.format(expid)), "w:gz") as tar: tar.add(exp_folder, arcname='') tar.close() + os.chmod(os.path.join(year_path, '{0}.tar.gz'.format(expid)), 0o775) except Exception as e: Log.critical("Can not write tar file: {0}".format(e)) return False @@ -1516,7 +1630,7 @@ class Autosubmit: :type experiment_id: str """ BasicConfig.read() - Log.set_file(os.path.join(BasicConfig.LOCAL_ROOT_DIR, "ASlogs", 'unarchive{0}.log'.format(experiment_id))) + Log.set_file(os.path.join(BasicConfig.LOCAL_ROOT_DIR, "ASlogs", 'unarchive_{0}.log'.format(experiment_id))) exp_folder = os.path.join(BasicConfig.LOCAL_ROOT_DIR, experiment_id) if os.path.exists(exp_folder): diff --git a/autosubmit/config/config_common.py b/autosubmit/config/config_common.py index 998f1bb0cfe3aa379cc7a1fca32179e415884543..83769e5e1586df4e290fe83ea75285442977b72f 100644 --- a/autosubmit/config/config_common.py +++ b/autosubmit/config/config_common.py @@ -204,6 +204,48 @@ class AutosubmitConfig(object): """ return str(self._jobs_parser.get_option(section, 'MEMORY_PER_TASK', '')) + def get_migrate_user_to(self, section): + """ + Returns the user to change to from platform config file. + + :return: migrate user to + :rtype: str + """ + return self._platforms_parser.get_option(section, 'USER_TO', '').lower() + + def set_new_user(self, section, new_user): + """ + Sets new user for given platform + :param new_user: + :param section: platform name + :type: str + """ + content = open(self._platforms_parser_file).read() + if re.search(section, content): + content = content.replace(re.search('USER =.*', content).group(0), "USER = " + new_user) + open(self._platforms_parser_file, 'w').write(content) + + def get_migrate_project_to(self, section): + """ + Returns the project to change to from platform config file. + + :return: migrate project to + :rtype: str + """ + return self._platforms_parser.get_option(section, 'PROJECT_TO', '').lower() + + def set_new_project(self, section, new_project): + """ + Sets new project for given platform + :param new_project: + :param section: platform name + :type: str + """ + content = open(self._platforms_parser_file).read() + if re.search(section, content): + content = content.replace(re.search('PROJECT =.*', content).group(0), "PROJECT = " + new_project) + open(self._platforms_parser_file, 'w').write(content) + def get_custom_directives(self, section): """ Gets custom directives needed for the given job type diff --git a/autosubmit/config/files/autosubmit.conf b/autosubmit/config/files/autosubmit.conf index 78dc9f4239d3fcd5a733550bcc0cca17f23cc0e2..90770b8d0a9611157495ea14d9dfa8518faae208 100644 --- a/autosubmit/config/files/autosubmit.conf +++ b/autosubmit/config/files/autosubmit.conf @@ -36,3 +36,7 @@ API = paramiko TYPE = pkl # Defines if the remote logs will be copied to the local platform. Default = True. COPY_REMOTE_LOGS = True + +[migrate] +# Changes experiment files owner. +TO_USER = diff --git a/autosubmit/config/files/platforms.conf b/autosubmit/config/files/platforms.conf index e5e6ff4c3f4a8152a1672eebcf920a4d86664e45..a06f59dd0c0a234b8027d5fe180590487bdb8227 100644 --- a/autosubmit/config/files/platforms.conf +++ b/autosubmit/config/files/platforms.conf @@ -16,8 +16,12 @@ # ADD_PROJECT_TO_HOST = False ## User for the machine scheduler. Required # USER = +## Optional. If given, Autosubmit will change owner of files in given platform when using migrate_exp. +# USER_TO = ## Path to the scratch directory for the machine. Required. # SCRATCH_DIR = /scratch +## Path to the machine's temporary directory for migrate purposes. +# TEMP_DIR = /tmp ## If true, Autosubmit test command can use this queue as a main queue. Defaults to False # TEST_SUITE = False ## If given, Autosubmit will add jobs to the given queue. Required for some platforms. diff --git a/autosubmit/platforms/paramiko_submitter.py b/autosubmit/platforms/paramiko_submitter.py index d84915c5ab1508c05fb7f12c8968a16d1b0502b3..404ab9eedbe5217a985bbf97a324bfee0530d35a 100644 --- a/autosubmit/platforms/paramiko_submitter.py +++ b/autosubmit/platforms/paramiko_submitter.py @@ -71,6 +71,7 @@ class ParamikoSubmitter(Submitter): local_platform.max_waiting_jobs = asconf.get_max_waiting_jobs() local_platform.total_jobs = asconf.get_total_jobs() local_platform.scratch = os.path.join(BasicConfig.LOCAL_ROOT_DIR, asconf.expid, BasicConfig.LOCAL_TMP_DIR) + local_platform.temp_dir = os.path.join(BasicConfig.LOCAL_ROOT_DIR, 'ASlogs') local_platform.root_dir = os.path.join(BasicConfig.LOCAL_ROOT_DIR, local_platform.expid) local_platform.host = 'localhost' platforms['local'] = local_platform @@ -129,6 +130,7 @@ class ParamikoSubmitter(Submitter): remote_platform.exclusivity = parser.get_option(section, 'EXCLUSIVITY', '').lower() remote_platform.user = parser.get_option(section, 'USER', None) remote_platform.scratch = parser.get_option(section, 'SCRATCH_DIR', None) + remote_platform.temp_dir = parser.get_option(section, 'TEMP_DIR', None) remote_platform._default_queue = parser.get_option(section, 'QUEUE', None) remote_platform._serial_queue = parser.get_option(section, 'SERIAL_QUEUE', None) remote_platform.processors_per_node = parser.get_option(section, 'PROCESSORS_PER_NODE', diff --git a/autosubmit/platforms/platform.py b/autosubmit/platforms/platform.py index 9b1aca5ca6b8438d1b2b45930fb1f43d3fcbf861..cef7957cdfa91f16c0665eea44745a5cbc3a64ac 100644 --- a/autosubmit/platforms/platform.py +++ b/autosubmit/platforms/platform.py @@ -36,6 +36,7 @@ class Platform(object): self.exclusivity = '' self.type = '' self.scratch = '' + self.temp_dir = '' self.root_dir = '' self.service = None self.scheduler = None @@ -129,6 +130,7 @@ class Platform(object): parameters['{0}EXCLUSIVITY'.format(prefix)] = self.exclusivity parameters['{0}TYPE'.format(prefix)] = self.type parameters['{0}SCRATCH_DIR'.format(prefix)] = self.scratch + parameters['{0}TEMP_DIR'.format(prefix)] = self.temp_dir parameters['{0}ROOTDIR'.format(prefix)] = self.root_dir parameters['{0}LOGDIR'.format(prefix)] = self.get_files_path() diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 89c6ee80561fd2d71921ee1a37a0fc0b979bf2b3..069307499fe720bc82e7ec3bf7e79e378e4969e4 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -23,6 +23,7 @@ Command list -install Install database for Autosubmit on the configured folder -archive Clean, compress and remove from the experiments' folder a finalized experiment -unarchive Restores an archived experiment +-migrate_exp Migrates an experiment from one user to another How to create an experiment @@ -833,6 +834,34 @@ Example: autosubmit unarchive cxxx +How to migrate an experiment +============================ +To migrate an experiment from one user to another, you need to add two parameters in the platforms configuration file: + + * USER_TO = + * TEMP_DIR = + +Then, just run the command: +:: + + autosubmit migrate_exp --ofer expid + + +Local files will be archived and remote files put in the HPC temporary directory. + +.. warning:: The temporary directory must be readable by both users (old owner and new owner). + +Then the new owner will have to run the command: +:: + + autosubmit migrate_exp --pickup expid + + + +Local files will be unarchived and remote files copied from the temporal loaction. + +.. warning:: The old owner might need to remove temporal files and archive. + How to configure email notifications ==================================== diff --git a/test/regression/default_conf/platforms.conf b/test/regression/default_conf/platforms.conf index 8a3a3c058ede27ea0f3c1a679dbb9c6793df0482..ae146bd42ec8e83e51bf9382627c67b73be25ff2 100644 --- a/test/regression/default_conf/platforms.conf +++ b/test/regression/default_conf/platforms.conf @@ -31,15 +31,16 @@ TEST_SUITE = False QUEUE = serial [marenostrum3] -TYPE = LSF +TYPE = slurm VERSION = mn HOST = mn-bsc32 PROJECT = bsc32 +QUEUE = debug ADD_PROJECT_TO_HOST = false -USER = bsc32649 +USER = bsc32704 SCRATCH_DIR = /gpfs/scratch TEST_SUITE = True -PROCESSORS_PER_NODE = 16 +PROCESSORS_PER_NODE = 48 [mistral] TYPE = slurm