From 49fd1691a54cfe49b4733482d5407b259ad45427 Mon Sep 17 00:00:00 2001 From: Domingo Manubens-Gil Date: Tue, 18 Apr 2017 16:13:19 +0200 Subject: [PATCH 1/7] Start branch issue70 --- autosubmit/autosubmit.py | 32 ++++++++++++++++++++++++++++++++ test/unit/test_handle_exp.py | 23 +++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 test/unit/test_handle_exp.py diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 5530ba3f5..2097ff0d2 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -214,6 +214,11 @@ class Autosubmit: subparser.add_argument('--hide', action='store_true', default=False, help='hides plot window') + # Handle + subparser = subparsers.add_parser('handle', description="Handle experiments to another user") + subparser.add_argument('expid', help='experiment identifier') + subparser.add_argument('-mf', '--mapfile', help='map users file') + # Check subparser = subparsers.add_parser('check', description="check configuration for specified experiment") subparser.add_argument('expid', help='experiment identifier') @@ -340,6 +345,8 @@ class Autosubmit: return Autosubmit.recovery(args.expid, args.noplot, args.save, args.all, args.hide) elif args.command == 'check': return Autosubmit.check(args.expid) + elif args.command == 'handle': + return Autosubmit.handle(args.expid, args.mapfile) elif args.command == 'create': return Autosubmit.create(args.expid, args.noplot, args.hide, args.output) elif args.command == 'configure': @@ -1052,6 +1059,31 @@ class Autosubmit: return True + @staticmethod + def handle(experiment_id, map_file): + """ + Handles experiment files to another users. It takes mapping information of old + user and new user from a map file. + + :param experiment_id: experiment identifier: + :param map_file: map file: + """" + BasicConfig.read() + exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, experiment_id) + if not os.path.exists(exp_path): + Log.critical("The directory {0} is needed and does not exist.", exp_path) + Log.warning("Does an experiment with the given id exist?") + return False + + log_file = os.path.join(BasicConfig.LOCAL_ROOT_DIR, experiment_id, BasicConfig.LOCAL_TMP_DIR, 'handle_exp.log') + Log.set_file(log_file) + + as_conf = AutosubmitConfig(experiment_id, BasicConfig, ConfigParserFactory()) + if not as_conf.check_conf_files(): + return False + + + @staticmethod def check(experiment_id): """ diff --git a/test/unit/test_handle_exp.py b/test/unit/test_handle_exp.py new file mode 100644 index 000000000..df468d9a9 --- /dev/null +++ b/test/unit/test_handle_exp.py @@ -0,0 +1,23 @@ +from unittest import TestCase +from mock import Mock, patch + + +class TestHandleExp(TestCase): + def setUp(self): + self.user_from = "old-user" + self.user_to = "new-user" + + def testFoo(self): + self.failUnless(False) + +# @patch('autosubmit.autosubmit.handle') +# def test_handle_experiment(self, db_common_mock): +# current_user_id = "old-user" +# self._build_db_mock(current_experiment_id, db_common_mock) +# user_id = handle_experiment(self.user_from, self.user_to) +# self.assertEquals("new_user", user_id) +# +# @staticmethod +# def _build_db_mock(current_experiment_id, mock_db_common): +# mock_db_common.last_name_used = Mock(return_value=current_experiment_id) +# mock_db_common.check_experiment_exists = Mock(return_value=False) -- GitLab From 2ea04c929bcce759e91bf9209142c2c2afd63a30 Mon Sep 17 00:00:00 2001 From: Domingo Manubens-Gil Date: Thu, 18 May 2017 18:37:05 +0200 Subject: [PATCH 2/7] Progress on the tests and methods to migrate experiment owner --- autosubmit/autosubmit.py | 29 ++++++++------ autosubmit/config/config_common.py | 9 +++++ autosubmit/config/files/autosubmit.conf | 4 ++ autosubmit/config/files/platforms.conf | 4 +- autosubmit/experiment/experiment_common.py | 34 +++++++++++++++++ ...t_ config.py => test_autosubmit_config.py} | 32 +++++++++++++--- test/unit/test_handle_exp.py | 23 ----------- test/unit/test_migrate_exp.py | 38 +++++++++++++++++++ 8 files changed, 132 insertions(+), 41 deletions(-) rename test/unit/{test_autosubmit_ config.py => test_autosubmit_config.py} (94%) delete mode 100644 test/unit/test_handle_exp.py create mode 100644 test/unit/test_migrate_exp.py diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 2097ff0d2..a51f18236 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -72,6 +72,7 @@ from bscearth.utils.log import Log from database.db_common import create_db from experiment.experiment_common import new_experiment from experiment.experiment_common import copy_experiment +from experiment.experiment_common import migrate_experiment from database.db_common import delete_experiment from database.db_common import get_autosubmit_version from monitor.monitor import Monitor @@ -214,10 +215,9 @@ class Autosubmit: subparser.add_argument('--hide', action='store_true', default=False, help='hides plot window') - # Handle - subparser = subparsers.add_parser('handle', description="Handle experiments to another user") + # Migrate + subparser = subparsers.add_parser('migrate', description="Migrate experiments from current user to another") subparser.add_argument('expid', help='experiment identifier') - subparser.add_argument('-mf', '--mapfile', help='map users file') # Check subparser = subparsers.add_parser('check', description="check configuration for specified experiment") @@ -345,8 +345,8 @@ class Autosubmit: return Autosubmit.recovery(args.expid, args.noplot, args.save, args.all, args.hide) elif args.command == 'check': return Autosubmit.check(args.expid) - elif args.command == 'handle': - return Autosubmit.handle(args.expid, args.mapfile) + elif args.command == 'migrate': + return Autosubmit.migrate(args.expid, args.mapfile) elif args.command == 'create': return Autosubmit.create(args.expid, args.noplot, args.hide, args.output) elif args.command == 'configure': @@ -1060,14 +1060,14 @@ class Autosubmit: return True @staticmethod - def handle(experiment_id, map_file): + def migrate(experiment_id, map_file): """ - Handles experiment files to another users. It takes mapping information of old + Migrates experiment files from current to other user. It takes mapping information of old user and new user from a map file. :param experiment_id: experiment identifier: :param map_file: map file: - """" + """ BasicConfig.read() exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, experiment_id) if not os.path.exists(exp_path): @@ -1075,12 +1075,19 @@ class Autosubmit: Log.warning("Does an experiment with the given id exist?") return False - log_file = os.path.join(BasicConfig.LOCAL_ROOT_DIR, experiment_id, BasicConfig.LOCAL_TMP_DIR, 'handle_exp.log') + log_file = os.path.join(BasicConfig.LOCAL_ROOT_DIR, experiment_id, BasicConfig.LOCAL_TMP_DIR, 'migrate_exp.log') Log.set_file(log_file) - as_conf = AutosubmitConfig(experiment_id, BasicConfig, ConfigParserFactory()) - if not as_conf.check_conf_files(): + #as_conf = AutosubmitConfig(experiment_id, BasicConfig, ConfigParserFactory()) + #if not as_conf.check_conf_files(): + #return False + user_to = "cprodhom" + + if not migrate_experiment(exp_path, user_to): + Log.critical("The directory owner for {0} cannot be changed to {1}.", exp_path, user_to) return False + + return True diff --git a/autosubmit/config/config_common.py b/autosubmit/config/config_common.py index 70356b200..a35958de5 100644 --- a/autosubmit/config/config_common.py +++ b/autosubmit/config/config_common.py @@ -902,6 +902,15 @@ class AutosubmitConfig(object): """ return self._conf_parser.get_option('storage', 'TYPE', 'pkl').lower() + def get_migrate_to(self): + """ + Returns the user to change experiment's owner to from autosubmit's config file. + + :return: migrate to user + :rtype: str + """ + return self._conf_parser.get_option('migrate', 'TO_USER', '').lower() + @staticmethod def is_valid_mail_address(mail_address): if re.match('^[_a-z0-9-]+(\.[_a-z0-9-]+)*@[a-z0-9-]+(\.[a-z0-9-]+)*(\.[a-z]{2,4})$', mail_address): diff --git a/autosubmit/config/files/autosubmit.conf b/autosubmit/config/files/autosubmit.conf index 78dc9f423..6bd2c4169 100644 --- a/autosubmit/config/files/autosubmit.conf +++ b/autosubmit/config/files/autosubmit.conf @@ -36,3 +36,7 @@ API = paramiko TYPE = pkl # Defines if the remote logs will be copied to the local platform. Default = True. COPY_REMOTE_LOGS = True + +[migrate] +# Changes owner of experiment files. +TO_USER = diff --git a/autosubmit/config/files/platforms.conf b/autosubmit/config/files/platforms.conf index 0fa5eee7f..b2f39fd59 100644 --- a/autosubmit/config/files/platforms.conf +++ b/autosubmit/config/files/platforms.conf @@ -16,6 +16,8 @@ # ADD_PROJECT_TO_HOST = False ## User for the machine scheduler. Required # USER = +## Optional. If given, Autosubmit will change owner of files in given platform. +# USER_TO = ## Path to the scratch directory for the machine. Required. # SCRATCH_DIR = /scratch ## If true, Autosubmit test command can use this queue as a main queue. Defaults to False @@ -45,4 +47,4 @@ # MAX_WALLCLOCK = 72:00 ## Max processors number per job submitted to the HPC. If not specified, defaults to empty. ## Optional. Required for wrappers. -# MAX_PROCESSORS = 1 \ No newline at end of file +# MAX_PROCESSORS = 1 diff --git a/autosubmit/experiment/experiment_common.py b/autosubmit/experiment/experiment_common.py index 43e78191f..cb9291740 100644 --- a/autosubmit/experiment/experiment_common.py +++ b/autosubmit/experiment/experiment_common.py @@ -20,6 +20,9 @@ """ Module containing functions to manage autosubmit's experiments. """ +import os +import pwd +import glob import string import autosubmit.database.db_common as db_common from bscearth.utils.log import Log @@ -169,3 +172,34 @@ def base36decode(number): :rtype: int """ return int(number, 36) + + +def migrate_experiment(exp_path, user_to): + """ + Migrates experiment files from current to new user. + Group id will be kept the same. + + :param exp_path: experiment path: + :param user_to: to user: + """ + to_uid = pwd.getpwnam(user_to).pw_uid + Log.info("The UID for {0} is {1}.", user_to, to_uid) + current_gid = os.stat(exp_path).st_gid + Log.info("GID will be kept to {0}.", current_gid) + recursive_file_permissions(exp_path, to_uid, current_gid) + return user_to, current_gid + +def recursive_file_permissions(path,uid=-1,gid=-1): + """ + Recursively updates file permissions on a given path. + UID and GID default to -1 + """ + os.chown(path,uid,gid) + for item in glob.glob(path+'/*'): + if os.path.isdir(item): + recursive_file_permissions(os.path.join(path,item),uid,gid) + else: + try: + os.chown(os.path.join(path,item),uid,gid) + except: + Log.critical('File permissions on {0} not updated due to error.', os.path.join(path,item)) diff --git a/test/unit/test_autosubmit_ config.py b/test/unit/test_autosubmit_config.py similarity index 94% rename from test/unit/test_autosubmit_ config.py rename to test/unit/test_autosubmit_config.py index eea91d336..7729d4a30 100644 --- a/test/unit/test_autosubmit_ config.py +++ b/test/unit/test_autosubmit_config.py @@ -136,6 +136,16 @@ class TestAutosubmitConfig(TestCase): # assert self._assert_get_option(parser_mock, 'MEMORY', expected_value, returned_value, default_value, str) + def test_get_user_to(self): + # arrange + expected_value = 'new_user' + default_value = '' + config, parser_mock = self._arrange_config(expected_value) + # act + returned_value = config.get_chown(self.section) + # assert + self._assert_get_option(parser_mock, 'USER_TO', expected_value, returned_value, default_value, str) + def test_that_reload_must_load_parsers(self): # arrange config = AutosubmitConfig(self.any_expid, FakeBasicConfig, ConfigParserFactory()) @@ -204,6 +214,16 @@ class TestAutosubmitConfig(TestCase): # assert open_mock.assert_any_call(getattr(config, '_conf_parser_file'), 'w') + def test_get_to_user(self): + # arrange + expected_value = 'new_user' + default_value = '' + config, parser_mock = self._arrange_config(expected_value) + # act + returned_value = config.get_to_user(self.section) + # assert + self._assert_get_option(parser_mock, 'TO_USER', expected_value, returned_value, default_value, str) + def test_load_project_parameters(self): # arrange parser_mock = Mock(spec=ConfigParser) @@ -444,11 +464,11 @@ class TestAutosubmitConfig(TestCase): class FakeBasicConfig: - DB_DIR = '/dummy/db/dir' - DB_FILE = '/dummy/db/file' - DB_PATH = '/dummy/db/path' - LOCAL_ROOT_DIR = '/dummy/local/root/dir' - LOCAL_TMP_DIR = '/dummy/local/temp/dir' - LOCAL_PROJ_DIR = '/dummy/local/proj/dir' + DB_DIR = '/scratch/Earth/dmanuben/dummy/db/dir' + DB_FILE = '/scratch/Earth/dmanuben/dummy/db/file' + DB_PATH = '/scratch/Earth/dmanuben/dummy/db/path' + LOCAL_ROOT_DIR = '/scratch/Earth/dmanuben/dummy/local/root/dir' + LOCAL_TMP_DIR = '/scratch/Earth/dmanuben/dummy/local/temp/dir' + LOCAL_PROJ_DIR = '/scratch/Earth/dmanuben/dummy/local/proj/dir' DEFAULT_PLATFORMS_CONF = '' DEFAULT_JOBS_CONF = '' diff --git a/test/unit/test_handle_exp.py b/test/unit/test_handle_exp.py deleted file mode 100644 index df468d9a9..000000000 --- a/test/unit/test_handle_exp.py +++ /dev/null @@ -1,23 +0,0 @@ -from unittest import TestCase -from mock import Mock, patch - - -class TestHandleExp(TestCase): - def setUp(self): - self.user_from = "old-user" - self.user_to = "new-user" - - def testFoo(self): - self.failUnless(False) - -# @patch('autosubmit.autosubmit.handle') -# def test_handle_experiment(self, db_common_mock): -# current_user_id = "old-user" -# self._build_db_mock(current_experiment_id, db_common_mock) -# user_id = handle_experiment(self.user_from, self.user_to) -# self.assertEquals("new_user", user_id) -# -# @staticmethod -# def _build_db_mock(current_experiment_id, mock_db_common): -# mock_db_common.last_name_used = Mock(return_value=current_experiment_id) -# mock_db_common.check_experiment_exists = Mock(return_value=False) diff --git a/test/unit/test_migrate_exp.py b/test/unit/test_migrate_exp.py new file mode 100644 index 000000000..eeeb1132a --- /dev/null +++ b/test/unit/test_migrate_exp.py @@ -0,0 +1,38 @@ +from unittest import TestCase +from mock import Mock, patch +from autosubmit.experiment.experiment_common import migrate_experiment +import os +import pwd + + +class TestMigrateExp(TestCase): + def setUp(self): + #self.user_from = "old-user" + self.user_from = "dmanuben" + #self.user_to = "new-user" + self.user_to = "dmanuben" + +# def testFoo(self): +# self.failUnless(False) + + @patch('autosubmit.experiment.experiment_common.os') + def test_migrate_experiment(self, mock_os): + current_user_id = "old-user" + user_id, group_id = migrate_experiment("any path", self.user_to) + + to_uid = pwd.getpwnam(self.user_to).pw_uid + mock_os.chown.assert_called_with("any path", to_uid, group_id) + #self.assertEquals("new_user", user_id) + +# @patch('autosubmit.experiment.experiment_common.db_common') +# def test_create_new_experiment(self, db_common_mock): +# current_experiment_id = "empty" +# self._build_db_mock(current_experiment_id, db_common_mock) +# experiment_id = new_experiment(self.description, self.version) +# self.assertEquals("a000", experiment_id) + +# +# @staticmethod +# def _build_db_mock(current_experiment_id, mock_db_common): +# mock_db_common.last_name_used = Mock(return_value=current_experiment_id) +# mock_db_common.check_experiment_exists = Mock(return_value=False) -- GitLab From ac6deeb74a1b1d7c9d72968f75e7b5e5385f5e6c Mon Sep 17 00:00:00 2001 From: Domingo Manubens-Gil Date: Mon, 29 May 2017 15:36:27 +0200 Subject: [PATCH 3/7] Migrate offer and pickup working by using archive/unarchive methods. Archived file permissions are 775 now --- autosubmit/autosubmit.py | 63 +++++++++++----------- autosubmit/config/config_common.py | 2 +- autosubmit/experiment/experiment_common.py | 34 ------------ test/unit/test_autosubmit_config.py | 20 ------- 4 files changed, 31 insertions(+), 88 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index a51f18236..0a545d9ec 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -50,6 +50,7 @@ import random import signal import datetime import portalocker +import pwd from pkg_resources import require, resource_listdir, resource_exists, resource_string from distutils.util import strtobool @@ -72,7 +73,8 @@ from bscearth.utils.log import Log from database.db_common import create_db from experiment.experiment_common import new_experiment from experiment.experiment_common import copy_experiment -from experiment.experiment_common import migrate_experiment +from experiment.experiment_common import migrate_experiment_offer +from experiment.experiment_common import migrate_experiment_pickup from database.db_common import delete_experiment from database.db_common import get_autosubmit_version from monitor.monitor import Monitor @@ -218,6 +220,9 @@ class Autosubmit: # Migrate subparser = subparsers.add_parser('migrate', description="Migrate experiments from current user to another") subparser.add_argument('expid', help='experiment identifier') + group = subparser.add_mutually_exclusive_group(required=True) + group.add_argument('-o', '--offer', action="store_true", default=False, help='Offer experiment') + group.add_argument('-p', '--pickup', action="store_true", default=False, help='Pick-up released experiment') # Check subparser = subparsers.add_parser('check', description="check configuration for specified experiment") @@ -346,7 +351,7 @@ class Autosubmit: elif args.command == 'check': return Autosubmit.check(args.expid) elif args.command == 'migrate': - return Autosubmit.migrate(args.expid, args.mapfile) + return Autosubmit.migrate(args.expid, args.offer, args.pickup) elif args.command == 'create': return Autosubmit.create(args.expid, args.noplot, args.hide, args.output) elif args.command == 'configure': @@ -1060,37 +1065,27 @@ class Autosubmit: return True @staticmethod - def migrate(experiment_id, map_file): + def migrate(experiment_id, offer, pickup): """ - Migrates experiment files from current to other user. It takes mapping information of old - user and new user from a map file. + Migrates experiment files from current to other user. + It takes mapping information for new user from config files. :param experiment_id: experiment identifier: - :param map_file: map file: """ - BasicConfig.read() - exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, experiment_id) - if not os.path.exists(exp_path): - Log.critical("The directory {0} is needed and does not exist.", exp_path) - Log.warning("Does an experiment with the given id exist?") - return False + if offer: + Autosubmit.archive(experiment_id, False) + log_file = os.path.join(BasicConfig.LOCAL_ROOT_DIR, "ASlogs", 'migrate_{0}.log'.format(experiment_id)) + Log.set_file(log_file) + Log.result("The experiment has been successfully offered.") - log_file = os.path.join(BasicConfig.LOCAL_ROOT_DIR, experiment_id, BasicConfig.LOCAL_TMP_DIR, 'migrate_exp.log') - Log.set_file(log_file) + elif pickup: + Autosubmit.unarchive(experiment_id) + log_file = os.path.join(BasicConfig.LOCAL_ROOT_DIR, "ASlogs", 'migrate_{0}.log'.format(experiment_id)) + Log.set_file(log_file) + Log.result("The experiment has been successfully picked up.") - #as_conf = AutosubmitConfig(experiment_id, BasicConfig, ConfigParserFactory()) - #if not as_conf.check_conf_files(): - #return False - user_to = "cprodhom" - - if not migrate_experiment(exp_path, user_to): - Log.critical("The directory owner for {0} cannot be changed to {1}.", exp_path, user_to) - return False - return True - - @staticmethod def check(experiment_id): """ @@ -1483,7 +1478,7 @@ class Autosubmit: return True @staticmethod - def archive(expid): + def archive(expid, clean=True): """ Archives an experiment: call clean (if experiment is of version 3 or later), compress folder to tar.gz and moves to year's folder @@ -1498,14 +1493,15 @@ class Autosubmit: Log.warning("Does an experiment with the given id exist?") return 1 - Log.set_file(os.path.join(BasicConfig.LOCAL_ROOT_DIR, "ASlogs", 'archive{0}.log'.format(expid))) + Log.set_file(os.path.join(BasicConfig.LOCAL_ROOT_DIR, "ASlogs", 'archive_{0}.log'.format(expid))) exp_folder = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid) - # Cleaning to reduce file size. - version = get_autosubmit_version(expid) - if version is not None and version.startswith('3') and not Autosubmit.clean(expid, True, True, True, False): - Log.critical("Can not archive project. Clean not successful") - return False + if clean: + # Cleaning to reduce file size. + version = get_autosubmit_version(expid) + if version is not None and version.startswith('3') and not Autosubmit.clean(expid, True, True, True, False): + Log.critical("Can not archive project. Clean not successful") + return False # Getting year of last completed. If not, year of expid folder year = None @@ -1530,6 +1526,7 @@ class Autosubmit: with tarfile.open(os.path.join(year_path, '{0}.tar.gz'.format(expid)), "w:gz") as tar: tar.add(exp_folder, arcname='') tar.close() + os.chmod(os.path.join(year_path, '{0}.tar.gz'.format(expid)), 0o775) except Exception as e: Log.critical("Can not write tar file: {0}".format(e)) return False @@ -1555,7 +1552,7 @@ class Autosubmit: :type experiment_id: str """ BasicConfig.read() - Log.set_file(os.path.join(BasicConfig.LOCAL_ROOT_DIR, "ASlogs", 'unarchive{0}.log'.format(experiment_id))) + Log.set_file(os.path.join(BasicConfig.LOCAL_ROOT_DIR, "ASlogs", 'unarchive_{0}.log'.format(experiment_id))) exp_folder = os.path.join(BasicConfig.LOCAL_ROOT_DIR, experiment_id) if os.path.exists(exp_folder): diff --git a/autosubmit/config/config_common.py b/autosubmit/config/config_common.py index a35958de5..1e55223a0 100644 --- a/autosubmit/config/config_common.py +++ b/autosubmit/config/config_common.py @@ -902,7 +902,7 @@ class AutosubmitConfig(object): """ return self._conf_parser.get_option('storage', 'TYPE', 'pkl').lower() - def get_migrate_to(self): + def get_exp_migrate_to_user(self): """ Returns the user to change experiment's owner to from autosubmit's config file. diff --git a/autosubmit/experiment/experiment_common.py b/autosubmit/experiment/experiment_common.py index cb9291740..43e78191f 100644 --- a/autosubmit/experiment/experiment_common.py +++ b/autosubmit/experiment/experiment_common.py @@ -20,9 +20,6 @@ """ Module containing functions to manage autosubmit's experiments. """ -import os -import pwd -import glob import string import autosubmit.database.db_common as db_common from bscearth.utils.log import Log @@ -172,34 +169,3 @@ def base36decode(number): :rtype: int """ return int(number, 36) - - -def migrate_experiment(exp_path, user_to): - """ - Migrates experiment files from current to new user. - Group id will be kept the same. - - :param exp_path: experiment path: - :param user_to: to user: - """ - to_uid = pwd.getpwnam(user_to).pw_uid - Log.info("The UID for {0} is {1}.", user_to, to_uid) - current_gid = os.stat(exp_path).st_gid - Log.info("GID will be kept to {0}.", current_gid) - recursive_file_permissions(exp_path, to_uid, current_gid) - return user_to, current_gid - -def recursive_file_permissions(path,uid=-1,gid=-1): - """ - Recursively updates file permissions on a given path. - UID and GID default to -1 - """ - os.chown(path,uid,gid) - for item in glob.glob(path+'/*'): - if os.path.isdir(item): - recursive_file_permissions(os.path.join(path,item),uid,gid) - else: - try: - os.chown(os.path.join(path,item),uid,gid) - except: - Log.critical('File permissions on {0} not updated due to error.', os.path.join(path,item)) diff --git a/test/unit/test_autosubmit_config.py b/test/unit/test_autosubmit_config.py index 7729d4a30..c2965d839 100644 --- a/test/unit/test_autosubmit_config.py +++ b/test/unit/test_autosubmit_config.py @@ -136,16 +136,6 @@ class TestAutosubmitConfig(TestCase): # assert self._assert_get_option(parser_mock, 'MEMORY', expected_value, returned_value, default_value, str) - def test_get_user_to(self): - # arrange - expected_value = 'new_user' - default_value = '' - config, parser_mock = self._arrange_config(expected_value) - # act - returned_value = config.get_chown(self.section) - # assert - self._assert_get_option(parser_mock, 'USER_TO', expected_value, returned_value, default_value, str) - def test_that_reload_must_load_parsers(self): # arrange config = AutosubmitConfig(self.any_expid, FakeBasicConfig, ConfigParserFactory()) @@ -214,16 +204,6 @@ class TestAutosubmitConfig(TestCase): # assert open_mock.assert_any_call(getattr(config, '_conf_parser_file'), 'w') - def test_get_to_user(self): - # arrange - expected_value = 'new_user' - default_value = '' - config, parser_mock = self._arrange_config(expected_value) - # act - returned_value = config.get_to_user(self.section) - # assert - self._assert_get_option(parser_mock, 'TO_USER', expected_value, returned_value, default_value, str) - def test_load_project_parameters(self): # arrange parser_mock = Mock(spec=ConfigParser) -- GitLab From 5de6701683cade26639bd2c96c5f522c595d6eb4 Mon Sep 17 00:00:00 2001 From: Domingo Manubens-Gil Date: Fri, 30 Jun 2017 11:51:51 +0200 Subject: [PATCH 4/7] Progress on migrate exp --- autosubmit/autosubmit.py | 51 +++++++++++++++++----- autosubmit/config/config_common.py | 9 ---- autosubmit/config/files/autosubmit.conf | 2 +- autosubmit/config/files/platforms.conf | 2 + autosubmit/platforms/paramiko_submitter.py | 2 + autosubmit/platforms/platform.py | 2 + 6 files changed, 48 insertions(+), 20 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 0a545d9ec..1204aa5e4 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -73,8 +73,6 @@ from bscearth.utils.log import Log from database.db_common import create_db from experiment.experiment_common import new_experiment from experiment.experiment_common import copy_experiment -from experiment.experiment_common import migrate_experiment_offer -from experiment.experiment_common import migrate_experiment_pickup from database.db_common import delete_experiment from database.db_common import get_autosubmit_version from monitor.monitor import Monitor @@ -1071,18 +1069,49 @@ class Autosubmit: It takes mapping information for new user from config files. :param experiment_id: experiment identifier: + :param pickup: + :param offer: """ + log_file = os.path.join(BasicConfig.LOCAL_ROOT_DIR, "ASlogs", 'migrate_{0}.log'.format(experiment_id)) + Log.set_file(log_file) + as_conf = AutosubmitConfig(experiment_id, BasicConfig, ConfigParserFactory()) + if not as_conf.check_conf_files(): + Log.critical('Can not run with invalid configuration') + return False + + Log.info('Migrating experiment {0}'.format(experiment_id)) + submitter = Autosubmit._get_submitter(as_conf) + submitter.load_platforms(as_conf) + if submitter.platforms is None: + return False + + Log.info("Checking remote platforms") + platforms = filter(lambda x: x not in ['local', 'LOCAL'], submitter.platforms) + if offer: - Autosubmit.archive(experiment_id, False) - log_file = os.path.join(BasicConfig.LOCAL_ROOT_DIR, "ASlogs", 'migrate_{0}.log'.format(experiment_id)) - Log.set_file(log_file) - Log.result("The experiment has been successfully offered.") + Log.info("Moving remote files/dirs") + for platform in platforms: + p = submitter.platforms[platform] + Log.info("Moving from {0} to {1}", os.path.join(p.root_dir), + os.path.join(p.temp_dir, experiment_id)) + p.move_file(os.path.join(p.root_dir), os.path.join(p.temp_dir, experiment_id)) + Log.result("Files/dirs on {0} have been successfully offered", platform) + + Log.info("Updating configuration with target user/project") + as_conf._conf_parser.get_option('migrate', 'TO_USER', '').lower() + as_conf.check_platforms_conf() + content = open(as_conf._conf_parser_file).read() + content = content.replace(re.search('SAFETYSLEEPTIME =.*', content).group(0), + "SAFETYSLEEPTIME = %d" % sleep_time) + open(self._conf_parser_file, 'w').write(content) + + Log.info("Moving local files/dirs") + Autosubmit.archive(experiment_id, False) + Log.result("The experiment has been successfully offered.") elif pickup: - Autosubmit.unarchive(experiment_id) - log_file = os.path.join(BasicConfig.LOCAL_ROOT_DIR, "ASlogs", 'migrate_{0}.log'.format(experiment_id)) - Log.set_file(log_file) - Log.result("The experiment has been successfully picked up.") + Autosubmit.unarchive(experiment_id) + Log.result("The experiment has been successfully picked up.") return True @@ -1483,6 +1512,8 @@ class Autosubmit: Archives an experiment: call clean (if experiment is of version 3 or later), compress folder to tar.gz and moves to year's folder + :param clean: + :return: :param expid: experiment identifier :type expid: str """ diff --git a/autosubmit/config/config_common.py b/autosubmit/config/config_common.py index 1e55223a0..70356b200 100644 --- a/autosubmit/config/config_common.py +++ b/autosubmit/config/config_common.py @@ -902,15 +902,6 @@ class AutosubmitConfig(object): """ return self._conf_parser.get_option('storage', 'TYPE', 'pkl').lower() - def get_exp_migrate_to_user(self): - """ - Returns the user to change experiment's owner to from autosubmit's config file. - - :return: migrate to user - :rtype: str - """ - return self._conf_parser.get_option('migrate', 'TO_USER', '').lower() - @staticmethod def is_valid_mail_address(mail_address): if re.match('^[_a-z0-9-]+(\.[_a-z0-9-]+)*@[a-z0-9-]+(\.[a-z0-9-]+)*(\.[a-z]{2,4})$', mail_address): diff --git a/autosubmit/config/files/autosubmit.conf b/autosubmit/config/files/autosubmit.conf index 6bd2c4169..90770b8d0 100644 --- a/autosubmit/config/files/autosubmit.conf +++ b/autosubmit/config/files/autosubmit.conf @@ -38,5 +38,5 @@ TYPE = pkl COPY_REMOTE_LOGS = True [migrate] -# Changes owner of experiment files. +# Changes experiment files owner. TO_USER = diff --git a/autosubmit/config/files/platforms.conf b/autosubmit/config/files/platforms.conf index b2f39fd59..70d628948 100644 --- a/autosubmit/config/files/platforms.conf +++ b/autosubmit/config/files/platforms.conf @@ -20,6 +20,8 @@ # USER_TO = ## Path to the scratch directory for the machine. Required. # SCRATCH_DIR = /scratch +## Path to the tmp directory for the machine. +# TMP_DIR = /tmp ## If true, Autosubmit test command can use this queue as a main queue. Defaults to False # TEST_SUITE = False ## If given, Autosubmit will add jobs to the given queue. Required for some platforms. diff --git a/autosubmit/platforms/paramiko_submitter.py b/autosubmit/platforms/paramiko_submitter.py index cc6599118..9b8ed504e 100644 --- a/autosubmit/platforms/paramiko_submitter.py +++ b/autosubmit/platforms/paramiko_submitter.py @@ -69,6 +69,7 @@ class ParamikoSubmitter(Submitter): local_platform.max_waiting_jobs = asconf.get_max_waiting_jobs() local_platform.total_jobs = asconf.get_total_jobs() local_platform.scratch = os.path.join(BasicConfig.LOCAL_ROOT_DIR, asconf.expid, BasicConfig.LOCAL_TMP_DIR) + local_platform.temp_dir = os.path.join(BasicConfig.LOCAL_ROOT_DIR, 'ASlogs') local_platform.root_dir = os.path.join(BasicConfig.LOCAL_ROOT_DIR, local_platform.expid) local_platform.host = 'localhost' platforms['local'] = local_platform @@ -127,6 +128,7 @@ class ParamikoSubmitter(Submitter): remote_platform.exclusivity = parser.get_option(section, 'EXCLUSIVITY', '').lower() remote_platform.user = parser.get_option(section, 'USER', None) remote_platform.scratch = parser.get_option(section, 'SCRATCH_DIR', None) + remote_platform.temp_dir = parser.get_option(section, 'TEMP_DIR', None) remote_platform._default_queue = parser.get_option(section, 'QUEUE', None) remote_platform._serial_queue = parser.get_option(section, 'SERIAL_QUEUE', None) remote_platform.processors_per_node = parser.get_option(section, 'PROCESSORS_PER_NODE', diff --git a/autosubmit/platforms/platform.py b/autosubmit/platforms/platform.py index 50334edaa..3d5cafd10 100644 --- a/autosubmit/platforms/platform.py +++ b/autosubmit/platforms/platform.py @@ -35,6 +35,7 @@ class Platform(object): self.exclusivity = '' self.type = '' self.scratch = '' + self.temp_dir = '' self.root_dir = '' self.service = None self.scheduler = None @@ -128,6 +129,7 @@ class Platform(object): parameters['{0}EXCLUSIVITY'.format(prefix)] = self.exclusivity parameters['{0}TYPE'.format(prefix)] = self.type parameters['{0}SCRATCH_DIR'.format(prefix)] = self.scratch + parameters['{0}TEMP_DIR'.format(prefix)] = self.temp_dir parameters['{0}ROOTDIR'.format(prefix)] = self.root_dir parameters['{0}LOGDIR'.format(prefix)] = self.get_files_path() -- GitLab From 8d912d188778dab9f7b0d96478f04f32d4e34aaa Mon Sep 17 00:00:00 2001 From: Domingo Manubens-Gil Date: Tue, 11 Jul 2017 19:43:05 +0200 Subject: [PATCH 5/7] Minor. Fix previous commit. --- autosubmit/autosubmit.py | 76 +++++++++++++++++++------- autosubmit/config/config_common.py | 21 +++++++ autosubmit/config/files/platforms.conf | 4 +- 3 files changed, 78 insertions(+), 23 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 1204aa5e4..25d56c5f5 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -73,6 +73,8 @@ from bscearth.utils.log import Log from database.db_common import create_db from experiment.experiment_common import new_experiment from experiment.experiment_common import copy_experiment +from experiment.experiment_common import migrate_experiment_offer +from experiment.experiment_common import migrate_experiment_pickup from database.db_common import delete_experiment from database.db_common import get_autosubmit_version from monitor.monitor import Monitor @@ -1074,43 +1076,75 @@ class Autosubmit: """ log_file = os.path.join(BasicConfig.LOCAL_ROOT_DIR, "ASlogs", 'migrate_{0}.log'.format(experiment_id)) Log.set_file(log_file) - as_conf = AutosubmitConfig(experiment_id, BasicConfig, ConfigParserFactory()) - if not as_conf.check_conf_files(): - Log.critical('Can not run with invalid configuration') - return False - Log.info('Migrating experiment {0}'.format(experiment_id)) - submitter = Autosubmit._get_submitter(as_conf) - submitter.load_platforms(as_conf) - if submitter.platforms is None: - return False + if offer: + Log.info('Migrating experiment {0}'.format(experiment_id)) + as_conf = AutosubmitConfig(experiment_id, BasicConfig, ConfigParserFactory()) + if not as_conf.check_conf_files(): + Log.critical('Can not proceed with invalid configuration') + return False - Log.info("Checking remote platforms") - platforms = filter(lambda x: x not in ['local', 'LOCAL'], submitter.platforms) + submitter = Autosubmit._get_submitter(as_conf) + submitter.load_platforms(as_conf) + if submitter.platforms is None: + return False - if offer: + Log.info("Checking remote platforms") + platforms = filter(lambda x: x not in ['local', 'LOCAL'], submitter.platforms) Log.info("Moving remote files/dirs") for platform in platforms: + Log.info("Updating platform configuration with target user") + if not as_conf.get_migrate_user_to(platform): + Log.critical("Missing target user in platforms configuration file") + return False + + as_conf.set_new_user(platform, as_conf.get_migrate_user_to(platform)) + Log.info("User in platform configuration file successfully updated to {0}", + as_conf.get_migrate_user_to(platform)) + p = submitter.platforms[platform] Log.info("Moving from {0} to {1}", os.path.join(p.root_dir), os.path.join(p.temp_dir, experiment_id)) - p.move_file(os.path.join(p.root_dir), os.path.join(p.temp_dir, experiment_id)) + if not p.move_file(os.path.join(p.root_dir), os.path.join(p.temp_dir, experiment_id)): + Log.critical("The files/dirs on {0} cannot be moved to {1}.", p.root_dir, + os.path.join(p.temp_dir, experiment_id)) + return False Log.result("Files/dirs on {0} have been successfully offered", platform) - Log.info("Updating configuration with target user/project") - as_conf._conf_parser.get_option('migrate', 'TO_USER', '').lower() - as_conf.check_platforms_conf() - content = open(as_conf._conf_parser_file).read() - content = content.replace(re.search('SAFETYSLEEPTIME =.*', content).group(0), - "SAFETYSLEEPTIME = %d" % sleep_time) - open(self._conf_parser_file, 'w').write(content) - Log.info("Moving local files/dirs") Autosubmit.archive(experiment_id, False) Log.result("The experiment has been successfully offered.") elif pickup: + Log.info('Migrating experiment {0}'.format(experiment_id)) + Log.info("Moving local files/dirs") Autosubmit.unarchive(experiment_id) + Log.info("Local files/dirs have been sucessfully picked up") + as_conf = AutosubmitConfig(experiment_id, BasicConfig, ConfigParserFactory()) + if not as_conf.check_conf_files(): + Log.critical('Can not proceed with invalid configuration') + return False + + submitter = Autosubmit._get_submitter(as_conf) + submitter.load_platforms(as_conf) + if submitter.platforms is None: + return False + + Log.info("Checking remote platforms") + platforms = filter(lambda x: x not in ['local', 'LOCAL'], submitter.platforms) + Log.info("Copying remote files/dirs") + for platform in platforms: + p = submitter.platforms[platform] + Log.info("Copying from {0} to {1}", os.path.join(p.temp_dir, experiment_id), + os.path.join(p.root_dir)) + if not p.send_command("cp -r " + os.path.join(p.temp_dir, experiment_id) + " " + + os.path.join(p.root_dir)): + Log.critical("The files/dirs on {0} cannot be copied to {1}.", + os.path.join(p.temp_dir, experiment_id), p.root_dir) + return False + + Log.result("Files/dirs on {0} have been successfully picked up", platform) + Log.result("The experiment has been successfully picked up.") return True diff --git a/autosubmit/config/config_common.py b/autosubmit/config/config_common.py index 70356b200..3cb15cc11 100644 --- a/autosubmit/config/config_common.py +++ b/autosubmit/config/config_common.py @@ -204,6 +204,27 @@ class AutosubmitConfig(object): """ return str(self._jobs_parser.get_option(section, 'MEMORY_PER_TASK', '')) + def get_migrate_user_to(self, section): + """ + Returns the user to change to from platform config file. + + :return: migrate user to + :rtype: str + """ + return self._platforms_parser.get_option(section, 'USER_TO', '').lower() + + def set_new_user(self, section, new_user): + """ + Sets new user for given platform + :param new_user: + :param section: platform name + :type: str + """ + content = open(self._platforms_parser_file).read() + if re.search(section, content): + content = content.replace(re.search('USER =.*', content).group(0), "USER = " + new_user) + open(self._platforms_parser_file, 'w').write(content) + def check_conf_files(self): """ Checks configuration files (autosubmit, experiment jobs and platforms), looking for invalid values, missing diff --git a/autosubmit/config/files/platforms.conf b/autosubmit/config/files/platforms.conf index 70d628948..608357c3f 100644 --- a/autosubmit/config/files/platforms.conf +++ b/autosubmit/config/files/platforms.conf @@ -20,8 +20,8 @@ # USER_TO = ## Path to the scratch directory for the machine. Required. # SCRATCH_DIR = /scratch -## Path to the tmp directory for the machine. -# TMP_DIR = /tmp +## Path to the machine's temporary directory for migrate purposes. +# TEMP_DIR = /tmp ## If true, Autosubmit test command can use this queue as a main queue. Defaults to False # TEST_SUITE = False ## If given, Autosubmit will add jobs to the given queue. Required for some platforms. -- GitLab From 5d4902322dc1fd975592b2e6ef37acedf9cf92d7 Mon Sep 17 00:00:00 2001 From: Domingo Manubens-Gil Date: Wed, 12 Jul 2017 18:46:21 +0200 Subject: [PATCH 6/7] Add remote files migration support. Fixes #70. Minor change tmp directory chmod to 775. --- autosubmit/autosubmit.py | 31 +++++++++++++++------ autosubmit/config/config_common.py | 21 ++++++++++++++ test/regression/default_conf/platforms.conf | 7 +++-- 3 files changed, 47 insertions(+), 12 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 25d56c5f5..bf434b4ef 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -73,8 +73,6 @@ from bscearth.utils.log import Log from database.db_common import create_db from experiment.experiment_common import new_experiment from experiment.experiment_common import copy_experiment -from experiment.experiment_common import migrate_experiment_offer -from experiment.experiment_common import migrate_experiment_pickup from database.db_common import delete_experiment from database.db_common import get_autosubmit_version from monitor.monitor import Monitor @@ -511,7 +509,8 @@ class Autosubmit: Log.debug("Creating temporal directory...") exp_id_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id) - os.mkdir(os.path.join(exp_id_path, "tmp"), 0o775) + os.mkdir(os.path.join(exp_id_path, "tmp")) + os.chmod(os.path.join(exp_id_path, "tmp"), 0o775) Log.debug("Creating pkl directory...") os.mkdir(os.path.join(exp_id_path, "pkl")) @@ -1091,9 +1090,8 @@ class Autosubmit: Log.info("Checking remote platforms") platforms = filter(lambda x: x not in ['local', 'LOCAL'], submitter.platforms) - Log.info("Moving remote files/dirs") for platform in platforms: - Log.info("Updating platform configuration with target user") + Log.info("Updating {0} platform configuration with target user", platform) if not as_conf.get_migrate_user_to(platform): Log.critical("Missing target user in platforms configuration file") return False @@ -1102,6 +1100,15 @@ class Autosubmit: Log.info("User in platform configuration file successfully updated to {0}", as_conf.get_migrate_user_to(platform)) + if as_conf.get_migrate_project_to(platform): + Log.info("Updating {0} platform configuration with target project", platform) + as_conf.set_new_project(platform, as_conf.get_migrate_project_to(platform)) + Log.info("Project in platform configuration file successfully updated to {0}", + as_conf.get_migrate_user_to(platform)) + else: + Log.warning("Project in platforms configuration file remains unchanged") + + Log.info("Moving remote files/dirs on {0}", platform) p = submitter.platforms[platform] Log.info("Moving from {0} to {1}", os.path.join(p.root_dir), os.path.join(p.temp_dir, experiment_id)) @@ -1109,31 +1116,37 @@ class Autosubmit: Log.critical("The files/dirs on {0} cannot be moved to {1}.", p.root_dir, os.path.join(p.temp_dir, experiment_id)) return False + Log.result("Files/dirs on {0} have been successfully offered", platform) Log.info("Moving local files/dirs") - Autosubmit.archive(experiment_id, False) + if not Autosubmit.archive(experiment_id, False): + Log.critical("The experiment cannot be offered") + return False + Log.result("The experiment has been successfully offered.") elif pickup: Log.info('Migrating experiment {0}'.format(experiment_id)) Log.info("Moving local files/dirs") - Autosubmit.unarchive(experiment_id) + if not Autosubmit.unarchive(experiment_id): + Log.critical("The experiment cannot be picked up") + return False Log.info("Local files/dirs have been sucessfully picked up") as_conf = AutosubmitConfig(experiment_id, BasicConfig, ConfigParserFactory()) if not as_conf.check_conf_files(): Log.critical('Can not proceed with invalid configuration') return False + Log.info("Checking remote platforms") submitter = Autosubmit._get_submitter(as_conf) submitter.load_platforms(as_conf) if submitter.platforms is None: return False - Log.info("Checking remote platforms") platforms = filter(lambda x: x not in ['local', 'LOCAL'], submitter.platforms) - Log.info("Copying remote files/dirs") for platform in platforms: + Log.info("Copying remote files/dirs on {0}", platform) p = submitter.platforms[platform] Log.info("Copying from {0} to {1}", os.path.join(p.temp_dir, experiment_id), os.path.join(p.root_dir)) diff --git a/autosubmit/config/config_common.py b/autosubmit/config/config_common.py index 3cb15cc11..e64e43a67 100644 --- a/autosubmit/config/config_common.py +++ b/autosubmit/config/config_common.py @@ -225,6 +225,27 @@ class AutosubmitConfig(object): content = content.replace(re.search('USER =.*', content).group(0), "USER = " + new_user) open(self._platforms_parser_file, 'w').write(content) + def get_migrate_project_to(self, section): + """ + Returns the project to change to from platform config file. + + :return: migrate project to + :rtype: str + """ + return self._platforms_parser.get_option(section, 'PROJECT_TO', '').lower() + + def set_new_project(self, section, new_project): + """ + Sets new project for given platform + :param new_project: + :param section: platform name + :type: str + """ + content = open(self._platforms_parser_file).read() + if re.search(section, content): + content = content.replace(re.search('PROJECT =.*', content).group(0), "PROJECT = " + new_project) + open(self._platforms_parser_file, 'w').write(content) + def check_conf_files(self): """ Checks configuration files (autosubmit, experiment jobs and platforms), looking for invalid values, missing diff --git a/test/regression/default_conf/platforms.conf b/test/regression/default_conf/platforms.conf index 8a3a3c058..ae146bd42 100644 --- a/test/regression/default_conf/platforms.conf +++ b/test/regression/default_conf/platforms.conf @@ -31,15 +31,16 @@ TEST_SUITE = False QUEUE = serial [marenostrum3] -TYPE = LSF +TYPE = slurm VERSION = mn HOST = mn-bsc32 PROJECT = bsc32 +QUEUE = debug ADD_PROJECT_TO_HOST = false -USER = bsc32649 +USER = bsc32704 SCRATCH_DIR = /gpfs/scratch TEST_SUITE = True -PROCESSORS_PER_NODE = 16 +PROCESSORS_PER_NODE = 48 [mistral] TYPE = slurm -- GitLab From 8d630ad3d1efb41c636f0826283e3b9c0ba87b78 Mon Sep 17 00:00:00 2001 From: Domingo Manubens-Gil Date: Fri, 4 Aug 2017 18:18:17 +0200 Subject: [PATCH 7/7] Remove test_migrate_exp, not properly written. Add documentation. Closes #70 --- autosubmit/config/files/platforms.conf | 2 +- docs/source/usage.rst | 29 ++++++++++++++++++++ test/unit/test_migrate_exp.py | 38 -------------------------- 3 files changed, 30 insertions(+), 39 deletions(-) delete mode 100644 test/unit/test_migrate_exp.py diff --git a/autosubmit/config/files/platforms.conf b/autosubmit/config/files/platforms.conf index 608357c3f..45ab24753 100644 --- a/autosubmit/config/files/platforms.conf +++ b/autosubmit/config/files/platforms.conf @@ -16,7 +16,7 @@ # ADD_PROJECT_TO_HOST = False ## User for the machine scheduler. Required # USER = -## Optional. If given, Autosubmit will change owner of files in given platform. +## Optional. If given, Autosubmit will change owner of files in given platform when using migrate_exp. # USER_TO = ## Path to the scratch directory for the machine. Required. # SCRATCH_DIR = /scratch diff --git a/docs/source/usage.rst b/docs/source/usage.rst index a90c07aeb..82b93c199 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -23,6 +23,7 @@ Command list -install Install database for Autosubmit on the configured folder -archive Clean, compress and remove from the experiments' folder a finalized experiment -unarchive Restores an archived experiment +-migrate_exp Migrates an experiment from one user to another How to create an experiment @@ -828,6 +829,34 @@ Example: autosubmit unarchive cxxx +How to migrate an experiment +============================ +To migrate an experiment from one user to another, you need to add two parameters in the platforms configuration file: + + * USER_TO = + * TEMP_DIR = + +Then, just run the command: +:: + + autosubmit migrate_exp --ofer expid + + +Local files will be archived and remote files put in the HPC temporary directory. + +.. warning:: The temporary directory must be readable by both users (old owner and new owner). + +Then the new owner will have to run the command: +:: + + autosubmit migrate_exp --pickup expid + + + +Local files will be unarchived and remote files copied from the temporal loaction. + +.. warning:: The old owner might need to remove temporal files and archive. + How to configure email notifications ==================================== diff --git a/test/unit/test_migrate_exp.py b/test/unit/test_migrate_exp.py deleted file mode 100644 index eeeb1132a..000000000 --- a/test/unit/test_migrate_exp.py +++ /dev/null @@ -1,38 +0,0 @@ -from unittest import TestCase -from mock import Mock, patch -from autosubmit.experiment.experiment_common import migrate_experiment -import os -import pwd - - -class TestMigrateExp(TestCase): - def setUp(self): - #self.user_from = "old-user" - self.user_from = "dmanuben" - #self.user_to = "new-user" - self.user_to = "dmanuben" - -# def testFoo(self): -# self.failUnless(False) - - @patch('autosubmit.experiment.experiment_common.os') - def test_migrate_experiment(self, mock_os): - current_user_id = "old-user" - user_id, group_id = migrate_experiment("any path", self.user_to) - - to_uid = pwd.getpwnam(self.user_to).pw_uid - mock_os.chown.assert_called_with("any path", to_uid, group_id) - #self.assertEquals("new_user", user_id) - -# @patch('autosubmit.experiment.experiment_common.db_common') -# def test_create_new_experiment(self, db_common_mock): -# current_experiment_id = "empty" -# self._build_db_mock(current_experiment_id, db_common_mock) -# experiment_id = new_experiment(self.description, self.version) -# self.assertEquals("a000", experiment_id) - -# -# @staticmethod -# def _build_db_mock(current_experiment_id, mock_db_common): -# mock_db_common.last_name_used = Mock(return_value=current_experiment_id) -# mock_db_common.check_experiment_exists = Mock(return_value=False) -- GitLab