From 3ba54a69a36fc739adc549c50522132467547ef9 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Wed, 29 Nov 2023 13:31:33 +0100 Subject: [PATCH] Disable submodules via configuration option set to false --- autosubmit/git/autosubmit_git.py | 55 +++---- .../set_and_share_the_configuration/index.rst | 5 + test/unit/gui/__init__.py | 0 test/unit/gui/test_autosubmit_git.py | 134 ++++++++++++++++++ 4 files changed, 168 insertions(+), 26 deletions(-) create mode 100644 test/unit/gui/__init__.py create mode 100644 test/unit/gui/test_autosubmit_git.py diff --git a/autosubmit/git/autosubmit_git.py b/autosubmit/git/autosubmit_git.py index 976b02f9f..727506447 100644 --- a/autosubmit/git/autosubmit_git.py +++ b/autosubmit/git/autosubmit_git.py @@ -14,18 +14,22 @@ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. +from os import path + # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . import locale -from os import path import os -from shutil import rmtree -import subprocess import shutil +import subprocess +from shutil import rmtree +from time import time +from typing import List, Union + # from autosubmit import Autosubmit from autosubmitconfigparser.config.basicconfig import BasicConfig -from time import time -from log.log import Log, AutosubmitCritical, AutosubmitError +from log.log import Log, AutosubmitCritical + Log.get_logger("Autosubmit") @@ -141,10 +145,8 @@ class AutosubmitGit: git_project_branch = as_conf.get_git_project_branch() git_remote_project_path = as_conf.get_git_remote_project_root() - if git_project_branch == '': - git_project_branch = 'master' git_project_commit = as_conf.get_git_project_commit() - git_project_submodules = as_conf.get_submodules_list() + git_project_submodules: Union[False, List[str]] = as_conf.get_submodules_list() git_project_submodules_depth = as_conf.get_project_submodules_depth() max_depth = -1 if len(git_project_submodules_depth) > 0: @@ -235,27 +237,28 @@ class AutosubmitGit: else: command_1 += "git checkout; " - if git_project_submodules.__len__() <= 0: - if max_depth > 0: - Log.info("Depth is incompatible with --recursive, ignoring recursive option") - command_1 += " git submodule update --init --depth {0}; ".format(max_depth) - else: - command_1 += " git submodule update --init --recursive; " - else: - command_1 += " git submodule init; ".format(project_destination) - index_submodule = 0 - for submodule in git_project_submodules: + if git_project_submodules is not False: + if len(git_project_submodules) == 0: if max_depth > 0: Log.info("Depth is incompatible with --recursive, ignoring recursive option") - if index_submodule < len(git_project_submodules_depth): - command_1 += " git submodule update --init --depth {0} {1}; ".format( - git_project_submodules_depth[index_submodule], submodule) - else: - command_1 += " git submodule update --init --depth {0} {1}; ".format( - max_depth, submodule) + command_1 += " git submodule update --init --depth {0}; ".format(max_depth) else: - command_1 += " git submodule update --init --recursive {0}; ".format(submodule) - index_submodule += 1 + command_1 += " git submodule update --init --recursive; " + else: + command_1 += " git submodule init; ".format(project_destination) + index_submodule = 0 + for submodule in git_project_submodules: + if max_depth > 0: + Log.info("Depth is incompatible with --recursive, ignoring recursive option") + if index_submodule < len(git_project_submodules_depth): + command_1 += " git submodule update --init --depth {0} {1}; ".format( + git_project_submodules_depth[index_submodule], submodule) + else: + command_1 += " git submodule update --init --depth {0} {1}; ".format( + max_depth, submodule) + else: + command_1 += " git submodule update --init --recursive {0}; ".format(submodule) + index_submodule += 1 if git_remote_project_path == '': try: if len(command_githook) > 0: diff --git a/docs/source/userguide/set_and_share_the_configuration/index.rst b/docs/source/userguide/set_and_share_the_configuration/index.rst index e0d0c8139..0088b052b 100644 --- a/docs/source/userguide/set_and_share_the_configuration/index.rst +++ b/docs/source/userguide/set_and_share_the_configuration/index.rst @@ -148,6 +148,11 @@ Edit or generate a `minimal.yml` with the following parameters, leaving the rest .. important:: The final configuration will be loaded in the following order: `PRE`:`$expid/%PROJDIR%/$as_proj_config_path` -> `$expid/conf` -> `POST`. Overwriting the parameters in the order they are loaded. +.. note:: + + ``PROJECT_SUBMODULES`` can be set to ``false`` (without quotes), which will be + evaluated as ``False`` in Python, and will disable the Git submodules (i.e. no + submodules will be cloned). CUSTOM_CONFIG: Syntax ^^^^^^^^^^^^^^^^^^^^^ diff --git a/test/unit/gui/__init__.py b/test/unit/gui/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/unit/gui/test_autosubmit_git.py b/test/unit/gui/test_autosubmit_git.py new file mode 100644 index 000000000..f70d11f29 --- /dev/null +++ b/test/unit/gui/test_autosubmit_git.py @@ -0,0 +1,134 @@ +from unittest import TestCase + +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import MagicMock, patch + +from autosubmit.git.autosubmit_git import AutosubmitGit +from autosubmitconfigparser.config.configcommon import AutosubmitConfig +from log.log import AutosubmitCritical + + +class TestAutosubmitGit(TestCase): + """Tests for ``AutosubmitGit``.""" + MockBasicConfig = MagicMock() + mock_subprocess = MagicMock() + + def setUp(self) -> None: + self.expid = 'a000' + self.temp_dir = TemporaryDirectory() + self.exp_dir = Path(self.temp_dir.name, f'{self.expid}') + self.conf_dir = self.exp_dir / 'conf' + self.conf_dir.mkdir(parents=True) + self.MockBasicConfig.LOCAL_ROOT_DIR = self.temp_dir.name + self.MockBasicConfig.LOCAL_PROJ_DIR = self.exp_dir / 'proj' + self.MockBasicConfig.LOCAL_PROJ_DIR.mkdir(parents=True) + + self.hpcarch = MagicMock() + + def mocked_git_subprocess(*args, **kwargs): + if args[0] == 'git --version': + return "2251" + + self.mock_subprocess.check_output.side_effect = mocked_git_subprocess + + def tearDown(self) -> None: + if self.temp_dir is not None: + self.temp_dir.cleanup() + + def test_submodules_fails_with_invalid_as_conf(self): + as_conf = MagicMock() + as_conf.is_valid_git_repository.return_value = False + with self.assertRaises(AutosubmitCritical): + AutosubmitGit.clone_repository(as_conf=as_conf, force=True, hpcarch=self.hpcarch) + + @patch('autosubmit.git.autosubmit_git.BasicConfig', new=MockBasicConfig) + @patch('autosubmitconfigparser.config.basicconfig.BasicConfig', new=MockBasicConfig) + @patch('autosubmitconfigparser.config.configcommon.BasicConfig', new=MockBasicConfig) + @patch('autosubmit.git.autosubmit_git.subprocess', new=mock_subprocess) + def test_submodules_empty_string(self): + """Verifies that submodules configuration is processed correctly with empty strings.""" + _as_conf = AutosubmitConfig(expid=self.expid, basic_config=self.MockBasicConfig) + _as_conf.experiment_data = { + 'GIT': { + 'PROJECT_ORIGIN': 'https://earth.bsc.es/gitlab/es/autosubmit.git', + 'PROJECT_BRANCH': 'master', + 'PROJECT_COMMIT': '123', + 'REMOTE_CLONE_ROOT': 'workflow', + 'PROJECT_SUBMODULES': '' + } + } + as_conf = MagicMock(wraps=_as_conf) + as_conf.is_valid_git_repository.return_value = True + as_conf.expid = self.expid + + force = False + AutosubmitGit.clone_repository(as_conf=as_conf, force=force, hpcarch=self.hpcarch) + + # Should be the last command, but to make sure we iterate all the commands. + # A good improvement would have to break the function called into smaller + # parts, like ``get_git_version``, ``clone_submodules(recursive=True)``, etc. + # as that would be a lot easier to test. + recursive_in_any_call = any([call for call in self.hpcarch.method_calls if + 'git submodule update --init --recursive' in str(call)]) + + assert recursive_in_any_call + + @patch('autosubmit.git.autosubmit_git.BasicConfig', new=MockBasicConfig) + @patch('autosubmitconfigparser.config.basicconfig.BasicConfig', new=MockBasicConfig) + @patch('autosubmitconfigparser.config.configcommon.BasicConfig', new=MockBasicConfig) + @patch('autosubmit.git.autosubmit_git.subprocess', new=mock_subprocess) + def test_submodules_list_not_empty(self): + """Verifies that submodules configuration is processed correctly with a list of strings.""" + _as_conf = AutosubmitConfig(expid=self.expid, basic_config=self.MockBasicConfig) + _as_conf.experiment_data = { + 'GIT': { + 'PROJECT_ORIGIN': 'https://earth.bsc.es/gitlab/es/autosubmit.git', + 'PROJECT_BRANCH': '', + 'PROJECT_COMMIT': '123', + 'REMOTE_CLONE_ROOT': 'workflow', + 'PROJECT_SUBMODULES': 'clone_me_a clone_me_b' + } + } + as_conf = MagicMock(wraps=_as_conf) + as_conf.is_valid_git_repository.return_value = True + as_conf.expid = self.expid + + force = False + AutosubmitGit.clone_repository(as_conf=as_conf, force=force, hpcarch=self.hpcarch) + + # Here the call happens in the hpcarch, not in subprocess + clone_me_a_in_any_call = any([call for call in self.hpcarch.method_calls if + 'clone_me_a' in str(call)]) + + assert clone_me_a_in_any_call + + @patch('autosubmit.git.autosubmit_git.BasicConfig', new=MockBasicConfig) + @patch('autosubmitconfigparser.config.basicconfig.BasicConfig', new=MockBasicConfig) + @patch('autosubmitconfigparser.config.configcommon.BasicConfig', new=MockBasicConfig) + @patch('autosubmit.git.autosubmit_git.subprocess', new=mock_subprocess) + def test_submodules_falsey_disables_submodules(self): + """Verifies that submodules are not used when users pass a Falsey bool value.""" + _as_conf = AutosubmitConfig(expid=self.expid, basic_config=self.MockBasicConfig) + _as_conf.experiment_data = { + 'GIT': { + 'PROJECT_ORIGIN': 'https://earth.bsc.es/gitlab/es/autosubmit.git', + 'PROJECT_BRANCH': '', + 'PROJECT_COMMIT': '123', + 'REMOTE_CLONE_ROOT': 'workflow', + 'PROJECT_SUBMODULES': False + } + } + as_conf = MagicMock(wraps=_as_conf) + as_conf.is_valid_git_repository.return_value = True + as_conf.expid = self.expid + + force = False + AutosubmitGit.clone_repository(as_conf=as_conf, force=force, hpcarch=self.hpcarch) + + # Because we have ``PROJECT_SUBMODULES: False``, there must be no calls + # to git submodules. + any_one_used_submodules = any([call for call in self.hpcarch.method_calls if + 'submodules' in str(call)]) + + assert not any_one_used_submodules -- GitLab