From e021ae092dcd22fb8988ce38d4fa3d441b14fed0 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 25 Jul 2019 09:21:32 +0200 Subject: [PATCH 1/3] --single_branch implementation , #419 --- autosubmit/config/config_common.py | 9 +++++++- autosubmit/config/files/expdef.conf | 3 +++ autosubmit/git/autosubmit_git.py | 32 +++++++++++++++++++++-------- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/autosubmit/config/config_common.py b/autosubmit/config/config_common.py index 3f872f17b..0e5426958 100644 --- a/autosubmit/config/config_common.py +++ b/autosubmit/config/config_common.py @@ -675,7 +675,14 @@ class AutosubmitConfig(object): """ return ' '.join(self._exp_parser.get_option('git', 'PROJECT_SUBMODULES','').split()).split() - + def get_fetch_single_branch(self): + """ + Returns fetch single branch from experiment's config file + Default is -single-branch + :return: fetch_single_branch(Y/N) + :rtype: boolean + """ + return self._exp_parser.get_option('git', 'FETCH_SINGLE_BRANCH', 'True').lower() def get_project_destination(self): """ Returns git commit from experiment's config file diff --git a/autosubmit/config/files/expdef.conf b/autosubmit/config/files/expdef.conf index 96e2b6fd3..0cf6ab6ce 100644 --- a/autosubmit/config/files/expdef.conf +++ b/autosubmit/config/files/expdef.conf @@ -48,6 +48,9 @@ PROJECT_BRANCH = PROJECT_COMMIT = # type = STRING, default = leave empty and will load all submodules, help = loadThisSubmodule alsoloadthis anotherLoad ... PROJECT_SUBMODULES = +# type = STRING, default = leave empty and will clone only a single branch, help = true,false +FETCH_SINGLE_BRANCH = true + # If PROJECT_TYPE is not svn, no need to change [svn] # type = STRING, help = 'https://svn.ec-earth.org/ecearth3' diff --git a/autosubmit/git/autosubmit_git.py b/autosubmit/git/autosubmit_git.py index 594642c0d..253d4fd99 100644 --- a/autosubmit/git/autosubmit_git.py +++ b/autosubmit/git/autosubmit_git.py @@ -129,6 +129,11 @@ class AutosubmitGit: git_project_branch = as_conf.get_git_project_branch() git_project_commit = as_conf.get_git_project_commit() git_project_submodules = as_conf.get_submodules_list() + if as_conf.get_fetch_single_branch() != "true": + git_single_branch = False + else: + git_single_branch = True + project_destination = as_conf.get_project_destination() project_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, as_conf.expid, BasicConfig.LOCAL_PROJ_DIR) git_path = as_conf.get_project_dir() @@ -147,12 +152,12 @@ class AutosubmitGit: Log.info("Fetching {0} into {1}", git_project_commit + " " + git_project_origin, project_path) try: - command = "cd {0}; git clone {1} {4}; cd {2}; git checkout {3};".format(project_path, + command = "cd {0}; git clone --single-branch {1} {4}; cd {2}; git checkout {3};".format(project_path, git_project_origin, git_path, git_project_commit, project_destination) if git_project_submodules.__len__() <= 0: - command += " git submodule update --init --recursive" + command += " git submodule update --depth --init --recursive" else: command += " cd {0}; git submodule init;".format(project_destination) @@ -168,18 +173,29 @@ class AutosubmitGit: Log.info("Cloning {0} into {1}", git_project_branch + " " + git_project_origin, project_path) try: command = "cd {0}; ".format(project_path) + if git_project_submodules.__len__() <= 0: - command += " git clone --recursive -b {0} {1} {2}".format(git_project_branch, git_project_origin, - project_destination) + if not git_single_branch: + command += " git clone --recursive -b {0} {1} {2}".format(git_project_branch, git_project_origin, + project_destination) + else: + command += " git clone --single-branch --recursive -b {0} {1} {2}".format(git_project_branch, git_project_origin, + project_destination) else: - command += " git clone -b {0} {1} {2};".format(git_project_branch, git_project_origin, + if not git_single_branch: + command += " git clone -b {0} {1} {2};".format(git_project_branch, git_project_origin, project_destination) + else: + command += " git clone --single-branch -b {0} {1} {2};".format(git_project_branch, + git_project_origin, + project_destination) + command += " cd {0}; git submodule init;".format(project_destination) for submodule in git_project_submodules: - command += " git submodule update {0};".format(submodule) - + command += " git submodule update --depth {0};".format(submodule) + Log.debug('{0}', command) output = subprocess.check_output(command, shell=True) - Log.debug('{0}:{1}', command, output) + #Log.debug('{0}:{1}', command, output) except subprocess.CalledProcessError: Log.error("Can not clone {0} into {1}", git_project_branch + " " + git_project_origin, project_path) shutil.rmtree(project_path) -- GitLab From 1f64baea336511822dca468ce325ebf5f8ea6042 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 12 Aug 2019 09:52:57 +0200 Subject: [PATCH 2/3] changes into git single-branch --- autosubmit/config/config_common.py | 2 +- autosubmit/config/files/expdef.conf | 2 +- autosubmit/git/autosubmit_git.py | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/autosubmit/config/config_common.py b/autosubmit/config/config_common.py index 0e5426958..c0d553567 100644 --- a/autosubmit/config/config_common.py +++ b/autosubmit/config/config_common.py @@ -682,7 +682,7 @@ class AutosubmitConfig(object): :return: fetch_single_branch(Y/N) :rtype: boolean """ - return self._exp_parser.get_option('git', 'FETCH_SINGLE_BRANCH', 'True').lower() + return self._exp_parser.get_option('git', 'FETCH_SINGLE_BRANCH', 'False').lower() def get_project_destination(self): """ Returns git commit from experiment's config file diff --git a/autosubmit/config/files/expdef.conf b/autosubmit/config/files/expdef.conf index 0cf6ab6ce..8a9639774 100644 --- a/autosubmit/config/files/expdef.conf +++ b/autosubmit/config/files/expdef.conf @@ -49,7 +49,7 @@ PROJECT_COMMIT = # type = STRING, default = leave empty and will load all submodules, help = loadThisSubmodule alsoloadthis anotherLoad ... PROJECT_SUBMODULES = # type = STRING, default = leave empty and will clone only a single branch, help = true,false -FETCH_SINGLE_BRANCH = true +FETCH_SINGLE_BRANCH = false # If PROJECT_TYPE is not svn, no need to change [svn] diff --git a/autosubmit/git/autosubmit_git.py b/autosubmit/git/autosubmit_git.py index 253d4fd99..6bed6a245 100644 --- a/autosubmit/git/autosubmit_git.py +++ b/autosubmit/git/autosubmit_git.py @@ -196,6 +196,10 @@ class AutosubmitGit: Log.debug('{0}', command) output = subprocess.check_output(command, shell=True) #Log.debug('{0}:{1}', command, output) + #command " git config --file .gitmodules --get-regexp path | awk '{ print $2 }' " you can change path per url + #https://stackoverflow.com/questions/27188899/shallow-clone-with-submodules-in-git-how-to-use-pointed-commits-and-not-latest + #../../svn/ecearth-mirror.git + # git clone -n --depth 1 --separate-git-dir sources https://earth.bsc.es/svn/ecearth-mirror.git sources except subprocess.CalledProcessError: Log.error("Can not clone {0} into {1}", git_project_branch + " " + git_project_origin, project_path) shutil.rmtree(project_path) -- GitLab From 58a2ba144b444944f0cc8a95c7a73960b36b5254 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 12 Aug 2019 13:00:49 +0200 Subject: [PATCH 3/3] changes into git single-branch --- autosubmit/git/autosubmit_git.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/autosubmit/git/autosubmit_git.py b/autosubmit/git/autosubmit_git.py index 6bed6a245..56acd9752 100644 --- a/autosubmit/git/autosubmit_git.py +++ b/autosubmit/git/autosubmit_git.py @@ -151,13 +151,18 @@ class AutosubmitGit: if git_project_commit: Log.info("Fetching {0} into {1}", git_project_commit + " " + git_project_origin, project_path) try: - - command = "cd {0}; git clone --single-branch {1} {4}; cd {2}; git checkout {3};".format(project_path, + if git_single_branch: + command = "cd {0}; git clone {1} {4}; cd {2}; git checkout {3};".format(project_path, + git_project_origin, git_path, + git_project_commit, + project_destination) + else: + command = "cd {0}; git clone {1} {4}; cd {2}; git checkout {3};".format(project_path, git_project_origin, git_path, git_project_commit, project_destination) if git_project_submodules.__len__() <= 0: - command += " git submodule update --depth --init --recursive" + command += " git submodule update --init --recursive" else: command += " cd {0}; git submodule init;".format(project_destination) @@ -179,27 +184,27 @@ class AutosubmitGit: command += " git clone --recursive -b {0} {1} {2}".format(git_project_branch, git_project_origin, project_destination) else: - command += " git clone --single-branch --recursive -b {0} {1} {2}".format(git_project_branch, git_project_origin, + command += " git clone --single-branch --depth=1 --recursive -b {0} {1} {2}".format(git_project_branch, git_project_origin, project_destination) else: if not git_single_branch: command += " git clone -b {0} {1} {2};".format(git_project_branch, git_project_origin, project_destination) else: - command += " git clone --single-branch -b {0} {1} {2};".format(git_project_branch, + command += " git clone --single-branch --depth=1 -b {0} {1} {2};".format(git_project_branch, git_project_origin, project_destination) command += " cd {0}; git submodule init;".format(project_destination) for submodule in git_project_submodules: - command += " git submodule update --depth {0};".format(submodule) + command += " git submodule update {0};".format(submodule) Log.debug('{0}', command) output = subprocess.check_output(command, shell=True) #Log.debug('{0}:{1}', command, output) #command " git config --file .gitmodules --get-regexp path | awk '{ print $2 }' " you can change path per url #https://stackoverflow.com/questions/27188899/shallow-clone-with-submodules-in-git-how-to-use-pointed-commits-and-not-latest #../../svn/ecearth-mirror.git - # git clone -n --depth 1 --separate-git-dir sources https://earth.bsc.es/svn/ecearth-mirror.git sources + # git clone -n --depth=1 --separate-git-dir sources https://earth.bsc.es/svn/ecearth-mirror.git sources except subprocess.CalledProcessError: Log.error("Can not clone {0} into {1}", git_project_branch + " " + git_project_origin, project_path) shutil.rmtree(project_path) -- GitLab