From edb6fe37e3e4dd74d115fa31cf962519f11f59c4 Mon Sep 17 00:00:00 2001 From: Daniel Beltran Mora Date: Mon, 10 Dec 2018 13:42:23 +0100 Subject: [PATCH 1/5] Adding an option for remove mandatory recursive , #297 --- autosubmit/config/config_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit/config/config_common.py b/autosubmit/config/config_common.py index 58d0a53d0..a78cb867e 100644 --- a/autosubmit/config/config_common.py +++ b/autosubmit/config/config_common.py @@ -436,7 +436,7 @@ class AutosubmitConfig(object): if project_type == 'git': result = result and parser.check_exists('git', 'PROJECT_ORIGIN') result = result and parser.check_exists('git', 'PROJECT_BRANCH') - result = result and parser.check_is_boolean('git', 'RECURSIVE') + result = result and parser.check_is_boolean('git', 'RECURSIVE', False) elif project_type == 'svn': result = result and parser.check_exists('svn', 'PROJECT_URL') result = result and parser.check_exists('svn', 'PROJECT_REVISION') -- GitLab From 706755501cfc9162f3fdd6bd95956945063896e0 Mon Sep 17 00:00:00 2001 From: Daniel Beltran Mora Date: Mon, 10 Dec 2018 13:58:05 +0100 Subject: [PATCH 2/5] now it is not mandatory, still work to do, #297 --- autosubmit/config/config_common.py | 2 +- autosubmit/git/autosubmit_git.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/autosubmit/config/config_common.py b/autosubmit/config/config_common.py index a78cb867e..e4cf1b91a 100644 --- a/autosubmit/config/config_common.py +++ b/autosubmit/config/config_common.py @@ -436,7 +436,6 @@ class AutosubmitConfig(object): if project_type == 'git': result = result and parser.check_exists('git', 'PROJECT_ORIGIN') result = result and parser.check_exists('git', 'PROJECT_BRANCH') - result = result and parser.check_is_boolean('git', 'RECURSIVE', False) elif project_type == 'svn': result = result and parser.check_exists('svn', 'PROJECT_URL') result = result and parser.check_exists('svn', 'PROJECT_REVISION') @@ -619,6 +618,7 @@ class AutosubmitConfig(object): :rtype: str """ return self._exp_parser.get_option('git', 'RECURSIVE', True) + def get_project_destination(self): """ Returns git commit from experiment's config file diff --git a/autosubmit/git/autosubmit_git.py b/autosubmit/git/autosubmit_git.py index adcb8f460..67b351239 100644 --- a/autosubmit/git/autosubmit_git.py +++ b/autosubmit/git/autosubmit_git.py @@ -95,7 +95,7 @@ class AutosubmitGit: git_project_origin = as_conf.get_git_project_origin() git_project_branch = as_conf.get_git_project_branch() git_project_commit = as_conf.get_git_project_commit() - if as_conf.get_git_project_recursive() : + if as_conf.get_git_recursive(): git_project_recursive = "--recursive" else: git_project_recursive = "" -- GitLab From 08753326fd6c860776feefe9e9cd8b2c5b975c2f Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 13 Dec 2018 12:04:20 +0100 Subject: [PATCH 3/5] Implemented and now is in testing phase , #297 --- autosubmit/config/config_common.py | 13 +++++++------ autosubmit/config/files/expdef.conf | 3 ++- autosubmit/git/autosubmit_git.py | 20 +++++++++++--------- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/autosubmit/config/config_common.py b/autosubmit/config/config_common.py index e4cf1b91a..6db779f1d 100644 --- a/autosubmit/config/config_common.py +++ b/autosubmit/config/config_common.py @@ -610,14 +610,15 @@ class AutosubmitConfig(object): :rtype: str """ return self._exp_parser.get_option('git', 'PROJECT_COMMIT', None) - def get_git_recursive(self): + def get_submodules_list(self): """ - Returns git recursive boolean from experiment's config file - - :return: git commit - :rtype: str + Returns submodules list from experiment's config file + Default is --recursive + :return: submodules to load + :rtype: list """ - return self._exp_parser.get_option('git', 'RECURSIVE', True) + return ' '.join(self._exp_parser.get('git', 'PROJECT_SUBMODULES').split()).split() + def get_project_destination(self): """ diff --git a/autosubmit/config/files/expdef.conf b/autosubmit/config/files/expdef.conf index cb6612491..f4bcf5a78 100644 --- a/autosubmit/config/files/expdef.conf +++ b/autosubmit/config/files/expdef.conf @@ -46,7 +46,8 @@ PROJECT_ORIGIN = PROJECT_BRANCH = # type = STRING, default = leave empty, help = if model branch is a TAG leave empty PROJECT_COMMIT = - +# type = STRING, default = leave empty and will load all submodules, help = https://github.com/torvalds/loadThisSubmodule https://github.com/torvalds/alsoLoadThisSubmodule +PROJECT_SUBMODULES = # If PROJECT_TYPE is not svn, no need to change [svn] # type = STRING, help = 'https://svn.ec-earth.org/ecearth3' diff --git a/autosubmit/git/autosubmit_git.py b/autosubmit/git/autosubmit_git.py index 67b351239..8696e8e4a 100644 --- a/autosubmit/git/autosubmit_git.py +++ b/autosubmit/git/autosubmit_git.py @@ -95,10 +95,7 @@ class AutosubmitGit: git_project_origin = as_conf.get_git_project_origin() git_project_branch = as_conf.get_git_project_branch() git_project_commit = as_conf.get_git_project_commit() - if as_conf.get_git_recursive(): - git_project_recursive = "--recursive" - else: - git_project_recursive = "" + git_project_submodules = as_conf.get_submodules_list() project_destination = as_conf.get_project_destination() project_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, as_conf.expid, BasicConfig.LOCAL_PROJ_DIR) git_path = as_conf.get_project_dir() @@ -116,9 +113,14 @@ class AutosubmitGit: if git_project_commit: Log.info("Fetching {0} into {1}", git_project_commit + " " + git_project_origin, project_path) try: - command = "cd {0}; git clone {1} {4}; cd {2}; git checkout {3}; " \ - "git submodule update --init {5}".format(project_path, git_project_origin, git_path, - git_project_commit, project_destination,git_project_recursive) + + command = "cd {0}; git clone {1} {4}; cd {2}; git checkout {3};".format(project_path, git_project_origin, git_path, + git_project_commit, project_destination) + if git_project_submodules.__len__() <= 0: + command+="git submodule update --init --recursive" + else: + for submodule in git_project_submodules: + command +="git submodule update --init {0};".format(submodule) output = subprocess.check_output(command, shell=True) except subprocess.CalledProcessError: Log.error("Can not checkout commit {0}: {1}", git_project_commit, output) @@ -128,8 +130,8 @@ class AutosubmitGit: else: Log.info("Cloning {0} into {1}", git_project_branch + " " + git_project_origin, project_path) try: - command = "cd {0}; git clone {4} -b {1} {2} {3}".format(project_path, git_project_branch, - git_project_origin, project_destination,git_project_recursive) + command = "cd {0}; git clone --recursive -b {1} {2} {3}".format(project_path, git_project_branch, + git_project_origin, project_destination) output = subprocess.check_output(command, shell=True) Log.debug('{0}:{1}', command, output) except subprocess.CalledProcessError: -- GitLab From a86cead1347e99290ab4fc192bbab978e639ea34 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 13 Dec 2018 17:30:12 +0100 Subject: [PATCH 4/5] Tweaks to the code, testing phase almost passed, #297 --- autosubmit/config/config_common.py | 3 ++- autosubmit/config/files/expdef.conf | 2 +- autosubmit/git/autosubmit_git.py | 15 +++++++++++---- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/autosubmit/config/config_common.py b/autosubmit/config/config_common.py index 6db779f1d..4e19c9076 100644 --- a/autosubmit/config/config_common.py +++ b/autosubmit/config/config_common.py @@ -436,6 +436,7 @@ class AutosubmitConfig(object): if project_type == 'git': result = result and parser.check_exists('git', 'PROJECT_ORIGIN') result = result and parser.check_exists('git', 'PROJECT_BRANCH') + elif project_type == 'svn': result = result and parser.check_exists('svn', 'PROJECT_URL') result = result and parser.check_exists('svn', 'PROJECT_REVISION') @@ -617,7 +618,7 @@ class AutosubmitConfig(object): :return: submodules to load :rtype: list """ - return ' '.join(self._exp_parser.get('git', 'PROJECT_SUBMODULES').split()).split() + return ' '.join(self._exp_parser.get_option('git', 'PROJECT_SUBMODULES','').split()).split() def get_project_destination(self): diff --git a/autosubmit/config/files/expdef.conf b/autosubmit/config/files/expdef.conf index f4bcf5a78..96e2b6fd3 100644 --- a/autosubmit/config/files/expdef.conf +++ b/autosubmit/config/files/expdef.conf @@ -46,7 +46,7 @@ PROJECT_ORIGIN = PROJECT_BRANCH = # type = STRING, default = leave empty, help = if model branch is a TAG leave empty PROJECT_COMMIT = -# type = STRING, default = leave empty and will load all submodules, help = https://github.com/torvalds/loadThisSubmodule https://github.com/torvalds/alsoLoadThisSubmodule +# type = STRING, default = leave empty and will load all submodules, help = loadThisSubmodule alsoloadthis anotherLoad ... PROJECT_SUBMODULES = # If PROJECT_TYPE is not svn, no need to change [svn] diff --git a/autosubmit/git/autosubmit_git.py b/autosubmit/git/autosubmit_git.py index 8696e8e4a..b02d0d8d4 100644 --- a/autosubmit/git/autosubmit_git.py +++ b/autosubmit/git/autosubmit_git.py @@ -117,10 +117,10 @@ class AutosubmitGit: command = "cd {0}; git clone {1} {4}; cd {2}; git checkout {3};".format(project_path, git_project_origin, git_path, git_project_commit, project_destination) if git_project_submodules.__len__() <= 0: - command+="git submodule update --init --recursive" + command+=" git submodule update --init --recursive" else: for submodule in git_project_submodules: - command +="git submodule update --init {0};".format(submodule) + command +=" git submodule update --init {0};".format(submodule) output = subprocess.check_output(command, shell=True) except subprocess.CalledProcessError: Log.error("Can not checkout commit {0}: {1}", git_project_commit, output) @@ -130,8 +130,15 @@ class AutosubmitGit: else: Log.info("Cloning {0} into {1}", git_project_branch + " " + git_project_origin, project_path) try: - command = "cd {0}; git clone --recursive -b {1} {2} {3}".format(project_path, git_project_branch, - git_project_origin, project_destination) + command = "cd {0}; ".format(project_path) + if git_project_submodules.__len__() <= 0: + command +=" git clone --recursive -b {0} {1} {2}".format(git_project_branch,git_project_origin, project_destination) + else: + command += " git clone -b {0} {1} {2};".format(git_project_branch, git_project_origin,project_destination) + command += " cd {0}; git submodule init;".format(project_destination) + for submodule in git_project_submodules: + command +=" git submodule update {0};".format(submodule) + output = subprocess.check_output(command, shell=True) Log.debug('{0}:{1}', command, output) except subprocess.CalledProcessError: -- GitLab From 9d91e94c76a10565b8682e41823c27df352a0072 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 14 Dec 2018 15:38:36 +0100 Subject: [PATCH 5/5] Test finished, minor tweak, #297 --- autosubmit/git/autosubmit_git.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/autosubmit/git/autosubmit_git.py b/autosubmit/git/autosubmit_git.py index b02d0d8d4..d0d2689b3 100644 --- a/autosubmit/git/autosubmit_git.py +++ b/autosubmit/git/autosubmit_git.py @@ -114,13 +114,17 @@ class AutosubmitGit: Log.info("Fetching {0} into {1}", git_project_commit + " " + git_project_origin, project_path) try: - command = "cd {0}; git clone {1} {4}; cd {2}; git checkout {3};".format(project_path, git_project_origin, git_path, - git_project_commit, project_destination) + command = "cd {0}; git clone {1} {4}; cd {2}; git checkout {3};".format(project_path, + git_project_origin, git_path, + git_project_commit, + project_destination) if git_project_submodules.__len__() <= 0: - command+=" git submodule update --init --recursive" + command += " git submodule update --init --recursive" else: + + command += " cd {0}; git submodule init;".format(project_destination) for submodule in git_project_submodules: - command +=" git submodule update --init {0};".format(submodule) + command += " git submodule update {0};".format(submodule) output = subprocess.check_output(command, shell=True) except subprocess.CalledProcessError: Log.error("Can not checkout commit {0}: {1}", git_project_commit, output) @@ -132,12 +136,14 @@ class AutosubmitGit: try: command = "cd {0}; ".format(project_path) if git_project_submodules.__len__() <= 0: - command +=" git clone --recursive -b {0} {1} {2}".format(git_project_branch,git_project_origin, project_destination) + command += " git clone --recursive -b {0} {1} {2}".format(git_project_branch, git_project_origin, + project_destination) else: - command += " git clone -b {0} {1} {2};".format(git_project_branch, git_project_origin,project_destination) + command += " git clone -b {0} {1} {2};".format(git_project_branch, git_project_origin, + project_destination) command += " cd {0}; git submodule init;".format(project_destination) for submodule in git_project_submodules: - command +=" git submodule update {0};".format(submodule) + command += " git submodule update {0};".format(submodule) output = subprocess.check_output(command, shell=True) Log.debug('{0}:{1}', command, output) -- GitLab