From 705ee652f650848890e461aca85fa82e4db852ad Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 7 Nov 2024 08:49:06 +0100 Subject: [PATCH 1/6] Fixes #1129. Changed "raise Exception" per Autosubmit critical Changed Critical exit per a warning if it is inside a git refresh/ autosubmit create --- autosubmit/autosubmit.py | 15 ++++++--------- autosubmit/platforms/paramiko_submitter.py | 9 ++++++--- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 9d343e10c..561d3f899 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -4718,16 +4718,13 @@ class Autosubmit: raise AutosubmitCritical("Autosubmit couldn't identify the project destination.", 7014) if project_type == "git": - submitter = Autosubmit._get_submitter(as_conf) - submitter.load_platforms(as_conf) try: - hpcarch = submitter.platforms.get(as_conf.get_platform(), "local") - except BaseException as e: - error = str(e) - try: - hpcarch = submitter.platforms[as_conf.get_platform()] - except Exception as e: - hpcarch = "local" + submitter = Autosubmit._get_submitter(as_conf) + submitter.load_platforms(as_conf) + hpcarch = submitter.platforms[as_conf.get_platform()] + except AutosubmitCritical as e: + Log.warning(f"{e.message}\nRemote git cloning is disabled") + hpcarch = "local" return AutosubmitGit.clone_repository(as_conf, force, hpcarch) elif project_type == "svn": svn_project_url = as_conf.get_svn_project_url() diff --git a/autosubmit/platforms/paramiko_submitter.py b/autosubmit/platforms/paramiko_submitter.py index dfdb27232..67050528b 100644 --- a/autosubmit/platforms/paramiko_submitter.py +++ b/autosubmit/platforms/paramiko_submitter.py @@ -21,7 +21,7 @@ import os from collections import defaultdict -from log.log import Log,AutosubmitError +from log.log import Log, AutosubmitError, AutosubmitCritical from autosubmitconfigparser.config.basicconfig import BasicConfig from autosubmitconfigparser.config.configcommon import AutosubmitConfig from .submitter import Submitter @@ -149,8 +149,11 @@ class ParamikoSubmitter(Submitter): remote_platform = PJMPlatform( asconf.expid, section, config) else: - raise Exception( - "Queue type not specified on platform {0}".format(section)) + if platform_type == "": + platform_type_value = "" + else: + platform_type_value = platform_type + raise AutosubmitCritical(f"PLATFORMS.{section.upper()}.TYPE: {platform_type_value} for {section.upper()} is not supported", 7012) remote_platform.main_process_id = os.getpid() except ParamikoPlatformException as e: Log.error("Queue exception: {0}".format(str(e))) -- GitLab From caff96d79ebc7f688d67baeba2dfa14fc13ea7a9 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 7 Nov 2024 09:30:01 +0100 Subject: [PATCH 2/6] Added pytest #1 --- test/unit/test_paramiko_submitter.py | 60 ++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 test/unit/test_paramiko_submitter.py diff --git a/test/unit/test_paramiko_submitter.py b/test/unit/test_paramiko_submitter.py new file mode 100644 index 000000000..292a13fd8 --- /dev/null +++ b/test/unit/test_paramiko_submitter.py @@ -0,0 +1,60 @@ +import pytest +from log.log import AutosubmitCritical +from autosubmit.platforms.paramiko_submitter import ParamikoSubmitter + + +@pytest.mark.parametrize("config", [ + { + "DEFAULT": { + "HPCARCH": "PYTEST-UNDEFINED", + }, + "LOCAL_ROOT_DIR": "blabla", + "LOCAL_TMP_DIR": 'tmp', + "PLATFORMS": { + "PYTEST-UNDEFINED": { + "host": "", + "user": "", + "project": "", + "scratch_dir": "", + "MAX_WALLCLOCK": "", + "DISABLE_RECOVERY_THREADS": True + } + }, + "JOBS": { + "job1": { + "PLATFORM": "PYTEST-UNDEFINED", + "SCRIPT": "echo 'hello world'", + }, + } + }, + { + "DEFAULT": { + "HPCARCH": "PYTEST-UNSUPPORTED", + }, + "LOCAL_ROOT_DIR": "blabla", + "LOCAL_TMP_DIR": 'tmp', + "PLATFORMS": { + "PYTEST-UNSUPPORTED": { + "type": "unknown", + "host": "", + "user": "", + "project": "", + "scratch_dir": "", + "MAX_WALLCLOCK": "", + "DISABLE_RECOVERY_THREADS": True + } + }, + "JOBS": { + "job1": { + "PLATFORM": "PYTEST-UNSUPPORTED", + "SCRIPT": "echo 'hello world'", + }, + } + } +], ids=["Undefined", "Unsupported"]) +def test_load_platforms(autosubmit_config, config): + experiment_id = 'random-id' + as_conf = autosubmit_config(experiment_id, config) + submitter = ParamikoSubmitter() + with pytest.raises(AutosubmitCritical): + submitter.load_platforms(as_conf) -- GitLab From dee4aa33c7551c3947c2b1c8f38f071a730adff5 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 7 Nov 2024 09:51:48 +0100 Subject: [PATCH 3/6] Added pytest #2 --- test/unit/{gui => }/test_autosubmit_git.py | 0 test/unit/test_autosubmit_git_pytest.py | 38 ++++++++++++++++++++++ 2 files changed, 38 insertions(+) rename test/unit/{gui => }/test_autosubmit_git.py (100%) create mode 100644 test/unit/test_autosubmit_git_pytest.py diff --git a/test/unit/gui/test_autosubmit_git.py b/test/unit/test_autosubmit_git.py similarity index 100% rename from test/unit/gui/test_autosubmit_git.py rename to test/unit/test_autosubmit_git.py diff --git a/test/unit/test_autosubmit_git_pytest.py b/test/unit/test_autosubmit_git_pytest.py new file mode 100644 index 000000000..b23b0ee50 --- /dev/null +++ b/test/unit/test_autosubmit_git_pytest.py @@ -0,0 +1,38 @@ +import pytest + +from autosubmit.autosubmit import Autosubmit + + +@pytest.mark.parametrize("config", [ + { + "DEFAULT": { + "HPCARCH": "PYTEST-UNDEFINED", + }, + "LOCAL_ROOT_DIR": "blabla", + "LOCAL_TMP_DIR": 'tmp', + "PLATFORMS": { + "PYTEST-UNDEFINED": { + "host": "", + "user": "", + "project": "", + "scratch_dir": "", + "MAX_WALLCLOCK": "", + "DISABLE_RECOVERY_THREADS": True + } + }, + "JOBS": { + "job1": { + "PLATFORM": "PYTEST-UNDEFINED", + "SCRIPT": "echo 'hello world'", + }, + } + }, +], ids=["GIT_CLONE"]) +def test_copy_code(autosubmit_config, config, mocker): + expid = 'random-id' + as_conf = autosubmit_config(expid, config) + mocker.patch('autosubmit.git.autosubmit_git.AutosubmitGit.clone_repository', return_value=True) + assert Autosubmit._copy_code(as_conf, expid, "git", True) + + + -- GitLab From 7aac514d8f9a63d1e7582aae712f0d9526452c9f Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 7 Nov 2024 09:52:21 +0100 Subject: [PATCH 4/6] changed test name --- test/unit/test_autosubmit_git_pytest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_autosubmit_git_pytest.py b/test/unit/test_autosubmit_git_pytest.py index b23b0ee50..530ba4188 100644 --- a/test/unit/test_autosubmit_git_pytest.py +++ b/test/unit/test_autosubmit_git_pytest.py @@ -27,7 +27,7 @@ from autosubmit.autosubmit import Autosubmit }, } }, -], ids=["GIT_CLONE"]) +], ids=["Git clone without type defined"]) def test_copy_code(autosubmit_config, config, mocker): expid = 'random-id' as_conf = autosubmit_config(expid, config) -- GitLab From 2d4c0151baec05d2512852a77dbef8e4abba8b19 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 7 Nov 2024 11:22:30 +0100 Subject: [PATCH 5/6] Improve coverage --- test/unit/test_autosubmit_git_pytest.py | 28 +++++++++++++++++++++---- test/unit/test_paramiko_submitter.py | 2 +- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/test/unit/test_autosubmit_git_pytest.py b/test/unit/test_autosubmit_git_pytest.py index 530ba4188..656db38b2 100644 --- a/test/unit/test_autosubmit_git_pytest.py +++ b/test/unit/test_autosubmit_git_pytest.py @@ -27,12 +27,32 @@ from autosubmit.autosubmit import Autosubmit }, } }, -], ids=["Git clone without type defined"]) + { + "DEFAULT": { + "HPCARCH": "PYTEST-PS", + }, + "LOCAL_ROOT_DIR": "blabla", + "LOCAL_TMP_DIR": 'tmp', + "PLATFORMS": { + "PYTEST-PS": { + "TYPE": "ps", + "host": "", + "user": "", + "project": "", + "scratch_dir": "", + "MAX_WALLCLOCK": "", + "DISABLE_RECOVERY_THREADS": True + } + }, + "JOBS": { + "job1": { + "PLATFORM": "PYTEST-PS", + "SCRIPT": "echo 'hello world'", + }, + } + }], ids=["Git clone without type defined", "Git clone with the correct type defined"]) def test_copy_code(autosubmit_config, config, mocker): expid = 'random-id' as_conf = autosubmit_config(expid, config) mocker.patch('autosubmit.git.autosubmit_git.AutosubmitGit.clone_repository', return_value=True) assert Autosubmit._copy_code(as_conf, expid, "git", True) - - - diff --git a/test/unit/test_paramiko_submitter.py b/test/unit/test_paramiko_submitter.py index 292a13fd8..7ffb00ef0 100644 --- a/test/unit/test_paramiko_submitter.py +++ b/test/unit/test_paramiko_submitter.py @@ -35,7 +35,7 @@ from autosubmit.platforms.paramiko_submitter import ParamikoSubmitter "LOCAL_TMP_DIR": 'tmp', "PLATFORMS": { "PYTEST-UNSUPPORTED": { - "type": "unknown", + "TYPE": "unknown", "host": "", "user": "", "project": "", -- GitLab From fca2689fba767746a04387dff90958d0d96df922 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 12 Nov 2024 10:36:40 +0100 Subject: [PATCH 6/6] addressed feeback --- autosubmit/platforms/paramiko_submitter.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/autosubmit/platforms/paramiko_submitter.py b/autosubmit/platforms/paramiko_submitter.py index 67050528b..3e6f664d0 100644 --- a/autosubmit/platforms/paramiko_submitter.py +++ b/autosubmit/platforms/paramiko_submitter.py @@ -149,10 +149,7 @@ class ParamikoSubmitter(Submitter): remote_platform = PJMPlatform( asconf.expid, section, config) else: - if platform_type == "": - platform_type_value = "" - else: - platform_type_value = platform_type + platform_type_value = platform_type or "" raise AutosubmitCritical(f"PLATFORMS.{section.upper()}.TYPE: {platform_type_value} for {section.upper()} is not supported", 7012) remote_platform.main_process_id = os.getpid() except ParamikoPlatformException as e: -- GitLab