From 84b8604c79dfc11992363a760fc64434012e45d7 Mon Sep 17 00:00:00 2001 From: Edgar Garriga Date: Wed, 17 Jul 2024 15:09:07 +0200 Subject: [PATCH 1/4] check project path , no test with flags --- autosubmit/autosubmit.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index edbf4134..322adb5a 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -4677,7 +4677,7 @@ class Autosubmit: :return: True if successful, False if not :rtype: bool """ - + Log.info(" **** INSIDE _copy_code") project_destination = as_conf.get_project_destination() if project_destination is None or len(project_destination) == 0: if project_type.lower() != "none": @@ -4732,16 +4732,25 @@ class Autosubmit: Log.debug("{0}", output) elif project_type == "local": + Log.info(" **** project_type : local") local_project_path = as_conf.get_local_project_path() if local_project_path is None or len(local_project_path) == 0: raise AutosubmitCritical("Empty project path! please change this parameter to a valid one.", 7014) + ### check if local_project_path is a valid path + if not Path(local_project_path).is_dir(): + raise AutosubmitCritical("Local project path is not a valid path and/or it doesnt exist.", 7014) + ### project_path = os.path.join( BasicConfig.LOCAL_ROOT_DIR, expid, BasicConfig.LOCAL_PROJ_DIR) local_destination = os.path.join(project_path, project_destination) + + Log.info(" **** project_type : local check vars: \n{0} - local_project_path (as_conf)\n{2} - project_path (basic config) \n{3} - project_destination (as_conf.get_project_destination)\n{1} - local_destination (project_path + project_destination)\n\n",local_project_path,local_destination,project_path,project_destination) if os.path.exists(project_path): + Log.info(" **** if path exist: project path") Log.info("Using project folder: {0}", project_path) if os.path.exists(local_destination): + Log.info(" **** if path exist: local destination") if force: try: cmd = ["rsync -ach --info=progress2 " + @@ -4751,8 +4760,11 @@ class Autosubmit: raise AutosubmitCritical("Can not rsync {0} into {1}. Exiting...".format( local_project_path, project_path), 7063) else: - os.mkdir(local_destination) + Log.info(" **** else path exist: local destination") + # os.makedirs(local_destination) #to create intermediate dirs #os.mkdir(local_destination) + Path(local_destination).mkdir(parents=True) #this will create parents dirs if its needed try: + Log.info(" **** try cp -R {0} & {1}",local_project_path, local_destination) output = subprocess.check_output( "cp -R " + local_project_path + "/* " + local_destination, shell=True) except subprocess.CalledProcessError: @@ -4763,13 +4775,16 @@ class Autosubmit: raise AutosubmitCritical("Can not copy {0} into {1}. Exiting...".format( local_project_path, project_path), 7063) else: - os.mkdir(project_path) - os.mkdir(local_destination) + Log.info(" **** else path exist: project path") + # os.makedirs(xxxx) #to create intermediate dirs + Path(project_path).mkdir(parents=True) #this will vreate parents dirs if its needed + Path(local_destination).mkdir(parents=True) Log.debug( "The project folder {0} has been created.", project_path) Log.info("Copying {0} into {1}", local_project_path, project_path) try: + Log.info(" **** else try cp -R {0} & {1}",local_project_path, local_destination) output = subprocess.check_output( "cp -R " + local_project_path + "/* " + local_destination, shell=True) except subprocess.CalledProcessError: -- GitLab From 15f2004d2855e71078bd111d5ec6a83145f5bd1e Mon Sep 17 00:00:00 2001 From: Edgar Garriga Date: Wed, 17 Jul 2024 15:43:14 +0200 Subject: [PATCH 2/4] modify os.path to pathlib --- autosubmit/autosubmit.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 322adb5a..d28d1e39 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -4740,16 +4740,18 @@ class Autosubmit: if not Path(local_project_path).is_dir(): raise AutosubmitCritical("Local project path is not a valid path and/or it doesnt exist.", 7014) ### - project_path = os.path.join( - BasicConfig.LOCAL_ROOT_DIR, expid, BasicConfig.LOCAL_PROJ_DIR) - local_destination = os.path.join(project_path, project_destination) - + #project_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, BasicConfig.LOCAL_PROJ_DIR) + #local_destination = os.path.join(project_path, project_destination) + project_path = BasicConfig.LOCAL_ROOT_DIR.joinpath(expid,BasicConfig.LOCAL_PROJ_DIR) + local_destination = project_path.joinpath(project_destination) Log.info(" **** project_type : local check vars: \n{0} - local_project_path (as_conf)\n{2} - project_path (basic config) \n{3} - project_destination (as_conf.get_project_destination)\n{1} - local_destination (project_path + project_destination)\n\n",local_project_path,local_destination,project_path,project_destination) - if os.path.exists(project_path): + #if os.path.exists(project_path): + if project_path.exists(): Log.info(" **** if path exist: project path") Log.info("Using project folder: {0}", project_path) - if os.path.exists(local_destination): + #if os.path.exists(local_destination): + if local_destination.exists(): Log.info(" **** if path exist: local destination") if force: try: -- GitLab From a8f1e019144af921585af2be6493ecd32c316f02 Mon Sep 17 00:00:00 2001 From: Edgar Garriga Date: Wed, 17 Jul 2024 15:51:36 +0200 Subject: [PATCH 3/4] test template --- test/unit/conftest.py | 109 +++++++++++++++++++++++++++++++ test/unit/test_copy_code_Path.py | 35 ++++++++++ 2 files changed, 144 insertions(+) create mode 100644 test/unit/conftest.py create mode 100644 test/unit/test_copy_code_Path.py diff --git a/test/unit/conftest.py b/test/unit/conftest.py new file mode 100644 index 00000000..9f06fe19 --- /dev/null +++ b/test/unit/conftest.py @@ -0,0 +1,109 @@ +# Fixtures available to multiple test files must be created in this file. + +import pytest +from dataclasses import dataclass +from pathlib import Path +from shutil import rmtree +from tempfile import TemporaryDirectory +from typing import Any, Dict, Callable, List + +from autosubmit.autosubmit import Autosubmit +from autosubmit.platforms.slurmplatform import SlurmPlatform, ParamikoPlatform +from autosubmitconfigparser.config.basicconfig import BasicConfig +from autosubmitconfigparser.config.configcommon import AutosubmitConfig +from autosubmitconfigparser.config.yamlparser import YAMLParserFactory + + +@dataclass +class AutosubmitExperiment: + """This holds information about an experiment created by Autosubmit.""" + expid: str + autosubmit: Autosubmit + exp_path: Path + tmp_dir: Path + aslogs_dir: Path + status_dir: Path + platform: ParamikoPlatform + + +@pytest.fixture(scope='function') +def autosubmit_exp(autosubmit: Autosubmit, request: pytest.FixtureRequest) -> Callable: + """Create an instance of ``Autosubmit`` with an experiment.""" + + original_root_dir = BasicConfig.LOCAL_ROOT_DIR + tmp_dir = TemporaryDirectory() + tmp_path = Path(tmp_dir.name) + + def _create_autosubmit_exp(expid: str): + # directories used when searching for logs to cat + root_dir = tmp_path + BasicConfig.LOCAL_ROOT_DIR = str(root_dir) + exp_path = root_dir / expid + exp_tmp_dir = exp_path / BasicConfig.LOCAL_TMP_DIR + aslogs_dir = exp_tmp_dir / BasicConfig.LOCAL_ASLOG_DIR + status_dir = exp_path / 'status' + aslogs_dir.mkdir(parents=True) + status_dir.mkdir() + + platform_config = { + "LOCAL_ROOT_DIR": BasicConfig.LOCAL_ROOT_DIR, + "LOCAL_TMP_DIR": str(exp_tmp_dir), + "LOCAL_ASLOG_DIR": str(aslogs_dir) + } + platform = SlurmPlatform(expid=expid, name='slurm_platform', config=platform_config) + platform.job_status = { + 'COMPLETED': [], + 'RUNNING': [], + 'QUEUING': [], + 'FAILED': [] + } + submit_platform_script = aslogs_dir / 'submit_local.sh' + submit_platform_script.touch(exist_ok=True) + + return AutosubmitExperiment( + expid=expid, + autosubmit=autosubmit, + exp_path=exp_path, + tmp_dir=exp_tmp_dir, + aslogs_dir=aslogs_dir, + status_dir=status_dir, + platform=platform + ) + + def finalizer(): + BasicConfig.LOCAL_ROOT_DIR = original_root_dir + rmtree(tmp_path) + + request.addfinalizer(finalizer) + + return _create_autosubmit_exp + + +@pytest.fixture(scope='module') +def autosubmit() -> Autosubmit: + """Create an instance of ``Autosubmit``. + + Useful when you need ``Autosubmit`` but do not need any experiments.""" + autosubmit = Autosubmit() + return autosubmit + + +@pytest.fixture(scope='function') +def create_as_conf() -> Callable: + def _create_as_conf(autosubmit_exp: AutosubmitExperiment, yaml_files: List[Path], experiment_data: Dict[str, Any]): + basic_config = BasicConfig + parser_factory = YAMLParserFactory() + as_conf = AutosubmitConfig( + expid=autosubmit_exp.expid, + basic_config=basic_config, + parser_factory=parser_factory + ) + for yaml_file in yaml_files: + parser = parser_factory.create_parser() + parser.data = parser.load(yaml_file) + as_conf.experiment_data.update(parser.data) + # add user-provided experiment data + as_conf.experiment_data.update(experiment_data) + return as_conf + + return _create_as_conf \ No newline at end of file diff --git a/test/unit/test_copy_code_Path.py b/test/unit/test_copy_code_Path.py new file mode 100644 index 00000000..1d2d74c7 --- /dev/null +++ b/test/unit/test_copy_code_Path.py @@ -0,0 +1,35 @@ +import pytest +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Callable, List +from tempfile import TemporaryDirectory + +from autosubmit.autosubmit import Autosubmit +from autosubmit.platforms.slurmplatform import SlurmPlatform, ParamikoPlatform +from autosubmitconfigparser.config.basicconfig import BasicConfig +from autosubmitconfigparser.config.configcommon import AutosubmitConfig +from autosubmitconfigparser.config.yamlparser import YAMLParserFactory + +from test.unit.conftest import AutosubmitExperiment, autosubmit_exp + +@pytest.fixture +def fake_description(): + return "test descript" + + +## define project:project_type: 'local' + + # to check project:project_destination + # to check local:project_path + #empty project_path + #local_project_path not valid && empty + + #project_path exist + #local_destination exist + #local_destination no exist + #check "cp -R " + local_project_path + "/* " + local_destination + #check no possible to cp + #project_path not exist + #check mkdirs project_path & local_destination (single folder + parent folder) + + #check what happens when any dir already exist on the cp functions -- GitLab From f7a367d3b9121388370fad61e4a3479d1c66d858 Mon Sep 17 00:00:00 2001 From: Edgar Garriga Date: Fri, 30 Aug 2024 12:23:00 +0200 Subject: [PATCH 4/4] cleaned code + fix corner case -> need to import re --- autosubmit/autosubmit.py | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index d28d1e39..fdf804c1 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -18,6 +18,7 @@ import collections import locale import platform import requests +import re # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . import threading @@ -4677,7 +4678,6 @@ class Autosubmit: :return: True if successful, False if not :rtype: bool """ - Log.info(" **** INSIDE _copy_code") project_destination = as_conf.get_project_destination() if project_destination is None or len(project_destination) == 0: if project_type.lower() != "none": @@ -4732,27 +4732,19 @@ class Autosubmit: Log.debug("{0}", output) elif project_type == "local": - Log.info(" **** project_type : local") local_project_path = as_conf.get_local_project_path() if local_project_path is None or len(local_project_path) == 0: raise AutosubmitCritical("Empty project path! please change this parameter to a valid one.", 7014) - ### check if local_project_path is a valid path + # check if local_project_path is a valid path if not Path(local_project_path).is_dir(): raise AutosubmitCritical("Local project path is not a valid path and/or it doesnt exist.", 7014) - ### - #project_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, BasicConfig.LOCAL_PROJ_DIR) - #local_destination = os.path.join(project_path, project_destination) - project_path = BasicConfig.LOCAL_ROOT_DIR.joinpath(expid,BasicConfig.LOCAL_PROJ_DIR) + + project_path = Path(BasicConfig.LOCAL_ROOT_DIR).joinpath(expid,BasicConfig.LOCAL_PROJ_DIR) local_destination = project_path.joinpath(project_destination) - Log.info(" **** project_type : local check vars: \n{0} - local_project_path (as_conf)\n{2} - project_path (basic config) \n{3} - project_destination (as_conf.get_project_destination)\n{1} - local_destination (project_path + project_destination)\n\n",local_project_path,local_destination,project_path,project_destination) - #if os.path.exists(project_path): if project_path.exists(): - Log.info(" **** if path exist: project path") Log.info("Using project folder: {0}", project_path) - #if os.path.exists(local_destination): - if local_destination.exists(): - Log.info(" **** if path exist: local destination") + if local_destination.is_dir(): # it will accept a dir with ext name... but we asume that it is already created, we should accept it, is it? if force: try: cmd = ["rsync -ach --info=progress2 " + @@ -4762,13 +4754,17 @@ class Autosubmit: raise AutosubmitCritical("Can not rsync {0} into {1}. Exiting...".format( local_project_path, project_path), 7063) else: - Log.info(" **** else path exist: local destination") - # os.makedirs(local_destination) #to create intermediate dirs #os.mkdir(local_destination) + invalid_char_pattern = re.compile(r'[^a-zA-Z0-9/_\\-]') + # check local_destination is a well formet path & create it + # if local_destination.suffix : + # raise AutosubmitCritical("Local destination path is not a valid path: ", 7014, local_destination) + if invalid_char_pattern.search(str(local_destination)): + raise AutosubmitCritical("Local destination path contains invalid characters ", 7014) + Path(local_destination).mkdir(parents=True) #this will create parents dirs if its needed try: - Log.info(" **** try cp -R {0} & {1}",local_project_path, local_destination) output = subprocess.check_output( - "cp -R " + local_project_path + "/* " + local_destination, shell=True) + "cp -R " + str(local_project_path) + "/* " + str(local_destination), shell=True) except subprocess.CalledProcessError: try: shutil.rmtree(project_path) @@ -4777,18 +4773,15 @@ class Autosubmit: raise AutosubmitCritical("Can not copy {0} into {1}. Exiting...".format( local_project_path, project_path), 7063) else: - Log.info(" **** else path exist: project path") - # os.makedirs(xxxx) #to create intermediate dirs - Path(project_path).mkdir(parents=True) #this will vreate parents dirs if its needed + Path(project_path).mkdir(parents=True) Path(local_destination).mkdir(parents=True) Log.debug( "The project folder {0} has been created.", project_path) Log.info("Copying {0} into {1}", local_project_path, project_path) try: - Log.info(" **** else try cp -R {0} & {1}",local_project_path, local_destination) output = subprocess.check_output( - "cp -R " + local_project_path + "/* " + local_destination, shell=True) + "cp -R " + str(local_project_path) + "/* " + str(local_destination), shell=True) except subprocess.CalledProcessError: try: shutil.rmtree(project_path) -- GitLab