diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index edbf413417b2c932fcd07f068c5879522b8a5773..fdf804c107fb59245c7ff370a22dbbd7b529ea2e 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -18,6 +18,7 @@ import collections import locale import platform import requests +import re # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . import threading @@ -4677,7 +4678,6 @@ class Autosubmit: :return: True if successful, False if not :rtype: bool """ - project_destination = as_conf.get_project_destination() if project_destination is None or len(project_destination) == 0: if project_type.lower() != "none": @@ -4735,13 +4735,16 @@ class Autosubmit: local_project_path = as_conf.get_local_project_path() if local_project_path is None or len(local_project_path) == 0: raise AutosubmitCritical("Empty project path! please change this parameter to a valid one.", 7014) - project_path = os.path.join( - BasicConfig.LOCAL_ROOT_DIR, expid, BasicConfig.LOCAL_PROJ_DIR) - local_destination = os.path.join(project_path, project_destination) + # check if local_project_path is a valid path + if not Path(local_project_path).is_dir(): + raise AutosubmitCritical("Local project path is not a valid path and/or it doesnt exist.", 7014) + + project_path = Path(BasicConfig.LOCAL_ROOT_DIR).joinpath(expid,BasicConfig.LOCAL_PROJ_DIR) + local_destination = project_path.joinpath(project_destination) - if os.path.exists(project_path): + if project_path.exists(): Log.info("Using project folder: {0}", project_path) - if os.path.exists(local_destination): + if local_destination.is_dir(): # it will accept a dir with ext name... but we asume that it is already created, we should accept it, is it? if force: try: cmd = ["rsync -ach --info=progress2 " + @@ -4751,10 +4754,17 @@ class Autosubmit: raise AutosubmitCritical("Can not rsync {0} into {1}. Exiting...".format( local_project_path, project_path), 7063) else: - os.mkdir(local_destination) + invalid_char_pattern = re.compile(r'[^a-zA-Z0-9/_\\-]') + # check local_destination is a well formet path & create it + # if local_destination.suffix : + # raise AutosubmitCritical("Local destination path is not a valid path: ", 7014, local_destination) + if invalid_char_pattern.search(str(local_destination)): + raise AutosubmitCritical("Local destination path contains invalid characters ", 7014) + + Path(local_destination).mkdir(parents=True) #this will create parents dirs if its needed try: output = subprocess.check_output( - "cp -R " + local_project_path + "/* " + local_destination, shell=True) + "cp -R " + str(local_project_path) + "/* " + str(local_destination), shell=True) except subprocess.CalledProcessError: try: shutil.rmtree(project_path) @@ -4763,15 +4773,15 @@ class Autosubmit: raise AutosubmitCritical("Can not copy {0} into {1}. Exiting...".format( local_project_path, project_path), 7063) else: - os.mkdir(project_path) - os.mkdir(local_destination) + Path(project_path).mkdir(parents=True) + Path(local_destination).mkdir(parents=True) Log.debug( "The project folder {0} has been created.", project_path) Log.info("Copying {0} into {1}", local_project_path, project_path) try: output = subprocess.check_output( - "cp -R " + local_project_path + "/* " + local_destination, shell=True) + "cp -R " + str(local_project_path) + "/* " + str(local_destination), shell=True) except subprocess.CalledProcessError: try: shutil.rmtree(project_path) diff --git a/test/unit/conftest.py b/test/unit/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..9f06fe1924de85df3b499099f342840db3c3c628 --- /dev/null +++ b/test/unit/conftest.py @@ -0,0 +1,109 @@ +# Fixtures available to multiple test files must be created in this file. + +import pytest +from dataclasses import dataclass +from pathlib import Path +from shutil import rmtree +from tempfile import TemporaryDirectory +from typing import Any, Dict, Callable, List + +from autosubmit.autosubmit import Autosubmit +from autosubmit.platforms.slurmplatform import SlurmPlatform, ParamikoPlatform +from autosubmitconfigparser.config.basicconfig import BasicConfig +from autosubmitconfigparser.config.configcommon import AutosubmitConfig +from autosubmitconfigparser.config.yamlparser import YAMLParserFactory + + +@dataclass +class AutosubmitExperiment: + """This holds information about an experiment created by Autosubmit.""" + expid: str + autosubmit: Autosubmit + exp_path: Path + tmp_dir: Path + aslogs_dir: Path + status_dir: Path + platform: ParamikoPlatform + + +@pytest.fixture(scope='function') +def autosubmit_exp(autosubmit: Autosubmit, request: pytest.FixtureRequest) -> Callable: + """Create an instance of ``Autosubmit`` with an experiment.""" + + original_root_dir = BasicConfig.LOCAL_ROOT_DIR + tmp_dir = TemporaryDirectory() + tmp_path = Path(tmp_dir.name) + + def _create_autosubmit_exp(expid: str): + # directories used when searching for logs to cat + root_dir = tmp_path + BasicConfig.LOCAL_ROOT_DIR = str(root_dir) + exp_path = root_dir / expid + exp_tmp_dir = exp_path / BasicConfig.LOCAL_TMP_DIR + aslogs_dir = exp_tmp_dir / BasicConfig.LOCAL_ASLOG_DIR + status_dir = exp_path / 'status' + aslogs_dir.mkdir(parents=True) + status_dir.mkdir() + + platform_config = { + "LOCAL_ROOT_DIR": BasicConfig.LOCAL_ROOT_DIR, + "LOCAL_TMP_DIR": str(exp_tmp_dir), + "LOCAL_ASLOG_DIR": str(aslogs_dir) + } + platform = SlurmPlatform(expid=expid, name='slurm_platform', config=platform_config) + platform.job_status = { + 'COMPLETED': [], + 'RUNNING': [], + 'QUEUING': [], + 'FAILED': [] + } + submit_platform_script = aslogs_dir / 'submit_local.sh' + submit_platform_script.touch(exist_ok=True) + + return AutosubmitExperiment( + expid=expid, + autosubmit=autosubmit, + exp_path=exp_path, + tmp_dir=exp_tmp_dir, + aslogs_dir=aslogs_dir, + status_dir=status_dir, + platform=platform + ) + + def finalizer(): + BasicConfig.LOCAL_ROOT_DIR = original_root_dir + rmtree(tmp_path) + + request.addfinalizer(finalizer) + + return _create_autosubmit_exp + + +@pytest.fixture(scope='module') +def autosubmit() -> Autosubmit: + """Create an instance of ``Autosubmit``. + + Useful when you need ``Autosubmit`` but do not need any experiments.""" + autosubmit = Autosubmit() + return autosubmit + + +@pytest.fixture(scope='function') +def create_as_conf() -> Callable: + def _create_as_conf(autosubmit_exp: AutosubmitExperiment, yaml_files: List[Path], experiment_data: Dict[str, Any]): + basic_config = BasicConfig + parser_factory = YAMLParserFactory() + as_conf = AutosubmitConfig( + expid=autosubmit_exp.expid, + basic_config=basic_config, + parser_factory=parser_factory + ) + for yaml_file in yaml_files: + parser = parser_factory.create_parser() + parser.data = parser.load(yaml_file) + as_conf.experiment_data.update(parser.data) + # add user-provided experiment data + as_conf.experiment_data.update(experiment_data) + return as_conf + + return _create_as_conf \ No newline at end of file diff --git a/test/unit/test_copy_code_Path.py b/test/unit/test_copy_code_Path.py new file mode 100644 index 0000000000000000000000000000000000000000..1d2d74c7f788243c84d75f5172dba9c60aee5e7d --- /dev/null +++ b/test/unit/test_copy_code_Path.py @@ -0,0 +1,35 @@ +import pytest +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Callable, List +from tempfile import TemporaryDirectory + +from autosubmit.autosubmit import Autosubmit +from autosubmit.platforms.slurmplatform import SlurmPlatform, ParamikoPlatform +from autosubmitconfigparser.config.basicconfig import BasicConfig +from autosubmitconfigparser.config.configcommon import AutosubmitConfig +from autosubmitconfigparser.config.yamlparser import YAMLParserFactory + +from test.unit.conftest import AutosubmitExperiment, autosubmit_exp + +@pytest.fixture +def fake_description(): + return "test descript" + + +## define project:project_type: 'local' + + # to check project:project_destination + # to check local:project_path + #empty project_path + #local_project_path not valid && empty + + #project_path exist + #local_destination exist + #local_destination no exist + #check "cp -R " + local_project_path + "/* " + local_destination + #check no possible to cp + #project_path not exist + #check mkdirs project_path & local_destination (single folder + parent folder) + + #check what happens when any dir already exist on the cp functions