diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 6bc39f8627db2dba04db63e5d975ad57b756e3e4..c8558c4042f378755b95965db471bac4a2b6e3cf 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -76,8 +76,8 @@ import signal import datetime # import log.fd_show as fd_show import portalocker -from pkg_resources import require, resource_listdir, resource_string, resource_filename -#import importlib +from importlib.resources import read_text, files as read_files +from importlib.metadata import version from collections import defaultdict from pyparsing import nestedExpr from .history.experiment_status import ExperimentStatus @@ -164,8 +164,7 @@ class Autosubmit: with open(version_path) as f: autosubmit_version = f.read().strip() else: - #autosubmit_version = importlib.metadata.version("autosubmit")[0] - autosubmit_version = require("autosubmit")[0].version + autosubmit_version = version("autosubmit") # from importlib.metadata exit = False @@ -1166,16 +1165,13 @@ class Autosubmit: yield '.'.join([*keys, key]).upper(), value else: yield key, value - template_files = resource_listdir('autosubmitconfigparser.config', 'files') - # template_files = [file.name for file in importlib.resources.files('autosubmitconfigparser.config').joinpath('files').iterdir() if - # file.is_file()] + template_files = [file.name for file in (read_files('autosubmitconfigparser.config')/'files').iterdir() if file.is_file()] if parameters is None: parameters = PARAMETERS parameter_comments = dict(_recurse_into_parameters(parameters)) for as_conf_file in template_files: - origin = resource_filename('autosubmitconfigparser.config', str(Path('files', as_conf_file))) - #origin = importlib.resources.files('autosubmitconfigparser.config').joinpath('files', as_conf_file) + origin = str(read_files('autosubmitconfigparser.config')/f'files/{as_conf_file}') target = None if dummy: @@ -3775,8 +3771,7 @@ class Autosubmit: """ if not os.path.exists(BasicConfig.DB_PATH): Log.info("Creating autosubmit database...") - qry = resource_string('autosubmit.database', 'data/autosubmit.sql').decode(locale.getlocale()[1]) - #qry = importlib.resources.read_text('autosubmit.database', 'data/autosubmit.sql').decode(locale.getlocale()[1]) + qry = read_files('autosubmit.database').joinpath('data/autosubmit.sql').read_text(locale.getlocale()[1]) if not create_db(qry): raise AutosubmitCritical("Can not write database file", 7004) Log.result("Autosubmit database created successfully") diff --git a/test/unit/test_expid.py b/test/unit/test_expid.py index fd789aa49c8fdfdaab8affaa92419cf293ab5298..7bd4affc881a523a1a0a00047ac6f546bc7f12b9 100644 --- a/test/unit/test_expid.py +++ b/test/unit/test_expid.py @@ -1,4 +1,5 @@ import tempfile +import os from unittest import TestCase from mock import Mock, patch from autosubmit.autosubmit import Autosubmit @@ -61,58 +62,60 @@ class TestExpid(TestCase): mock_db_common.last_name_used = Mock(return_value=current_experiment_id) mock_db_common.check_experiment_exists = Mock(return_value=False) - @patch('autosubmit.autosubmit.resource_listdir') - @patch('autosubmit.autosubmit.resource_filename') - def test_autosubmit_generate_config(self, resource_filename_mock, resource_listdir_mock): + @patch('autosubmit.autosubmit.read_files') + def test_autosubmit_generate_config(self, read_files_mock): expid = 'ff99' original_local_root_dir = BasicConfig.LOCAL_ROOT_DIR + + with tempfile.TemporaryDirectory() as temp_dir: + temp_file_path = os.path.join(temp_dir, 'files') + os.makedirs(temp_file_path) + + with tempfile.NamedTemporaryFile(dir=temp_file_path, suffix='.yaml', mode='w') as source_yaml: + # Our processed and commented YAML output file must be written here + Path(temp_dir, expid, 'conf').mkdir(parents=True) + BasicConfig.LOCAL_ROOT_DIR = temp_dir - with tempfile.NamedTemporaryFile(suffix='.yaml', mode='w') as source_yaml, tempfile.TemporaryDirectory() as temp_dir: - # Our processed and commented YAML output file must be written here - Path(temp_dir, expid, 'conf').mkdir(parents=True) - BasicConfig.LOCAL_ROOT_DIR = temp_dir - - source_yaml.write( -dedent('''JOB: - JOBNAME: SIM - PLATFORM: local -CONFIG: - TEST: The answer? - ROOT: No''')) - source_yaml.flush() - resource_listdir_mock.return_value = [Path(source_yaml.name).name] - resource_filename_mock.return_value = source_yaml.name - - parameters = { - 'JOB': { - 'JOBNAME': 'sim' - }, - 'CONFIG': { - 'CONFIG.TEST': '42' + source_yaml.write( + dedent(''' + JOB: + JOBNAME: SIM + PLATFORM: local + CONFIG: + TEST: The answer? + ROOT: No''')) + source_yaml.flush() + read_files_mock.return_value = Path(temp_dir) + + parameters = { + 'JOB': { + 'JOBNAME': 'sim' + }, + 'CONFIG': { + 'CONFIG.TEST': '42' + } } - } - Autosubmit.generate_as_config(exp_id=expid, parameters=parameters) + Autosubmit.generate_as_config(exp_id=expid, parameters=parameters) - source_text = Path(source_yaml.name).read_text() - source_name = Path(source_yaml.name) - output_text = Path(temp_dir, expid, 'conf', f'{source_name.stem}_{expid}.yml').read_text() + source_text = Path(source_yaml.name).read_text() + source_name = Path(source_yaml.name) + output_text = Path(temp_dir, expid, 'conf', f'{source_name.stem}_{expid}.yml').read_text() - self.assertNotEqual(source_text, output_text) - self.assertFalse('# sim' in source_text) - self.assertTrue('# sim' in output_text) - self.assertFalse('# 42' in source_text) - self.assertTrue('# 42' in output_text) + self.assertNotEqual(source_text, output_text) + self.assertFalse('# sim' in source_text) + self.assertTrue('# sim' in output_text) + self.assertFalse('# 42' in source_text) + print(output_text) + self.assertTrue('# 42' in output_text) # Reset the local root dir. BasicConfig.LOCAL_ROOT_DIR = original_local_root_dir @patch('autosubmit.autosubmit.YAML.dump') - @patch('autosubmit.autosubmit.resource_listdir') - @patch('autosubmit.autosubmit.resource_filename') + @patch('autosubmit.autosubmit.read_files') def test_autosubmit_generate_config_resource_listdir_order( self, - resource_filename_mock, - resource_listdir_mock, + read_files_mock, yaml_mock ) -> None: """ @@ -150,13 +153,16 @@ CONFIG: with tempfile.TemporaryDirectory() as temp_dir: Path(temp_dir, expid, 'conf').mkdir(parents=True) + temp_file_path = os.path.join(temp_dir, 'files') + os.makedirs(temp_file_path) + BasicConfig.LOCAL_ROOT_DIR = temp_dir resources_return = [] filenames_return = [] for file_name in test['resources']: - input_path = Path(temp_dir, file_name) + input_path = Path(temp_file_path, file_name) with open(input_path, 'w+') as source_yaml: source_yaml.write('TEST: YES') @@ -165,8 +171,7 @@ CONFIG: resources_return.append(input_path.name) # path filenames_return.append(source_yaml.name) # textiowrapper - resource_listdir_mock.return_value = resources_return - resource_filename_mock.side_effect = filenames_return + read_files_mock.return_value = Path(temp_dir) Autosubmit.generate_as_config( exp_id=expid,