From c61f986b39c28da6700263e53d4942612bf0b954 Mon Sep 17 00:00:00 2001 From: Irene Simo Munoz Date: Wed, 9 Oct 2024 10:02:17 +0200 Subject: [PATCH 01/10] Switch from pkg_resources to importlib commands --- autosubmit/autosubmit.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 6bc39f862..1fd38b6f2 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -76,8 +76,8 @@ import signal import datetime # import log.fd_show as fd_show import portalocker -from pkg_resources import require, resource_listdir, resource_string, resource_filename -#import importlib +# from pkg_resources import require, resource_listdir, resource_string, resource_filename +import importlib from collections import defaultdict from pyparsing import nestedExpr from .history.experiment_status import ExperimentStatus @@ -164,8 +164,8 @@ class Autosubmit: with open(version_path) as f: autosubmit_version = f.read().strip() else: - #autosubmit_version = importlib.metadata.version("autosubmit")[0] - autosubmit_version = require("autosubmit")[0].version + autosubmit_version = importlib.metadata.version("autosubmit")[0] + # autosubmit_version = require("autosubmit")[0].version exit = False @@ -1166,16 +1166,15 @@ class Autosubmit: yield '.'.join([*keys, key]).upper(), value else: yield key, value - template_files = resource_listdir('autosubmitconfigparser.config', 'files') - # template_files = [file.name for file in importlib.resources.files('autosubmitconfigparser.config').joinpath('files').iterdir() if - # file.is_file()] + # template_files = resource_listdir('autosubmitconfigparser.config', 'files') + template_files = [file.name for file in importlib.resources.files('autosubmitconfigparser.config').joinpath('files').iterdir() if file.is_file()] if parameters is None: parameters = PARAMETERS parameter_comments = dict(_recurse_into_parameters(parameters)) for as_conf_file in template_files: - origin = resource_filename('autosubmitconfigparser.config', str(Path('files', as_conf_file))) - #origin = importlib.resources.files('autosubmitconfigparser.config').joinpath('files', as_conf_file) + # origin = resource_filename('autosubmitconfigparser.config', str(Path('files', as_conf_file))) + origin = importlib.resources.files('autosubmitconfigparser.config').joinpath('files', as_conf_file) target = None if dummy: @@ -3775,8 +3774,8 @@ class Autosubmit: """ if not os.path.exists(BasicConfig.DB_PATH): Log.info("Creating autosubmit database...") - qry = resource_string('autosubmit.database', 'data/autosubmit.sql').decode(locale.getlocale()[1]) - #qry = importlib.resources.read_text('autosubmit.database', 'data/autosubmit.sql').decode(locale.getlocale()[1]) + # qry = resource_string('autosubmit.database', 'data/autosubmit.sql').decode(locale.getlocale()[1]) + qry = importlib.resources.read_text('autosubmit.database', 'data/autosubmit.sql').decode(locale.getlocale()[1]) if not create_db(qry): raise AutosubmitCritical("Can not write database file", 7004) Log.result("Autosubmit database created successfully") -- GitLab From 89b571a6a34329eeb6cb7861deebb7859ce15954 Mon Sep 17 00:00:00 2001 From: Irene Simo Munoz Date: Wed, 9 Oct 2024 12:44:49 +0200 Subject: [PATCH 02/10] Add importlib-resources requirement Iss1432 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index e012e0ffd..6ad49396c 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,7 @@ with open(path.join(here, 'VERSION')) as f: install_requires = [ 'xlib==0.21', 'setuptools<=68.2.2', + 'importlib-resources>=3.2.0', 'bscearth.utils<=0.5.2', 'requests<=2.31.0', 'networkx<=2.6.3', -- GitLab From b3a8dd52d5f3b45f627a8d5805ea134622810793 Mon Sep 17 00:00:00 2001 From: Irene Simo Munoz Date: Thu, 10 Oct 2024 15:46:08 +0200 Subject: [PATCH 03/10] Fixed bugs #1432 Made sure lines with importlib do the same as previous lines that used pkg_resources --- autosubmit/autosubmit.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 1fd38b6f2..8d6013800 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -77,6 +77,7 @@ import datetime # import log.fd_show as fd_show import portalocker # from pkg_resources import require, resource_listdir, resource_string, resource_filename +import importlib_metadata import importlib from collections import defaultdict from pyparsing import nestedExpr @@ -164,7 +165,7 @@ class Autosubmit: with open(version_path) as f: autosubmit_version = f.read().strip() else: - autosubmit_version = importlib.metadata.version("autosubmit")[0] + autosubmit_version = importlib_metadata.version("autosubmit") # autosubmit_version = require("autosubmit")[0].version exit = False @@ -1174,7 +1175,7 @@ class Autosubmit: for as_conf_file in template_files: # origin = resource_filename('autosubmitconfigparser.config', str(Path('files', as_conf_file))) - origin = importlib.resources.files('autosubmitconfigparser.config').joinpath('files', as_conf_file) + origin = str(importlib.resources.files('autosubmitconfigparser.config').joinpath('files', as_conf_file)) target = None if dummy: -- GitLab From 77e8fac34c17964442eb8622089a4bb1e1c62f4f Mon Sep 17 00:00:00 2001 From: Irene Simo Munoz Date: Thu, 10 Oct 2024 15:49:12 +0200 Subject: [PATCH 04/10] Added extra requirement --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 6ad49396c..032888dfc 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ install_requires = [ 'xlib==0.21', 'setuptools<=68.2.2', 'importlib-resources>=3.2.0', + 'importlib-metadata>=8.5.0', 'bscearth.utils<=0.5.2', 'requests<=2.31.0', 'networkx<=2.6.3', -- GitLab From b01c18a3c7904cb8905ad096f61398ecbaec8d7b Mon Sep 17 00:00:00 2001 From: Irene Simo Munoz Date: Thu, 10 Oct 2024 16:36:39 +0200 Subject: [PATCH 05/10] Clean and remove pkg_resources old lines --- autosubmit/autosubmit.py | 5 ----- setup.py | 1 - 2 files changed, 6 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 8d6013800..6d9519183 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -76,7 +76,6 @@ import signal import datetime # import log.fd_show as fd_show import portalocker -# from pkg_resources import require, resource_listdir, resource_string, resource_filename import importlib_metadata import importlib from collections import defaultdict @@ -166,7 +165,6 @@ class Autosubmit: autosubmit_version = f.read().strip() else: autosubmit_version = importlib_metadata.version("autosubmit") - # autosubmit_version = require("autosubmit")[0].version exit = False @@ -1167,14 +1165,12 @@ class Autosubmit: yield '.'.join([*keys, key]).upper(), value else: yield key, value - # template_files = resource_listdir('autosubmitconfigparser.config', 'files') template_files = [file.name for file in importlib.resources.files('autosubmitconfigparser.config').joinpath('files').iterdir() if file.is_file()] if parameters is None: parameters = PARAMETERS parameter_comments = dict(_recurse_into_parameters(parameters)) for as_conf_file in template_files: - # origin = resource_filename('autosubmitconfigparser.config', str(Path('files', as_conf_file))) origin = str(importlib.resources.files('autosubmitconfigparser.config').joinpath('files', as_conf_file)) target = None @@ -3775,7 +3771,6 @@ class Autosubmit: """ if not os.path.exists(BasicConfig.DB_PATH): Log.info("Creating autosubmit database...") - # qry = resource_string('autosubmit.database', 'data/autosubmit.sql').decode(locale.getlocale()[1]) qry = importlib.resources.read_text('autosubmit.database', 'data/autosubmit.sql').decode(locale.getlocale()[1]) if not create_db(qry): raise AutosubmitCritical("Can not write database file", 7004) diff --git a/setup.py b/setup.py index 032888dfc..51149c7f6 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,6 @@ with open(path.join(here, 'VERSION')) as f: install_requires = [ 'xlib==0.21', 'setuptools<=68.2.2', - 'importlib-resources>=3.2.0', 'importlib-metadata>=8.5.0', 'bscearth.utils<=0.5.2', 'requests<=2.31.0', -- GitLab From 0f24ca90fe2e6e2702ced1d9ddfa99efab112cc8 Mon Sep 17 00:00:00 2001 From: Irene Simo Munoz Date: Fri, 11 Oct 2024 09:14:07 +0200 Subject: [PATCH 06/10] Fix importlib-metadata requirement --- autosubmit/autosubmit.py | 4 ++-- setup.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 6d9519183..8c880853b 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -76,8 +76,8 @@ import signal import datetime # import log.fd_show as fd_show import portalocker -import importlib_metadata import importlib +from importlib.metadata import version from collections import defaultdict from pyparsing import nestedExpr from .history.experiment_status import ExperimentStatus @@ -164,7 +164,7 @@ class Autosubmit: with open(version_path) as f: autosubmit_version = f.read().strip() else: - autosubmit_version = importlib_metadata.version("autosubmit") + autosubmit_version = version("autosubmit") # from importlib.metadata exit = False diff --git a/setup.py b/setup.py index 51149c7f6..e012e0ffd 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,6 @@ with open(path.join(here, 'VERSION')) as f: install_requires = [ 'xlib==0.21', 'setuptools<=68.2.2', - 'importlib-metadata>=8.5.0', 'bscearth.utils<=0.5.2', 'requests<=2.31.0', 'networkx<=2.6.3', -- GitLab From 4be6aef31d5a6e43db6080e14c1472995bc8e36c Mon Sep 17 00:00:00 2001 From: Irene Simo Munoz Date: Fri, 11 Oct 2024 09:47:42 +0200 Subject: [PATCH 07/10] Improve import statements and path creation --- autosubmit/autosubmit.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 8c880853b..65b3809ba 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -76,7 +76,7 @@ import signal import datetime # import log.fd_show as fd_show import portalocker -import importlib +from importlib.reosurces import read_text, files as read_files from importlib.metadata import version from collections import defaultdict from pyparsing import nestedExpr @@ -1165,13 +1165,13 @@ class Autosubmit: yield '.'.join([*keys, key]).upper(), value else: yield key, value - template_files = [file.name for file in importlib.resources.files('autosubmitconfigparser.config').joinpath('files').iterdir() if file.is_file()] + template_files = [file.name for file in (read_files('autosubmitconfigparser.config')/'files').iterdir() if file.is_file()] if parameters is None: parameters = PARAMETERS parameter_comments = dict(_recurse_into_parameters(parameters)) for as_conf_file in template_files: - origin = str(importlib.resources.files('autosubmitconfigparser.config').joinpath('files', as_conf_file)) + origin = str(read_files('autosubmitconfigparser.config')/f'files/{as_conf_file}') target = None if dummy: @@ -3771,7 +3771,7 @@ class Autosubmit: """ if not os.path.exists(BasicConfig.DB_PATH): Log.info("Creating autosubmit database...") - qry = importlib.resources.read_text('autosubmit.database', 'data/autosubmit.sql').decode(locale.getlocale()[1]) + qry = read_text('autosubmit.database', 'data/autosubmit.sql').decode(locale.getlocale()[1]) if not create_db(qry): raise AutosubmitCritical("Can not write database file", 7004) Log.result("Autosubmit database created successfully") -- GitLab From 035bfa83c7317a98c17c8bf70eb6dbbfaa139094 Mon Sep 17 00:00:00 2001 From: Irene Simo Munoz Date: Fri, 11 Oct 2024 09:55:30 +0200 Subject: [PATCH 08/10] Fix typo --- autosubmit/autosubmit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 65b3809ba..9b298b6e9 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -76,7 +76,7 @@ import signal import datetime # import log.fd_show as fd_show import portalocker -from importlib.reosurces import read_text, files as read_files +from importlib.resources import read_text, files as read_files from importlib.metadata import version from collections import defaultdict from pyparsing import nestedExpr -- GitLab From 85e1499b3f5a8c6937138a65df559c664d6fc0bc Mon Sep 17 00:00:00 2001 From: Irene Simo Munoz Date: Mon, 14 Oct 2024 09:43:56 +0200 Subject: [PATCH 09/10] qry fix for pipeline failure? --- autosubmit/autosubmit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 9b298b6e9..c8558c404 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -3771,7 +3771,7 @@ class Autosubmit: """ if not os.path.exists(BasicConfig.DB_PATH): Log.info("Creating autosubmit database...") - qry = read_text('autosubmit.database', 'data/autosubmit.sql').decode(locale.getlocale()[1]) + qry = read_files('autosubmit.database').joinpath('data/autosubmit.sql').read_text(locale.getlocale()[1]) if not create_db(qry): raise AutosubmitCritical("Can not write database file", 7004) Log.result("Autosubmit database created successfully") -- GitLab From ef2ee14bf3d1dfe4c59522115a791c822b0ea83f Mon Sep 17 00:00:00 2001 From: Irene Simo Munoz Date: Fri, 18 Oct 2024 14:51:15 +0200 Subject: [PATCH 10/10] Fix CICD pipeline failure for vim __init__.py Pipeline was failing because of unit test mocking old (from pkg_resources) functions. Updated the expid test to mock only currently used functions, i.e. removed from available attributes and made the necessary tweaks to ensure is tested as was. --- test/unit/test_expid.py | 89 ++++++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 42 deletions(-) diff --git a/test/unit/test_expid.py b/test/unit/test_expid.py index fd789aa49..7bd4affc8 100644 --- a/test/unit/test_expid.py +++ b/test/unit/test_expid.py @@ -1,4 +1,5 @@ import tempfile +import os from unittest import TestCase from mock import Mock, patch from autosubmit.autosubmit import Autosubmit @@ -61,58 +62,60 @@ class TestExpid(TestCase): mock_db_common.last_name_used = Mock(return_value=current_experiment_id) mock_db_common.check_experiment_exists = Mock(return_value=False) - @patch('autosubmit.autosubmit.resource_listdir') - @patch('autosubmit.autosubmit.resource_filename') - def test_autosubmit_generate_config(self, resource_filename_mock, resource_listdir_mock): + @patch('autosubmit.autosubmit.read_files') + def test_autosubmit_generate_config(self, read_files_mock): expid = 'ff99' original_local_root_dir = BasicConfig.LOCAL_ROOT_DIR + + with tempfile.TemporaryDirectory() as temp_dir: + temp_file_path = os.path.join(temp_dir, 'files') + os.makedirs(temp_file_path) + + with tempfile.NamedTemporaryFile(dir=temp_file_path, suffix='.yaml', mode='w') as source_yaml: + # Our processed and commented YAML output file must be written here + Path(temp_dir, expid, 'conf').mkdir(parents=True) + BasicConfig.LOCAL_ROOT_DIR = temp_dir - with tempfile.NamedTemporaryFile(suffix='.yaml', mode='w') as source_yaml, tempfile.TemporaryDirectory() as temp_dir: - # Our processed and commented YAML output file must be written here - Path(temp_dir, expid, 'conf').mkdir(parents=True) - BasicConfig.LOCAL_ROOT_DIR = temp_dir - - source_yaml.write( -dedent('''JOB: - JOBNAME: SIM - PLATFORM: local -CONFIG: - TEST: The answer? - ROOT: No''')) - source_yaml.flush() - resource_listdir_mock.return_value = [Path(source_yaml.name).name] - resource_filename_mock.return_value = source_yaml.name - - parameters = { - 'JOB': { - 'JOBNAME': 'sim' - }, - 'CONFIG': { - 'CONFIG.TEST': '42' + source_yaml.write( + dedent(''' + JOB: + JOBNAME: SIM + PLATFORM: local + CONFIG: + TEST: The answer? + ROOT: No''')) + source_yaml.flush() + read_files_mock.return_value = Path(temp_dir) + + parameters = { + 'JOB': { + 'JOBNAME': 'sim' + }, + 'CONFIG': { + 'CONFIG.TEST': '42' + } } - } - Autosubmit.generate_as_config(exp_id=expid, parameters=parameters) + Autosubmit.generate_as_config(exp_id=expid, parameters=parameters) - source_text = Path(source_yaml.name).read_text() - source_name = Path(source_yaml.name) - output_text = Path(temp_dir, expid, 'conf', f'{source_name.stem}_{expid}.yml').read_text() + source_text = Path(source_yaml.name).read_text() + source_name = Path(source_yaml.name) + output_text = Path(temp_dir, expid, 'conf', f'{source_name.stem}_{expid}.yml').read_text() - self.assertNotEqual(source_text, output_text) - self.assertFalse('# sim' in source_text) - self.assertTrue('# sim' in output_text) - self.assertFalse('# 42' in source_text) - self.assertTrue('# 42' in output_text) + self.assertNotEqual(source_text, output_text) + self.assertFalse('# sim' in source_text) + self.assertTrue('# sim' in output_text) + self.assertFalse('# 42' in source_text) + print(output_text) + self.assertTrue('# 42' in output_text) # Reset the local root dir. BasicConfig.LOCAL_ROOT_DIR = original_local_root_dir @patch('autosubmit.autosubmit.YAML.dump') - @patch('autosubmit.autosubmit.resource_listdir') - @patch('autosubmit.autosubmit.resource_filename') + @patch('autosubmit.autosubmit.read_files') def test_autosubmit_generate_config_resource_listdir_order( self, - resource_filename_mock, - resource_listdir_mock, + read_files_mock, yaml_mock ) -> None: """ @@ -150,13 +153,16 @@ CONFIG: with tempfile.TemporaryDirectory() as temp_dir: Path(temp_dir, expid, 'conf').mkdir(parents=True) + temp_file_path = os.path.join(temp_dir, 'files') + os.makedirs(temp_file_path) + BasicConfig.LOCAL_ROOT_DIR = temp_dir resources_return = [] filenames_return = [] for file_name in test['resources']: - input_path = Path(temp_dir, file_name) + input_path = Path(temp_file_path, file_name) with open(input_path, 'w+') as source_yaml: source_yaml.write('TEST: YES') @@ -165,8 +171,7 @@ CONFIG: resources_return.append(input_path.name) # path filenames_return.append(source_yaml.name) # textiowrapper - resource_listdir_mock.return_value = resources_return - resource_filename_mock.side_effect = filenames_return + read_files_mock.return_value = Path(temp_dir) Autosubmit.generate_as_config( exp_id=expid, -- GitLab