From 989e74667dd40e7aca3c65df71a22fc3593de226 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Mon, 10 Jun 2024 20:26:18 +0200 Subject: [PATCH 1/2] GL-1320: Fix CICD build, use pytest --- .gitignore | 4 ++- .gitlab-ci.yml | 4 +-- environment.yml | 5 ++-- pytest.ini | 12 ++++++++ requeriments.txt | 2 +- setup.py | 45 +---------------------------- test/unit/test_database_managers.py | 2 ++ test/unit/test_history.py | 4 ++- test/unit/test_migrate.py | 1 + 9 files changed, 27 insertions(+), 52 deletions(-) create mode 100644 pytest.ini diff --git a/.gitignore b/.gitignore index 40e7cb3ec..fe2b46562 100644 --- a/.gitignore +++ b/.gitignore @@ -16,8 +16,10 @@ build/ # Coverage files coverage/ -/.coverage +.coverage test/coverage.xml +coverage.xml +test/htmlcov # Docker dockerfiles/id_rsa* diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f0f0a2895..fc6931e89 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,14 +21,14 @@ prepare: - git submodule update --init --recursive - conda update -n base -c defaults conda - conda update conda - - conda install -n autosubmit3 coverage=6 + - conda install -n autosubmit3 pytest pytest-cov pytest-mock - conda env update -f environment.yml -n autosubmit3 python=3.7.3 test_python3: stage: test script: - conda activate autosubmit3 - - python3 -m 'nose' --exclude=regression --verbosity=3 test/unit --with-coverage --cover-package=autosubmit --cover-inclusive --cover-xml --cover-xml-file=test/coverage.xml + - pytest coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/' # These artifacts are saved with every build in GitLab and can be reviewed later. If # we have a folder with HTML files, as in this example, users can navigate with their diff --git a/environment.yml b/environment.yml index 56b5d33ea..a92b1b01c 100644 --- a/environment.yml +++ b/environment.yml @@ -4,8 +4,6 @@ channels: dependencies: - python=3.7 -- nose -- nose-cov - paramiko - graphviz - python-dateutil @@ -38,8 +36,9 @@ dependencies: - ruamel.yaml - pythondialog - pytest + - pytest-cov + - pytest-mock - nose - - coverage - requests - configobj - psutil diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..5ab9b5fe7 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,12 @@ +[pytest] +addopts = + --verbose --ignore=regression + --cov=autosubmit --cov-config=.coveragerc + --cov-report=html:test/htmlcov --cov-report=xml:test/coverage.xml + --strict-markers +testpaths = + test/unit +doctest_optionflags = + NORMALIZE_WHITESPACE + IGNORE_EXCEPTION_DETAIL + ELLIPSIS diff --git a/requeriments.txt b/requeriments.txt index d93b5fe67..99b2a1419 100644 --- a/requeriments.txt +++ b/requeriments.txt @@ -5,7 +5,6 @@ bscearth.utils<=0.5.2 requests<=2.31.0 networkx<=2.6.3 portalocker<=2.7.0 -mock<=5.1.0 paramiko<=3.4 pyparsing==3.1.1 matplotlib<=3.8.3 @@ -25,4 +24,5 @@ autosubmitconfigparser==1.0.67 pathlib configparser pytest +pytest-cov pytest-mock diff --git a/setup.py b/setup.py index 7e83757fa..3a17bebdc 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,6 @@ setup( 'requests<=2.31.0', 'networkx<=2.6.3', 'portalocker<=2.7.0', - 'mock<=5.1.0', 'paramiko<=3.4', 'pyparsing==3.1.1', 'matplotlib<=3.8.3', @@ -66,52 +65,10 @@ setup( 'configparser', 'pathlib', 'setproctitle' - ], - extras_require={ - ':python_version <= "3.7"': - [ - 'PyNaCl==1.5.0', - 'pythondialog==3.5.3', - 'xlib==0.21', - 'setuptools==68.2.2', - 'cryptography==41.0.5', - 'bscearth.utils==0.5.2', - 'requests==2.31.0', - 'networkx==2.6.3', - 'portalocker==2.7.0', - 'mock==5.1.0', - 'paramiko==3.3.1', - 'matplotlib==3.5.3', - 'python_dateutil==2.8.2', - 'argparse==1.4.0', - 'configobj==5.0.8', - 'packaging==23.2', - 'bcrypt==4.0.1', - 'charset_normalizer==3.3.1', - 'kiwisolver==1.4.5', - 'fonttools==4.43.1', - 'cycler==0.12.1', - 'typing_extensions==4.8.0', - 'psutil==5.6.1', - 'Pygments==2.3.1', - 'coverage==5.0', - 'nose==1.3.7', - 'six==1.12.0', - 'Cython==0.29.6', - 'cffi==1.12.2', - 'py==1.8.0', - 'atomicwrites==1.3.0', - 'attrs==19.1.0', - 'more_itertools==6.0.0', - 'urllib3==1.24.1', - 'idna==2.8', - 'Pillow==6.2.1', - 'numpy==1.17.4', - ], - }, classifiers=[ "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "License :: OSI Approved :: GNU General Public License (GPL)", "Operating System :: POSIX :: Linux", diff --git a/test/unit/test_database_managers.py b/test/unit/test_database_managers.py index 9999fe948..c0bc9d3c0 100644 --- a/test/unit/test_database_managers.py +++ b/test/unit/test_database_managers.py @@ -17,6 +17,7 @@ # along with Autosubmit. If not, see . import unittest +import pytest import time import random import os @@ -35,6 +36,7 @@ BasicConfig.read() JOBDATA_DIR = BasicConfig.JOBDATA_DIR LOCAL_ROOT_DIR = BasicConfig.LOCAL_ROOT_DIR +@pytest.mark.skip() @unittest.skip('TODO: looks like another test that used actual experiments data') class TestExperimentStatusDatabaseManager(unittest.TestCase): """ Covers Experiment Status Database Manager """ diff --git a/test/unit/test_history.py b/test/unit/test_history.py index 68495d8e9..5869ad189 100644 --- a/test/unit/test_history.py +++ b/test/unit/test_history.py @@ -17,6 +17,7 @@ # along with Autosubmit. If not, see . import unittest +import pytest import traceback import os import time @@ -36,6 +37,7 @@ JOBDATA_DIR = BasicConfig.JOBDATA_DIR LOCAL_ROOT_DIR = BasicConfig.LOCAL_ROOT_DIR job = namedtuple("Job", ["name", "date", "member", "status_str", "children"]) +@pytest.mark.skip() @unittest.skip('TODO: another test that uses actual data. See if there is anything useful, and extract into functional/integration/unit tests that run on any machine') class TestExperimentHistory(unittest.TestCase): # @classmethod @@ -289,7 +291,7 @@ class TestExperimentHistory(unittest.TestCase): self.assertTrue(inserted_job_data_dc.qos == "debug") - +@pytest.mark.skip() class TestLogging(unittest.TestCase): def setUp(self): diff --git a/test/unit/test_migrate.py b/test/unit/test_migrate.py index 0dd87f2e8..92f49b5ec 100644 --- a/test/unit/test_migrate.py +++ b/test/unit/test_migrate.py @@ -12,6 +12,7 @@ from log.log import AutosubmitCritical from test.unit.utils.common import create_database, init_expid +@pytest.mark.skip('This test requires a running SSH server, with password-less authentication') class TestMigrate: @pytest.fixture(scope='class') -- GitLab From 2427b7ee27b54fe416d48719a2220f1e262a7d60 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Fri, 19 Jul 2024 13:22:07 +0200 Subject: [PATCH 2/2] WIP: fixing tests with @dbeltran's suggestions --- autosubmit/job/job_packager.py | 2 +- test/unit/test_wrappers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/autosubmit/job/job_packager.py b/autosubmit/job/job_packager.py index 1ba86f2c3..c74f79de4 100644 --- a/autosubmit/job/job_packager.py +++ b/autosubmit/job/job_packager.py @@ -722,8 +722,8 @@ class JobPackagerVertical(object): :return: List of jobs that are wrapped together. :rtype: List() of Job Object """ - self.total_wallclock = "00:00" # reset total wallclock for package job.update_parameters(wrapper_info[-1],{}) # update_parameter has moved, so this is now needed. + self.total_wallclock = job.wallclock # reset total wallclock for package stack = [(job, 1)] while stack: job, level = stack.pop() diff --git a/test/unit/test_wrappers.py b/test/unit/test_wrappers.py index 2c605bef3..c47b884bd 100644 --- a/test/unit/test_wrappers.py +++ b/test/unit/test_wrappers.py @@ -566,7 +566,7 @@ class TestWrappers(TestCase): wrapper_limits["min_h"] = 2 wrapper_limits["max_by_section"] = max_wrapped_job_by_section returned_packages = self.job_packager._build_vertical_packages( - section_list, wrapper_limits) + section_list, wrapper_limits, self.wrapper_info) self.job_list._ordered_jobs_by_date_member["WRAPPERS"]["d1"] = dict() self.job_list._ordered_jobs_by_date_member["WRAPPERS"]["d1"]["m1"] = [d1_m1_1_s3] -- GitLab