From b265f4119bfe5e3abe14fefc677b3a20c16ddcb1 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Thu, 16 May 2024 15:21:56 +0200 Subject: [PATCH 1/4] Add dependency groups (optionals) to our setup.py --- setup.py | 170 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 98 insertions(+), 72 deletions(-) diff --git a/setup.py b/setup.py index d4a763a16..422edfa36 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,101 @@ here = path.abspath(path.dirname(__file__)) with open(path.join(here, 'VERSION')) as f: version = f.read().strip() +install_requires = [ + 'xlib==0.21', + 'setuptools<=68.2.2', + 'bscearth.utils<=0.5.2', + 'requests<=2.31.0', + 'networkx<=2.6.3', + 'portalocker<=2.7.0', + 'mock<=5.1.0', + 'paramiko<=3.4', + 'pyparsing==3.1.1', + 'matplotlib<=3.8.3', + 'argparse<=1.4.0', + 'packaging<=23.2', + 'ruamel.yaml.clib<=0.2.8', + 'typing_extensions<=4.9.0', + 'typing<=3.7.4.3', + 'psutil<=5.6.1', + 'networkx<=2.6.3', + 'py3dotplus==1.1.0', + 'matplotlib<=3.8.3', + 'numpy<2', + 'ruamel.yaml==0.17.21', + 'rocrate==0.*', + 'autosubmitconfigparser==1.0.62', + 'configparser', + 'pathlib', + 'setproctitle' +] + +python_version_less_37_require = [ + 'PyNaCl==1.5.0', + 'pythondialog==3.5.3', + 'xlib==0.21', + 'setuptools==68.2.2', + 'cryptography==41.0.5', + 'bscearth.utils==0.5.2', + 'requests==2.31.0', + 'networkx==2.6.3', + 'portalocker==2.7.0', + 'mock==5.1.0', + 'paramiko==3.3.1', + 'matplotlib==3.5.3', + 'python_dateutil==2.8.2', + 'argparse==1.4.0', + 'configobj==5.0.8', + 'packaging==23.2', + 'bcrypt==4.0.1', + 'charset_normalizer==3.3.1', + 'kiwisolver==1.4.5', + 'fonttools==4.43.1', + 'cycler==0.12.1', + 'typing_extensions==4.8.0', + 'psutil==5.6.1', + 'Pygments==2.3.1', + 'coverage==5.0', + 'nose==1.3.7', + 'six==1.12.0', + 'Cython==0.29.6', + 'cffi==1.12.2', + 'py==1.8.0', + 'atomicwrites==1.3.0', + 'attrs==19.1.0', + 'more_itertools==6.0.0', + 'urllib3==1.24.1', + 'idna==2.8', + 'Pillow==6.2.1', + 'numpy==1.17.4', + 'typing_extensions<=4.9.0', + 'typing<=3.7.4.3', + 'sqlalchemy[mypy]', +] + +pg_require = [ + 'psycopg2' +] + +tests_require = [ + 'mock==5.1.0', + 'nose==1.3.7', + 'pytest==8.2.*', + 'pytest-cov', + 'pytest-mock', + 'testcontainers' +] + +# You can add more groups, e.g. all_require = tests_require + graph_require, etc... +all_require = tests_require + pg_require + +extras_require = { + 'postgres': pg_require, + 'tests': tests_require, + ':python_version <= "3.7"': python_version_less_37_require, + 'all': all_require +} + setup( name='autosubmit', license='GNU GPL v3', @@ -39,79 +134,11 @@ setup( url='http://www.bsc.es/projects/earthscience/autosubmit/', download_url='https://earth.bsc.es/wiki/doku.php?id=tools:autosubmit', keywords=['climate', 'weather', 'workflow', 'HPC'], - install_requires=[ - 'xlib==0.21', - 'setuptools<=68.2.2', - 'bscearth.utils<=0.5.2', - 'requests<=2.31.0', - 'networkx<=2.6.3', - 'portalocker<=2.7.0', - 'mock<=5.1.0', - 'paramiko<=3.4', - 'pyparsing==3.1.1', - 'matplotlib<=3.8.3', - 'argparse<=1.4.0', - 'packaging<=23.2', - 'ruamel.yaml.clib<=0.2.8', - 'typing_extensions<=4.9.0', - 'typing<=3.7.4.3', - 'psutil<=5.6.1', - 'networkx<=2.6.3', - 'py3dotplus==1.1.0', - 'matplotlib<=3.8.3', - 'numpy<2', - 'ruamel.yaml==0.17.21', - 'rocrate==0.*', - 'autosubmitconfigparser==1.0.62', - 'configparser', - 'pathlib', - 'setproctitle' - - ], - extras_require={ - ':python_version <= "3.7"': - [ - 'PyNaCl==1.5.0', - 'pythondialog==3.5.3', - 'xlib==0.21', - 'setuptools==68.2.2', - 'cryptography==41.0.5', - 'bscearth.utils==0.5.2', - 'requests==2.31.0', - 'networkx==2.6.3', - 'portalocker==2.7.0', - 'mock==5.1.0', - 'paramiko==3.3.1', - 'matplotlib==3.5.3', - 'python_dateutil==2.8.2', - 'argparse==1.4.0', - 'configobj==5.0.8', - 'packaging==23.2', - 'bcrypt==4.0.1', - 'charset_normalizer==3.3.1', - 'kiwisolver==1.4.5', - 'fonttools==4.43.1', - 'cycler==0.12.1', - 'typing_extensions==4.8.0', - 'psutil==5.6.1', - 'Pygments==2.3.1', - 'coverage==5.0', - 'nose==1.3.7', - 'six==1.12.0', - 'Cython==0.29.6', - 'cffi==1.12.2', - 'py==1.8.0', - 'atomicwrites==1.3.0', - 'attrs==19.1.0', - 'more_itertools==6.0.0', - 'urllib3==1.24.1', - 'idna==2.8', - 'Pillow==6.2.1', - 'numpy==1.17.4', - ], - }, + install_requires=install_requires, + extras_require=extras_require, classifiers=[ "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "License :: OSI Approved :: GNU General Public License (GPL)", "Operating System :: POSIX :: Linux", @@ -131,4 +158,3 @@ setup( }, scripts=['bin/autosubmit'] ) - -- GitLab From bb285521554a31e6b2aaa3902c1e8a3ebf4cd618 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Thu, 16 May 2024 15:57:30 +0200 Subject: [PATCH 2/4] Use pytest for running the tests --- .gitignore | 1 + .gitlab-ci.yml | 4 ++-- pytest.ini | 9 +++++++++ setup.py | 3 ++- 4 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 pytest.ini diff --git a/.gitignore b/.gitignore index 40e7cb3ec..3fa94f77f 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ build/ coverage/ /.coverage test/coverage.xml +test/htmlcov # Docker dockerfiles/id_rsa* diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f0f0a2895..6998195a1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,14 +21,14 @@ prepare: - git submodule update --init --recursive - conda update -n base -c defaults conda - conda update conda - - conda install -n autosubmit3 coverage=6 + - conda install -n autosubmit3 pytest pytest-cov - conda env update -f environment.yml -n autosubmit3 python=3.7.3 test_python3: stage: test script: - conda activate autosubmit3 - - python3 -m 'nose' --exclude=regression --verbosity=3 test/unit --with-coverage --cover-package=autosubmit --cover-inclusive --cover-xml --cover-xml-file=test/coverage.xml + - pytest coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/' # These artifacts are saved with every build in GitLab and can be reviewed later. If # we have a folder with HTML files, as in this example, users can navigate with their diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..298d65bd2 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,9 @@ +[pytest] +addopts = + --verbose --ignore=regression --cov=autosubmit --cov-config=.coveragerc --cov-report=html:test/htmlcov --cov-report=xml:test/coverage.xml +testpaths = + test/unit +doctest_optionflags = + NORMALIZE_WHITESPACE + IGNORE_EXCEPTION_DETAIL + ELLIPSIS diff --git a/setup.py b/setup.py index 422edfa36..83bbc1ee5 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,8 @@ install_requires = [ 'autosubmitconfigparser==1.0.62', 'configparser', 'pathlib', - 'setproctitle' + 'setproctitle', + 'sqlalchemy[mypy]' ] python_version_less_37_require = [ -- GitLab From 95c7e54c09edede424de38c5cec4f67990e354cd Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Thu, 6 Jun 2024 13:42:41 +0200 Subject: [PATCH 3/4] Remove py<3.8 (pointed by @dbeltran that it is not working and can be removed) --- setup.py | 44 -------------------------------------------- 1 file changed, 44 deletions(-) diff --git a/setup.py b/setup.py index 83bbc1ee5..4a2779dda 100644 --- a/setup.py +++ b/setup.py @@ -57,49 +57,6 @@ install_requires = [ 'sqlalchemy[mypy]' ] -python_version_less_37_require = [ - 'PyNaCl==1.5.0', - 'pythondialog==3.5.3', - 'xlib==0.21', - 'setuptools==68.2.2', - 'cryptography==41.0.5', - 'bscearth.utils==0.5.2', - 'requests==2.31.0', - 'networkx==2.6.3', - 'portalocker==2.7.0', - 'mock==5.1.0', - 'paramiko==3.3.1', - 'matplotlib==3.5.3', - 'python_dateutil==2.8.2', - 'argparse==1.4.0', - 'configobj==5.0.8', - 'packaging==23.2', - 'bcrypt==4.0.1', - 'charset_normalizer==3.3.1', - 'kiwisolver==1.4.5', - 'fonttools==4.43.1', - 'cycler==0.12.1', - 'typing_extensions==4.8.0', - 'psutil==5.6.1', - 'Pygments==2.3.1', - 'coverage==5.0', - 'nose==1.3.7', - 'six==1.12.0', - 'Cython==0.29.6', - 'cffi==1.12.2', - 'py==1.8.0', - 'atomicwrites==1.3.0', - 'attrs==19.1.0', - 'more_itertools==6.0.0', - 'urllib3==1.24.1', - 'idna==2.8', - 'Pillow==6.2.1', - 'numpy==1.17.4', - 'typing_extensions<=4.9.0', - 'typing<=3.7.4.3', - 'sqlalchemy[mypy]', -] - pg_require = [ 'psycopg2' ] @@ -119,7 +76,6 @@ all_require = tests_require + pg_require extras_require = { 'postgres': pg_require, 'tests': tests_require, - ':python_version <= "3.7"': python_version_less_37_require, 'all': all_require } -- GitLab From 5b21493e4aede5ded0fa7bfec730e0dba3d66082 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Thu, 6 Jun 2024 15:25:08 +0200 Subject: [PATCH 4/4] WIP easybuild tests --- easybuild/README.md | 17 ++++++ .../autosubmit-4-foss-2021b-Python-3.9.6.eb | 55 +++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 easybuild/README.md create mode 100644 easybuild/autosubmit-4-foss-2021b-Python-3.9.6.eb diff --git a/easybuild/README.md b/easybuild/README.md new file mode 100644 index 000000000..c5c5eafae --- /dev/null +++ b/easybuild/README.md @@ -0,0 +1,17 @@ +Files to build an EasyBuild installation for Autosubmit. + +Ref: https://earth.bsc.es/gitlab/es/autosubmit/-/issues/1257 + +Building locally, to test it first: + +```bash +$ docker login ghcr.io -u kinow +$ docker run --rm -ti \ + -u easybuild \ + -v $(pwd -P)/autosubmit-4-foss-2021b-Python-3.9.6.eb:/home/easybuild/autosubmit-4-foss-2021b-Python-3.9.6.eb \ + -w /home/easybuild ghcr.io/easybuilders/ubuntu-20.04:2024-01-17-7559408166.106 \ + /bin/bash +easybuild@e9c0fc4f3b5d:~$ pip install easybuild +easybuild@e9c0fc4f3b5d:~$ export PATH="$PATH:/home/easybuild/.local/bin:/usr/lmod/8.6.12/libexec/" +easybuild@e9c0fc4f3b5d:~$ EB_VERBOSE=1 eb autosubmit-*.eb --rebuild +``` diff --git a/easybuild/autosubmit-4-foss-2021b-Python-3.9.6.eb b/easybuild/autosubmit-4-foss-2021b-Python-3.9.6.eb new file mode 100644 index 000000000..4834364d3 --- /dev/null +++ b/easybuild/autosubmit-4-foss-2021b-Python-3.9.6.eb @@ -0,0 +1,55 @@ +easyblock = "PythonPackage" + +name = 'autosubmit' +version = 'v4.1.6' + +homepage = 'http://autosubmit.readthedocs.org' +description = """A versatile tool for managing ESM on HPC platforms""" + +toolchain = SYSTEM +#toolchain = {'name': 'foss', 'version': '2021b'} + +#sources = [SOURCELOWER_TAR_GZ] +#source_urls = ['https://pypi.python.org/packages/source/a/autosubmit'] + +sources = ['autosubmit-%(version)s.tar.gz'] + +python = 'Python' +pyver = '3.8.10' +pyshortver = '3.6' +versionsuffix = '-%s-%s' % (python, pyver) + +# Numpy 1.21 is the last release with 3.7+ support. +# Numpy 1.24 is the last release with 3.8+ support. +# Numpy 1.25 and greater requires 3.9+. +# https://endoflife.date/numpy#python-support +# Since our current setup.py declares 3.7 as minimum requirement, +# this function patches the setup.py file temporarily. +preinstallopts = """if grep -E "numpy<2" setup.py ; then \ + sed -i "s/numpy<2/numpy<1.22/g" setup.py ; \ +else \ + sed -i "s/install_requires\s*=\s*\[/install_requires=\['numpy<1.22',/g" setup.py ; \ +fi ;\ +if grep -E "matplotlib" setup.py ; then \ + +fi \ +&& +""" + +dependencies = [ +# (python, pyver), +# ('libpng','1.6.37'), +] + +#use_pip = True +# override extensions sanity check, default filter that imports a Python module doesn't work here +exts_filter = ('', "") + +sanity_check_paths = { + 'files': ['bin/autosubmit'], + 'dirs': [], +} + +#sanity_check_commands = [('autosubmit -v','')] + +moduleclass = 'tools' -- GitLab