diff --git a/.gitignore b/.gitignore index 40e7cb3ec7e789f1d55658d9559726592bd3eb72..3fa94f77fa61e46a88b7683abe7d1cbedef94519 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ build/ coverage/ /.coverage test/coverage.xml +test/htmlcov # Docker dockerfiles/id_rsa* diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f0f0a2895a8c4965919d7d6d8c4045ce8b0a805e..6998195a11f3db020e9ca8fb69c89e6ecdaa30da 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,14 +21,14 @@ prepare: - git submodule update --init --recursive - conda update -n base -c defaults conda - conda update conda - - conda install -n autosubmit3 coverage=6 + - conda install -n autosubmit3 pytest pytest-cov - conda env update -f environment.yml -n autosubmit3 python=3.7.3 test_python3: stage: test script: - conda activate autosubmit3 - - python3 -m 'nose' --exclude=regression --verbosity=3 test/unit --with-coverage --cover-package=autosubmit --cover-inclusive --cover-xml --cover-xml-file=test/coverage.xml + - pytest coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/' # These artifacts are saved with every build in GitLab and can be reviewed later. If # we have a folder with HTML files, as in this example, users can navigate with their diff --git a/easybuild/README.md b/easybuild/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c5c5eafae6fbd7c4ff9b13aba7eae69568ef0e74 --- /dev/null +++ b/easybuild/README.md @@ -0,0 +1,17 @@ +Files to build an EasyBuild installation for Autosubmit. + +Ref: https://earth.bsc.es/gitlab/es/autosubmit/-/issues/1257 + +Building locally, to test it first: + +```bash +$ docker login ghcr.io -u kinow +$ docker run --rm -ti \ + -u easybuild \ + -v $(pwd -P)/autosubmit-4-foss-2021b-Python-3.9.6.eb:/home/easybuild/autosubmit-4-foss-2021b-Python-3.9.6.eb \ + -w /home/easybuild ghcr.io/easybuilders/ubuntu-20.04:2024-01-17-7559408166.106 \ + /bin/bash +easybuild@e9c0fc4f3b5d:~$ pip install easybuild +easybuild@e9c0fc4f3b5d:~$ export PATH="$PATH:/home/easybuild/.local/bin:/usr/lmod/8.6.12/libexec/" +easybuild@e9c0fc4f3b5d:~$ EB_VERBOSE=1 eb autosubmit-*.eb --rebuild +``` diff --git a/easybuild/autosubmit-4-foss-2021b-Python-3.9.6.eb b/easybuild/autosubmit-4-foss-2021b-Python-3.9.6.eb new file mode 100644 index 0000000000000000000000000000000000000000..4834364d33e8e8fa14c41781c7030848932c4e57 --- /dev/null +++ b/easybuild/autosubmit-4-foss-2021b-Python-3.9.6.eb @@ -0,0 +1,55 @@ +easyblock = "PythonPackage" + +name = 'autosubmit' +version = 'v4.1.6' + +homepage = 'http://autosubmit.readthedocs.org' +description = """A versatile tool for managing ESM on HPC platforms""" + +toolchain = SYSTEM +#toolchain = {'name': 'foss', 'version': '2021b'} + +#sources = [SOURCELOWER_TAR_GZ] +#source_urls = ['https://pypi.python.org/packages/source/a/autosubmit'] + +sources = ['autosubmit-%(version)s.tar.gz'] + +python = 'Python' +pyver = '3.8.10' +pyshortver = '3.6' +versionsuffix = '-%s-%s' % (python, pyver) + +# Numpy 1.21 is the last release with 3.7+ support. +# Numpy 1.24 is the last release with 3.8+ support. +# Numpy 1.25 and greater requires 3.9+. +# https://endoflife.date/numpy#python-support +# Since our current setup.py declares 3.7 as minimum requirement, +# this function patches the setup.py file temporarily. +preinstallopts = """if grep -E "numpy<2" setup.py ; then \ + sed -i "s/numpy<2/numpy<1.22/g" setup.py ; \ +else \ + sed -i "s/install_requires\s*=\s*\[/install_requires=\['numpy<1.22',/g" setup.py ; \ +fi ;\ +if grep -E "matplotlib" setup.py ; then \ + +fi \ +&& +""" + +dependencies = [ +# (python, pyver), +# ('libpng','1.6.37'), +] + +#use_pip = True +# override extensions sanity check, default filter that imports a Python module doesn't work here +exts_filter = ('', "") + +sanity_check_paths = { + 'files': ['bin/autosubmit'], + 'dirs': [], +} + +#sanity_check_commands = [('autosubmit -v','')] + +moduleclass = 'tools' diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000000000000000000000000000000000000..298d65bd2835f3a8f31227edde8a362ebfc1cdea --- /dev/null +++ b/pytest.ini @@ -0,0 +1,9 @@ +[pytest] +addopts = + --verbose --ignore=regression --cov=autosubmit --cov-config=.coveragerc --cov-report=html:test/htmlcov --cov-report=xml:test/coverage.xml +testpaths = + test/unit +doctest_optionflags = + NORMALIZE_WHITESPACE + IGNORE_EXCEPTION_DETAIL + ELLIPSIS diff --git a/setup.py b/setup.py index d4a763a162dc03f748edc3f1e18b57fb18efe243..4a2779dda06936257f1d01d9f6bf52e546902e35 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,58 @@ here = path.abspath(path.dirname(__file__)) with open(path.join(here, 'VERSION')) as f: version = f.read().strip() +install_requires = [ + 'xlib==0.21', + 'setuptools<=68.2.2', + 'bscearth.utils<=0.5.2', + 'requests<=2.31.0', + 'networkx<=2.6.3', + 'portalocker<=2.7.0', + 'mock<=5.1.0', + 'paramiko<=3.4', + 'pyparsing==3.1.1', + 'matplotlib<=3.8.3', + 'argparse<=1.4.0', + 'packaging<=23.2', + 'ruamel.yaml.clib<=0.2.8', + 'typing_extensions<=4.9.0', + 'typing<=3.7.4.3', + 'psutil<=5.6.1', + 'networkx<=2.6.3', + 'py3dotplus==1.1.0', + 'matplotlib<=3.8.3', + 'numpy<2', + 'ruamel.yaml==0.17.21', + 'rocrate==0.*', + 'autosubmitconfigparser==1.0.62', + 'configparser', + 'pathlib', + 'setproctitle', + 'sqlalchemy[mypy]' +] + +pg_require = [ + 'psycopg2' +] + +tests_require = [ + 'mock==5.1.0', + 'nose==1.3.7', + 'pytest==8.2.*', + 'pytest-cov', + 'pytest-mock', + 'testcontainers' +] + +# You can add more groups, e.g. all_require = tests_require + graph_require, etc... +all_require = tests_require + pg_require + +extras_require = { + 'postgres': pg_require, + 'tests': tests_require, + 'all': all_require +} + setup( name='autosubmit', license='GNU GPL v3', @@ -39,79 +91,11 @@ setup( url='http://www.bsc.es/projects/earthscience/autosubmit/', download_url='https://earth.bsc.es/wiki/doku.php?id=tools:autosubmit', keywords=['climate', 'weather', 'workflow', 'HPC'], - install_requires=[ - 'xlib==0.21', - 'setuptools<=68.2.2', - 'bscearth.utils<=0.5.2', - 'requests<=2.31.0', - 'networkx<=2.6.3', - 'portalocker<=2.7.0', - 'mock<=5.1.0', - 'paramiko<=3.4', - 'pyparsing==3.1.1', - 'matplotlib<=3.8.3', - 'argparse<=1.4.0', - 'packaging<=23.2', - 'ruamel.yaml.clib<=0.2.8', - 'typing_extensions<=4.9.0', - 'typing<=3.7.4.3', - 'psutil<=5.6.1', - 'networkx<=2.6.3', - 'py3dotplus==1.1.0', - 'matplotlib<=3.8.3', - 'numpy<2', - 'ruamel.yaml==0.17.21', - 'rocrate==0.*', - 'autosubmitconfigparser==1.0.62', - 'configparser', - 'pathlib', - 'setproctitle' - - ], - extras_require={ - ':python_version <= "3.7"': - [ - 'PyNaCl==1.5.0', - 'pythondialog==3.5.3', - 'xlib==0.21', - 'setuptools==68.2.2', - 'cryptography==41.0.5', - 'bscearth.utils==0.5.2', - 'requests==2.31.0', - 'networkx==2.6.3', - 'portalocker==2.7.0', - 'mock==5.1.0', - 'paramiko==3.3.1', - 'matplotlib==3.5.3', - 'python_dateutil==2.8.2', - 'argparse==1.4.0', - 'configobj==5.0.8', - 'packaging==23.2', - 'bcrypt==4.0.1', - 'charset_normalizer==3.3.1', - 'kiwisolver==1.4.5', - 'fonttools==4.43.1', - 'cycler==0.12.1', - 'typing_extensions==4.8.0', - 'psutil==5.6.1', - 'Pygments==2.3.1', - 'coverage==5.0', - 'nose==1.3.7', - 'six==1.12.0', - 'Cython==0.29.6', - 'cffi==1.12.2', - 'py==1.8.0', - 'atomicwrites==1.3.0', - 'attrs==19.1.0', - 'more_itertools==6.0.0', - 'urllib3==1.24.1', - 'idna==2.8', - 'Pillow==6.2.1', - 'numpy==1.17.4', - ], - }, + install_requires=install_requires, + extras_require=extras_require, classifiers=[ "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "License :: OSI Approved :: GNU General Public License (GPL)", "Operating System :: POSIX :: Linux", @@ -131,4 +115,3 @@ setup( }, scripts=['bin/autosubmit'] ) -