diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 5768d7c6..d5cf40f6 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -14,6 +14,6 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Codespell uses: codespell-project/actions-codespell@v1 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 676ddaf0..c5b53711 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -16,9 +16,9 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.11" - name: Install system dependencies diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e8d2c199..bd457e1a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -16,9 +16,9 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.11" - name: Install Black diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5ab438dd..8b726821 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: include: - - python: "3.9" + - python: "3.10" install_datalad: datalad # install_condor: condor # TODO 20.04 may be needed for condor @@ -49,9 +49,9 @@ jobs: os: ubuntu-22.04 reproman_tests_deps: full-except-datalad steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - name: Set up system @@ -120,9 +120,7 @@ jobs: pip install -e .[tests] pip install -e .[devel-docs] pip install -e .[devel-utils] - - name: Verify setup.py build - run: python setup.py build - name: Run tests run: python -m pytest -s -v -rsfE --integration --cov=reproman --cov-report=xml reproman - name: Upload coverage to codecov - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v4 diff --git a/docs/source/conf.py b/docs/source/conf.py index 880d97c5..56972a86 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -58,6 +58,12 @@ # for the module reference autosummary_generate = True +# Man page RST files under source/generated/man/ are not committed to git +# (they are generated by `python setup.py build_manpage`), so references +# to man page labels (e.g. man_reproman-login) will be undefined during +# a normal docs build. Suppress those warnings. +suppress_warnings = ["ref.ref", "toc.not_readable"] + # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/pyproject.toml b/pyproject.toml index cfd7bcf7..d54c652e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,6 @@ [tool.black] line-length = 100 -target-version = ['py39'] \ No newline at end of file +target-version = ['py310'] + +[tool.pytest.ini_options] +testpaths = ["reproman"] \ No newline at end of file diff --git a/reproman/cmd.py b/reproman/cmd.py index 5ed58525..d822017b 100644 --- a/reproman/cmd.py +++ b/reproman/cmd.py @@ -10,7 +10,6 @@ """ - import subprocess import sys import logging diff --git a/reproman/cmdline/helpers.py b/reproman/cmdline/helpers.py index 62d77216..d288a3d7 100644 --- a/reproman/cmdline/helpers.py +++ b/reproman/cmdline/helpers.py @@ -171,8 +171,7 @@ def run_via_pbs(args, pbs): # TODO: we might need better way to join them, escaping spaces etc. There must be a stock helper # exe_args = ' '.join(map(repr, args[1:])) if len(args) > 1 else '' exe_args = " ".join(args[1:]) if len(args) > 1 else "" - f.write( - """\ + f.write("""\ Executable = %(exe)s Initialdir = %(pwd)s Output = %(logs)s @@ -181,9 +180,7 @@ def run_via_pbs(args, pbs): arguments = %(exe_args)s queue -""" - % locals() - ) +""" % locals()) f.close() Runner().run(["condor_submit", f.name]) lgr.info("Scheduled execution via %s. Logs will be stored under %s" % (pbs, logs)) diff --git a/reproman/cmdline/main.py b/reproman/cmdline/main.py index 0c1acca0..59763c6e 100644 --- a/reproman/cmdline/main.py +++ b/reproman/cmdline/main.py @@ -73,11 +73,9 @@ def setup_parser(formatter_class=argparse.RawDescriptionHelpFormatter, return_su # main parser parser = argparse.ArgumentParser( fromfile_prefix_chars="@", - description=dedent_docstring( - """\ + description=dedent_docstring("""\ ReproMan aims to ease construction and execution of computation environments - based on collected provenance data.""" - ), + based on collected provenance data."""), epilog='"Reproducibly Manage Your Environments"', formatter_class=formatter_class, add_help=False, @@ -217,12 +215,10 @@ def setup_parser(formatter_class=argparse.RawDescriptionHelpFormatter, return_su parser.description, "\n".join(cmd_summary), textwrap.fill( - dedent_docstring( - """\ + dedent_docstring("""\ Detailed usage information for individual commands is available via command-specific --help, i.e.: - reproman --help""" - ), + reproman --help"""), 75, initial_indent="", subsequent_indent="", diff --git a/reproman/distributions/conda.py b/reproman/distributions/conda.py index 3f04c63f..2a22861e 100644 --- a/reproman/distributions/conda.py +++ b/reproman/distributions/conda.py @@ -6,6 +6,7 @@ # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Orchestrator sub-class to provide management of the localhost environment.""" + import json import os from collections import defaultdict @@ -439,8 +440,8 @@ def identify_distributions(self, paths): continue # Retrieve the environment details env_export = self._get_conda_env_export(root_path, conda_path) - (conda_package_details, file_to_pkg) = self._get_conda_package_details(conda_path) - (conda_pip_package_details, file_to_pip_pkg) = self._get_conda_pip_package_details( + conda_package_details, file_to_pkg = self._get_conda_package_details(conda_path) + conda_pip_package_details, file_to_pip_pkg = self._get_conda_pip_package_details( env_export, conda_path ) # Join our conda and pip packages diff --git a/reproman/distributions/debian.py b/reproman/distributions/debian.py index 1f69de36..41ea4932 100644 --- a/reproman/distributions/debian.py +++ b/reproman/distributions/debian.py @@ -6,6 +6,7 @@ # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Support for Debian(-based) distribution(s).""" + import os import re @@ -526,7 +527,7 @@ def _get_pkgs_install_date(self, pkg_dicts): out = exc.stdout # One file not found, so continue # Parse the output and store by filename for outlines in out.splitlines(): - (fname, ftime) = outlines.split(": ") + fname, ftime = outlines.split(": ") results[fname] = str(pytz.utc.localize(datetime.utcfromtimestamp(float(ftime)))) # Now lookup the packages in the results diff --git a/reproman/distributions/piputils.py b/reproman/distributions/piputils.py index 246256a6..318cbeba 100644 --- a/reproman/distributions/piputils.py +++ b/reproman/distributions/piputils.py @@ -6,6 +6,7 @@ # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Utilities for working with pip.""" + import itertools import json import os diff --git a/reproman/distributions/redhat.py b/reproman/distributions/redhat.py index 864ddd54..53797ea8 100644 --- a/reproman/distributions/redhat.py +++ b/reproman/distributions/redhat.py @@ -6,6 +6,7 @@ # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Support for Redhat(-based) distribution(s).""" + import attr diff --git a/reproman/distributions/singularity.py b/reproman/distributions/singularity.py index 6f567a7c..79f567e9 100644 --- a/reproman/distributions/singularity.py +++ b/reproman/distributions/singularity.py @@ -14,7 +14,6 @@ import tempfile import uuid - lgr = logging.getLogger("reproman.distributions.singularity") from .base import Package diff --git a/reproman/distributions/tests/test_conda.py b/reproman/distributions/tests/test_conda.py index fc9a5981..99817705 100644 --- a/reproman/distributions/tests/test_conda.py +++ b/reproman/distributions/tests/test_conda.py @@ -259,7 +259,7 @@ def test_conda_init_install_and_detect(tmpdir): assert len(dists) == 1, "Exactly one Conda distribution expected." - (distributions, unknown_files) = dists[0] + distributions, unknown_files = dists[0] # RepromanProvenance.write(sys.stdout, distributions) diff --git a/reproman/distributions/tests/test_debian.py b/reproman/distributions/tests/test_debian.py index 4ba3d908..a9fc17ad 100644 --- a/reproman/distributions/tests/test_debian.py +++ b/reproman/distributions/tests/test_debian.py @@ -35,7 +35,7 @@ def test_dpkg_manager_identify_packages(): files = [COMMON_SYSTEM_PATH] tracer = DebTracer() - (packages, unknown_files) = tracer.identify_packages_from_files(files) + packages, unknown_files = tracer.identify_packages_from_files(files) # Make sure that our common path was identified assert not unknown_files, "%s should be identified" % COMMON_SYSTEM_PATH assert len(packages) == 1 @@ -148,7 +148,7 @@ def test_utf8_file(): ] manager = DebTracer() # Simple sanity check that the pipeline works with utf-8 - (packages, unknown_files) = manager.identify_packages_from_files(files) + packages, unknown_files = manager.identify_packages_from_files(files) packages = manager.get_details_for_packages(packages) # If the file exists, it should be in ca-certificates if os.path.isfile(files[0]): @@ -232,7 +232,7 @@ def setup_packages(): def test_package_satisfies(setup_packages): - (p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2) = setup_packages + p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2 = setup_packages assert p1.compare(p1, mode="satisfied_by") assert p1v10.compare(p1v10, mode="satisfied_by") assert not p1v10.compare(p1, mode="satisfied_by") @@ -251,7 +251,7 @@ def test_package_satisfies(setup_packages): @pytest.fixture def setup_distributions(setup_packages): - (p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2) = setup_packages + p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2 = setup_packages d1 = DebianDistribution(name="debian 1") d1.packages = [p1] d2 = DebianDistribution(name="debian 2") @@ -260,8 +260,8 @@ def setup_distributions(setup_packages): def test_distribution_satisfies_package(setup_distributions, setup_packages): - (d1, d2) = setup_distributions - (p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2) = setup_packages + d1, d2 = setup_distributions + p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2 = setup_packages assert p1.compare(d1, mode="satisfied_by") assert not p1v10.compare(d1, mode="satisfied_by") assert p1.compare(d2, mode="satisfied_by") @@ -270,13 +270,13 @@ def test_distribution_satisfies_package(setup_distributions, setup_packages): def test_distribution_statisfies(setup_distributions): - (d1, d2) = setup_distributions + d1, d2 = setup_distributions assert not d2.compare(d1, mode="satisfied_by") assert d1.compare(d2, mode="satisfied_by") def test_distribution_sub(setup_packages): - (p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2) = setup_packages + p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2 = setup_packages d1 = DebianDistribution(name="debian 1") d1.packages = [p1, p2] d2 = DebianDistribution(name="debian 2") @@ -288,7 +288,7 @@ def test_distribution_sub(setup_packages): def test_package_is_identical_to(setup_packages): - (p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2) = setup_packages + p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2 = setup_packages assert p1.compare(p1, mode="identical_to") assert p1v10.compare(p1v10, mode="identical_to") assert p1v11.compare(p1v11, mode="identical_to") diff --git a/reproman/distributions/tests/test_piputils.py b/reproman/distributions/tests/test_piputils.py index 709054da..37873bf6 100644 --- a/reproman/distributions/tests/test_piputils.py +++ b/reproman/distributions/tests/test_piputils.py @@ -75,13 +75,10 @@ def test_parse_pip_show(): """ # Check parsing of show output for a "standard" package. - out_files = ( - out_base - + """\ + out_files = out_base + """\ Files: pkg-0.3.0.dist-info/DESCRIPTION.rst pkg/__init__.py""" - ) info_files = piputils.parse_pip_show(out_files) fields = { @@ -103,12 +100,9 @@ def test_parse_pip_show(): # Check parsing of show output for an editable packages that lacks # files. - out_no_files = ( - out_base - + """\ + out_no_files = out_base + """\ Files: Cannot locate installed-files.txt""" - ) info_nofiles = piputils.parse_pip_show(out_no_files) assert set(info_nofiles.keys()) == fields assert info_nofiles["Files"] == [] diff --git a/reproman/distributions/tests/test_redhat.py b/reproman/distributions/tests/test_redhat.py index 7de35090..349ee9c8 100644 --- a/reproman/distributions/tests/test_redhat.py +++ b/reproman/distributions/tests/test_redhat.py @@ -36,8 +36,7 @@ def docker_container(): def centos_spec(): tmp = tempfile.NamedTemporaryFile(delete=False) with open(tmp.name, "w") as f: - f.write( - """# ReproMan Environment Configuration File + f.write("""# ReproMan Environment Configuration File # This file was created by ReproMan 0.0.5 on 2018-05-23 22:03:22.820177 version: 0.0.1 distributions: @@ -95,8 +94,7 @@ def centos_spec(): files: - /usr/sbin/fido version: CentOS Linux release 7.4.1708 (Core) -""" - ) +""") return tmp.name @@ -115,7 +113,7 @@ def setup_packages(): @pytest.fixture def setup_distributions(setup_packages): - (p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2) = setup_packages + p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2 = setup_packages d1 = RedhatDistribution(name="debian 1") d1.packages = [p1] d2 = RedhatDistribution(name="debian 2") @@ -124,7 +122,7 @@ def setup_distributions(setup_packages): def test_package_satisfies(setup_packages): - (p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2) = setup_packages + p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2 = setup_packages assert p1.compare(p1, mode="satisfied_by") assert p1v10.compare(p1v10, mode="satisfied_by") assert not p1v10.compare(p1, mode="satisfied_by") @@ -142,8 +140,8 @@ def test_package_satisfies(setup_packages): def test_distribution_satisfies_package(setup_distributions, setup_packages): - (d1, d2) = setup_distributions - (p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2) = setup_packages + d1, d2 = setup_distributions + p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2 = setup_packages assert p1.compare(d1, mode="satisfied_by") assert not p1v10.compare(d1, mode="satisfied_by") assert p1.compare(d2, mode="satisfied_by") @@ -152,13 +150,13 @@ def test_distribution_satisfies_package(setup_distributions, setup_packages): def test_distribution_statisfies(setup_distributions): - (d1, d2) = setup_distributions + d1, d2 = setup_distributions assert not d2.compare(d1, mode="satisfied_by") assert d1.compare(d2, mode="satisfied_by") def test_distribution_sub(setup_packages): - (p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2) = setup_packages + p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2 = setup_packages d1 = RedhatDistribution(name="debian 1") d1.packages = [p1, p2] d2 = RedhatDistribution(name="debian 2") @@ -223,7 +221,7 @@ def test_distribution(docker_container, centos_spec): def test_package_is_identical_to(setup_packages): - (p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2) = setup_packages + p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2 = setup_packages assert p1.compare(p1, mode="identical_to") assert p1v10.compare(p1v10, mode="identical_to") assert p1v11.compare(p1v11, mode="identical_to") diff --git a/reproman/distributions/tests/test_vcs.py b/reproman/distributions/tests/test_vcs.py index 04a1b764..4dbfecf8 100644 --- a/reproman/distributions/tests/test_vcs.py +++ b/reproman/distributions/tests/test_vcs.py @@ -463,7 +463,7 @@ def test_git_install_add_remotes(traced_repo_copy, tmpdir): def test_svn(svn_repo): - (svn_repo_root, checked_out_dir) = svn_repo + svn_repo_root, checked_out_dir = svn_repo svn_file = os.path.join(checked_out_dir, "foo") uuid_file = os.path.join(svn_repo_root, "db", "uuid") uuid = open(uuid_file).readlines()[0].strip() @@ -483,7 +483,7 @@ def test_svn(svn_repo): def test_empty_svn(svn_repo_empty): - (svn_repo_root, checked_out_dir) = svn_repo_empty + svn_repo_root, checked_out_dir = svn_repo_empty tracer = VCSTracer() distributions = list(tracer.identify_distributions([checked_out_dir])) svn_repo = distributions[0][0].packages[0] diff --git a/reproman/distributions/vcs.py b/reproman/distributions/vcs.py index be52f9ba..5f4e15ff 100644 --- a/reproman/distributions/vcs.py +++ b/reproman/distributions/vcs.py @@ -34,7 +34,6 @@ from reproman.distributions.base import Distribution from reproman.distributions.base import TypedList - # # TODO: use metaclass I guess... ? # def get_vcs_distribution(RepoClass, name, Name): # """A helper to generate VCS distribution classes""" diff --git a/reproman/distributions/venv.py b/reproman/distributions/venv.py index 03dd2f0b..b9278790 100644 --- a/reproman/distributions/venv.py +++ b/reproman/distributions/venv.py @@ -6,6 +6,7 @@ # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Support for Python's virtualenv.""" + from collections import defaultdict import logging import os diff --git a/reproman/dochelpers.py b/reproman/dochelpers.py index 73f9b0e2..07c44ac0 100644 --- a/reproman/dochelpers.py +++ b/reproman/dochelpers.py @@ -18,7 +18,6 @@ import sys import traceback - lgr = logging.getLogger("reproman.docutils") __add_init2doc = False diff --git a/reproman/formats/reproman.py b/reproman/formats/reproman.py index 153a81c6..07162381 100644 --- a/reproman/formats/reproman.py +++ b/reproman/formats/reproman.py @@ -9,6 +9,7 @@ Plugin support for provenance YAML files following ReproMan spec. """ + from __future__ import absolute_import import collections diff --git a/reproman/formats/trig.py b/reproman/formats/trig.py index 2243a001..292f208c 100644 --- a/reproman/formats/trig.py +++ b/reproman/formats/trig.py @@ -45,14 +45,12 @@ def _load(cls, source): def get_packages(self): - results = self._src.query( - """SELECT DISTINCT ?command ?version + results = self._src.query("""SELECT DISTINCT ?command ?version WHERE { ?x nipype:command ?full_command . bind( strbefore( $full_command, " " ) as ?command ) . ?x nipype:version ?version . - }""" - ) + }""") return results diff --git a/reproman/interface/backend_parameters.py b/reproman/interface/backend_parameters.py index b5e99994..59c23c14 100644 --- a/reproman/interface/backend_parameters.py +++ b/reproman/interface/backend_parameters.py @@ -22,7 +22,6 @@ from reproman.support.exceptions import ResourceError from reproman.support.param import Parameter - lgr = getLogger("reproman.api.backend_parameters") diff --git a/reproman/interface/common_opts.py b/reproman/interface/common_opts.py index 23835551..ec61c9e3 100644 --- a/reproman/interface/common_opts.py +++ b/reproman/interface/common_opts.py @@ -13,7 +13,6 @@ from reproman.support.constraints import EnsureChoice from reproman.support.constraints import EnsureInt, EnsureNone, EnsureStr - trace_opt = Parameter( args=("--trace",), action="store_true", doc="""if set, trace execution within the environment""" ) diff --git a/reproman/interface/retrace.py b/reproman/interface/retrace.py index 460afafa..9d377ae5 100644 --- a/reproman/interface/retrace.py +++ b/reproman/interface/retrace.py @@ -123,7 +123,7 @@ def __call__(path=None, spec=None, output_file=None, resref=None, resref_type="a # Generalize # TODO: RF so that only the above portion is reprozip specific. # If we are to reuse their layout largely -- the rest should stay as is - (distributions, files) = identify_distributions(paths, session=session) + distributions, files = identify_distributions(paths, session=session) from reproman.distributions.base import EnvironmentSpec spec = EnvironmentSpec( diff --git a/reproman/interface/run.py b/reproman/interface/run.py index 6a4fff5f..a970c2f5 100644 --- a/reproman/interface/run.py +++ b/reproman/interface/run.py @@ -290,8 +290,7 @@ class Run(Interface): doc="""YAML files that define job parameters. Multiple paths can be given. If a parameter is defined in multiple specs, the value from the last path that defines it is used[CMD: . Use --list to see - available parameters for the built-in templates CMD].""" - + _more_than_once_doc, + available parameters for the built-in templates CMD].""" + _more_than_once_doc, ), job_parameters=Parameter( metavar="PARAM", @@ -305,8 +304,7 @@ class Run(Interface): The values are available as fields in the templates used to generate both the run script and submission script[CMD: . Use --list to see available parameters for the built-in templates - CMD].""" - + _more_than_once_doc, + CMD].""" + _more_than_once_doc, ), inputs=Parameter( args=("-i", "--input"), @@ -316,8 +314,7 @@ class Run(Interface): doc="""An input path to the command. How input paths are used depends on the orchestrator, but, at the very least, the orchestrator should try to make these paths available on the - resource.""" - + _more_than_once_doc, + resource.""" + _more_than_once_doc, ), outputs=Parameter( args=("-o", "--output"), @@ -325,8 +322,7 @@ class Run(Interface): metavar="PATH", action="append", doc="""An output path to the command. How output paths are handled - depends on the orchestrator.""" - + _more_than_once_doc, + depends on the orchestrator.""" + _more_than_once_doc, ), follow=Parameter( args=("--follow",), diff --git a/reproman/interface/test.py b/reproman/interface/test.py index e8101c8a..afa6193b 100644 --- a/reproman/interface/test.py +++ b/reproman/interface/test.py @@ -7,7 +7,6 @@ # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Run internal ReproMan (unit)tests to verify correct operation on the system""" - __docformat__ = "restructuredtext" diff --git a/reproman/interface/tests/test_execute.py b/reproman/interface/tests/test_execute.py index f130f991..a72eb8a7 100644 --- a/reproman/interface/tests/test_execute.py +++ b/reproman/interface/tests/test_execute.py @@ -33,7 +33,6 @@ from ...cmd import Runner from ...utils import swallow_logs - docker_container = get_docker_fixture( TEST_SSH_DOCKER_DIGEST, name="testing-container", scope="module", seccomp_unconfined=True ) diff --git a/reproman/interface/tests/test_ls.py b/reproman/interface/tests/test_ls.py index 305e15ea..66c08112 100644 --- a/reproman/interface/tests/test_ls.py +++ b/reproman/interface/tests/test_ls.py @@ -81,8 +81,11 @@ def test_ls_interface(ls_fn): # Test --refresh output results = ls_fn(refresh=True) assert "NOT FOUND" in results["326b0fdfbf838"] - assert "CONNECTION ERROR" in results["i-22221ddf096c22bb0"] - assert "CONNECTION ERROR" in results["i-3333f40de2b9b8967"] + # EC2 resources with fake credentials may report either "CONNECTION ERROR" + # (connect() raises) or "NOT FOUND" (connect() succeeds but sets id=None), + # depending on boto3/botocore version behavior. + for ec2_id in ("i-22221ddf096c22bb0", "i-3333f40de2b9b8967"): + assert "CONNECTION ERROR" in results[ec2_id] or "NOT FOUND" in results[ec2_id] def test_ls_interface_limited(ls_fn): diff --git a/reproman/resource/singularity.py b/reproman/resource/singularity.py index f159a4d4..a49a1f6e 100644 --- a/reproman/resource/singularity.py +++ b/reproman/resource/singularity.py @@ -212,7 +212,7 @@ def get(self, src_path, dest_path=None, uid=-1, gid=-1): def listdir(self, path): cmd = ["singularity", "exec", "instance://{}".format(self.name), "ls", "-1", path] - (stdout, stderr) = self._runner.run(cmd) + stdout, stderr = self._runner.run(cmd) return [f for f in stdout.split("\n") if f not in ("", ".", "..")] diff --git a/reproman/resource/tests/test_docker_container.py b/reproman/resource/tests/test_docker_container.py index 7033cd98..11524aab 100644 --- a/reproman/resource/tests/test_docker_container.py +++ b/reproman/resource/tests/test_docker_container.py @@ -20,7 +20,6 @@ from pytest import raises - setup_ubuntu = get_docker_fixture( TEST_SSH_DOCKER_DIGEST, scope="module", name="reproman-test-ssh-container" ) diff --git a/reproman/resource/tests/test_session.py b/reproman/resource/tests/test_session.py index 5a883faa..b0d98f4e 100644 --- a/reproman/resource/tests/test_session.py +++ b/reproman/resource/tests/test_session.py @@ -24,7 +24,6 @@ from ...tests.fixtures import get_singularity_fixture from ...consts import TEST_SSH_DOCKER_DIGEST - testing_container = get_docker_fixture( TEST_SSH_DOCKER_DIGEST, name="testing-container", diff --git a/reproman/resource/tests/test_shell.py b/reproman/resource/tests/test_shell.py index 5b0006bf..755697cf 100644 --- a/reproman/resource/tests/test_shell.py +++ b/reproman/resource/tests/test_shell.py @@ -53,8 +53,7 @@ def test_source_file(resource_test_dir): # Create a temporary test file temp_file = tempfile.NamedTemporaryFile(dir=resource_test_dir) with temp_file as f: - f.write( - """ + f.write(""" echo "Enabling special environment" echo "We could even spit out an stderr output">&2 export EXPORTED_VAR=" @@ -62,10 +61,7 @@ def test_source_file(resource_test_dir): " export PATH=/custom:$PATH NON_EXPORTED_VAR=2 # but may be those should be handled?? - """.encode( - "utf8" - ) - ) + """.encode("utf8")) f.flush() script = temp_file.name session = ShellSession() @@ -95,7 +91,7 @@ def test_isdir(): def test_exists(): session = ShellSession() - (_, name) = tempfile.mkstemp() + _, name = tempfile.mkstemp() assert session.exists(name) os.remove(name) assert not session.exists(name) @@ -105,17 +101,13 @@ def test_exists(): def test_source_file_param(resource_test_dir): temp_file = tempfile.NamedTemporaryFile(dir=resource_test_dir) with temp_file as f: - f.write( - """ + f.write(""" if ! [ "$1" = "test" ]; then exit 1 fi export EXPORTED_VAR=${1}1 NON_EXPORTED_VAR=2 # but may be those should be handled?? - """.encode( - "utf8" - ) - ) + """.encode("utf8")) f.flush() script = temp_file.name session = ShellSession() diff --git a/reproman/support/external_versions.py b/reproman/support/external_versions.py index a34dff42..11617ca4 100644 --- a/reproman/support/external_versions.py +++ b/reproman/support/external_versions.py @@ -6,10 +6,12 @@ # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Module to help maintain a registry of versions for external modules etc""" + import sys +from importlib.metadata import version as get_distribution_version, PackageNotFoundError from os import linesep -from distutils.version import LooseVersion +from looseversion import LooseVersion from reproman.dochelpers import exc_str from reproman.log import lgr @@ -118,10 +120,9 @@ class ExternalVersions(object): To avoid collision between names of python modules and command line tools, prepend names for command line tools with `cmd:`. - It maintains a dictionary of `distuil.version.LooseVersion`s to make - comparisons easy. Note that even if version string conform the StrictVersion - "standard", LooseVersion will be used. If version can't be deduced for the - external, `UnknownVersion()` is assigned. If external is not present (can't + It maintains a dictionary of `LooseVersion`s to make + comparisons easy. If version can't be deduced for the external, + `UnknownVersion()` is assigned. If external is not present (can't be imported, or custom check throws exception), None is returned without storing it, so later call will re-evaluate fully. """ @@ -157,13 +158,11 @@ def _deduce_version(klass, value): version = getattr(value, attr) break - # try pkg_resources + # try importlib.metadata if version is None and hasattr(value, "__name__"): try: - import pkg_resources - - version = pkg_resources.get_distribution(value.__name__).version - except Exception: + version = get_distribution_version(value.__name__) + except PackageNotFoundError: pass # assume that value is the version diff --git a/reproman/support/jobs/submitters.py b/reproman/support/jobs/submitters.py index dbb36269..931ec1ea 100644 --- a/reproman/support/jobs/submitters.py +++ b/reproman/support/jobs/submitters.py @@ -20,7 +20,6 @@ from reproman.cmd import CommandError from reproman.dochelpers import borrowdoc - lgr = logging.getLogger("reproman.support.jobs.submitters") diff --git a/reproman/support/jobs/tests/test_orchestrators.py b/reproman/support/jobs/tests/test_orchestrators.py index ccc924b5..9bf56209 100644 --- a/reproman/support/jobs/tests/test_orchestrators.py +++ b/reproman/support/jobs/tests/test_orchestrators.py @@ -29,7 +29,6 @@ from reproman.tests.skip import skipif from reproman.tests.utils import create_tree - try: lsf_config = os.environ["REPROMAN_LSF_TEST_CONFIG"].split(":") lsf_config = {"host": lsf_config[0], "user": lsf_config[1], "tmpdir_root": lsf_config[2]} diff --git a/reproman/support/protocol.py b/reproman/support/protocol.py index 1c1c429a..7212957f 100644 --- a/reproman/support/protocol.py +++ b/reproman/support/protocol.py @@ -12,7 +12,6 @@ import logging import time - lgr = logging.getLogger("reproman.protocol") diff --git a/reproman/support/tests/test_external_versions.py b/reproman/support/tests/test_external_versions.py index 4f90c798..5e41fc65 100644 --- a/reproman/support/tests/test_external_versions.py +++ b/reproman/support/tests/test_external_versions.py @@ -13,7 +13,8 @@ from ... import __version__ from ...dochelpers import exc_str from ...version import __version__ -from ..external_versions import ExternalVersions, LooseVersion +from looseversion import LooseVersion +from ..external_versions import ExternalVersions from ...tests.utils import assert_true, assert_false from ...tests.utils import assert_equal, assert_greater_equal, assert_greater from ..exceptions import CommandError diff --git a/reproman/tests/skip.py b/reproman/tests/skip.py index b50d6289..cae1b0e2 100644 --- a/reproman/tests/skip.py +++ b/reproman/tests/skip.py @@ -40,6 +40,7 @@ def windows(): `mark.skipif_NAME` and `skipif.NAME`. So, for the above example, there would now be `mark.skipif_windows` and `skipif.windows`. """ + import abc import os diff --git a/reproman/tests/test_constraints.py b/reproman/tests/test_constraints.py index 8501b92b..7bcd9b8a 100644 --- a/reproman/tests/test_constraints.py +++ b/reproman/tests/test_constraints.py @@ -7,7 +7,6 @@ ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Unit tests for basic constraints functionality.""" - import sys from ..support import constraints as ct diff --git a/reproman/tests/test_skip.py b/reproman/tests/test_skip.py index 9a8e3c52..de831375 100644 --- a/reproman/tests/test_skip.py +++ b/reproman/tests/test_skip.py @@ -14,7 +14,6 @@ from reproman.tests.skip import mark from reproman.tests.skip import skipif - with patch.dict("os.environ", {"REPROMAN_TESTS_NONETWORK": "1"}): @mark.skipif_no_network diff --git a/reproman/tests/test_utils.py b/reproman/tests/test_utils.py index 594a8bc6..d8c6e6c8 100644 --- a/reproman/tests/test_utils.py +++ b/reproman/tests/test_utils.py @@ -437,20 +437,24 @@ def test_assure_unicode(): eq_(assure_unicode("grandchild_äöü東"), "grandchild_äöü東") # now, non-utf8 # Decoding could be deduced with high confidence when the string is - # really encoded in that codepage - mom_koi8r = "мама".encode("koi8-r") - eq_(assure_unicode(mom_koi8r), "мама") - eq_(assure_unicode(mom_koi8r, confidence=0.9), "мама") + # really encoded in that codepage. + # Use a longer string so chardet can reliably detect KOI8-R -- + # chardet 6 rewrote single-byte detection and needs more bytes to + # distinguish KOI8-R from Thai (CP874) for very short inputs. + mom_koi8r = "мама мыла раму".encode("koi8-r") + eq_(assure_unicode(mom_koi8r), "мама мыла раму") + # chardet 7 reports ~0.22 confidence for KOI8 family variants + eq_(assure_unicode(mom_koi8r, confidence=0.2), "мама мыла раму") + # For iso-8859-1, use a short string: chardet 7 guesses cp1250 which + # decodes 0xe1 identically as á, so the result is correct despite the + # wrong encoding label. mom_iso8859 = "mamá".encode("iso-8859-1") eq_(assure_unicode(mom_iso8859), "mamá") - eq_(assure_unicode(mom_iso8859, confidence=0.5), "mamá") - # but when we mix, it does still guess something allowing to decode: + # chardet 7 reports very low confidence (~0.04) for short single-byte strings + eq_(assure_unicode(mom_iso8859, confidence=0.01), "mamá") + # when we mix encodings, chardet may guess wrong and fail to decode, + # which should raise ValueError mixedin = mom_koi8r + "東".encode("iso2022_jp") + "東".encode("utf-8") - ok_(isinstance(assure_unicode(mixedin), str)) - # but should fail if we request high confidence result: - - # FIXME: For some reason this doesn't raise a ValueError on our Travis - # py3.5/system_site_packages run. with assert_raises(ValueError): assure_unicode(mixedin, confidence=0.9) # For other, non string values, actually just returns original value diff --git a/reproman/tests/utils.py b/reproman/tests/utils.py index 8e48e31c..4a8d0e1e 100644 --- a/reproman/tests/utils.py +++ b/reproman/tests/utils.py @@ -548,14 +548,12 @@ def create_pymodule(directory): """ os.makedirs(directory) with open(os.path.join(directory, "setup.py"), "w") as ofh: - ofh.write( - """\ + ofh.write("""\ from setuptools import setup setup(name='nmtest', version='0.1.0', - py_modules=['nmtest'])""" - ) + py_modules=['nmtest'])""") with open(os.path.join(directory, "nmtest"), "w") as ofh: ofh.write("") diff --git a/reproman/utils.py b/reproman/utils.py index 12d41657..a2041dd9 100644 --- a/reproman/utils.py +++ b/reproman/utils.py @@ -495,6 +495,23 @@ def assure_bytes(s, encoding="utf-8"): return s.encode(encoding) +def _chardet_detect(s: bytes) -> dict: + """Wrapper around chardet.detect with encoding_era=ALL for chardet >= 6. + + chardet 6 defaults to encoding_era=MODERN_WEB which excludes legacy + encodings like KOI8-R. We need to consider ALL eras for backward + compatibility. + """ + from chardet import detect + from reproman.support.external_versions import external_versions + + if external_versions["chardet"] >= "6": + from chardet.enums import EncodingEra + + return detect(s, encoding_era=EncodingEra.ALL) + return detect(s) + + def assure_unicode(s, encoding=None, confidence=None): """Convert/decode to str if of 'bytes' @@ -519,9 +536,7 @@ def assure_unicode(s, encoding=None, confidence=None): lgr.debug("Failed to decode a string as utf-8: %s", exc_str(exc)) # And now we could try to guess - from chardet import detect - - enc = detect(s) + enc = _chardet_detect(s) denc = enc.get("encoding", None) if denc: denc_confidence = enc.get("confidence", 0) @@ -530,7 +545,10 @@ def assure_unicode(s, encoding=None, confidence=None): "Failed to auto-detect encoding with high enough " "confidence. Highest confidence was %s for %s" % (denc_confidence, denc) ) - return s.decode(denc) + try: + return s.decode(denc) + except (UnicodeDecodeError, LookupError) as exc: + raise ValueError("Failed to decode with guessed encoding %s: %s" % (denc, exc)) else: raise ValueError( "Could not decode value as utf-8, or to guess its encoding: %s" % repr(s) diff --git a/reproman/version.py b/reproman/version.py index f116d579..26a7bef2 100644 --- a/reproman/version.py +++ b/reproman/version.py @@ -34,8 +34,6 @@ line = git.stdout.readlines()[0] _ = git.stderr.readlines() # Just take describe and replace initial '-' with .dev to be more "pythonish" - # Encoding simply because distutils' LooseVersion compares only StringType - # and thus misses in __cmp__ necessary wrapping for unicode strings __full_version__ = line.strip().decode("ascii").replace("-", ".dev", 1).encode() # To follow PEP440 we can't have all the git fanciness __version__ = __full_version__.split("-")[0] diff --git a/setup.py b/setup.py index afea885e..9393ae72 100755 --- a/setup.py +++ b/setup.py @@ -13,7 +13,6 @@ from os.path import splitext from os.path import dirname -from setuptools import findall from setuptools import setup, find_packages # manpage build imports @@ -29,9 +28,12 @@ def findsome(subdir, extensions): Leading directory (reproman) gets stripped """ + import glob + + top = opj("reproman", subdir) return [ f.split(pathsep, 1)[1] - for f in findall(opj("reproman", subdir)) + for f in glob.glob(opj(top, "**"), recursive=True) if splitext(f)[-1].lstrip(".") in extensions ] @@ -55,6 +57,7 @@ def findsome(subdir, extensions): "fabric>=2.3.1", "humanize", "jinja2", + "looseversion", "packaging", "paramiko", "pyOpenSSL", @@ -160,7 +163,7 @@ def findsome(subdir, extensions): # python3 -m pip install --user --upgrade setuptools wheel twine long_description_content_type="text/markdown", packages=reproman_pkgs, - python_requires=">=3.9", + python_requires=">=3.10", install_requires=requires["core"], extras_require=requires, entry_points={ diff --git a/setup_support.py b/setup_support.py index 5da72975..e3849ebe 100644 --- a/setup_support.py +++ b/setup_support.py @@ -9,8 +9,8 @@ import os from os.path import dirname, join as opj -from distutils.core import Command -from distutils.errors import DistutilsOptionError +from setuptools import Command +from setuptools.errors import OptionError as DistutilsOptionError import datetime import formatters as fmt diff --git a/tox.ini b/tox.ini index 6f91d801..1909f769 100644 --- a/tox.ini +++ b/tox.ini @@ -1,27 +1,25 @@ [tox] -envlist = py39,py310,py311,py312 +envlist = py310,py311,py312,py313,py314 #,flake8 [testenv] -commands = python setup.py develop - # -s must be used since some places to not play nicely with swallowed by nose output - # and there were no easy way to monkey patch nose for that - py.test -s {posargs} - # interesting hints at http://blog.ionelmc.ro/2015/04/14/tox-tricks-and-patterns/ but yet to adopt - #{posargs:py.test -s} -deps = -r{toxinidir}/requirements.txt -# # tox 2. introduced isolation from invocation environment # HOME is used by annex standalone atm # https://git-annex.branchable.com/bugs/standalone_builds_shouldn__39__t_pollute___126____47__.ssh_with_helpers_merely_upon_annex_init/ # so let's pass it, though in the future we should isolate # it back to guarantee that the tests do not rely on anything in # current user HOME -passenv=HOME +passenv = HOME,REPROMAN_*,USER,DBUS_SESSION_BUS_ADDRESS +commands = + # -s must be used since some places to not play nicely with swallowed by nose output + # and there were no easy way to monkey patch nose for that + py.test -s {posargs} + # interesting hints at http://blog.ionelmc.ro/2015/04/14/tox-tricks-and-patterns/ but yet to adopt + #{posargs:py.test -s} +deps = -r{toxinidir}/requirements.txt [testenv:cover] -commands = python setup.py develop - coverage run py.test {posargs} +commands = coverage run py.test {posargs} [testenv:flake8] commands = flake8 {posargs}