From cf71eedd424b9cb7423fec6cc43ad13d3e7adf81 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 31 Jul 2026 17:33:19 +0200 Subject: [PATCH] feat: add docs bundle test runners --- bzl/bundle_rules.bzl | 94 +++++++++++++++++++ docs.bzl | 16 +++- docs/how-to/test_to_doc_links.rst | 24 +++++ docs/reference/bazel_macros.rst | 10 +- score_pytest/main.py | 4 + src/tests/docs_bzl/helpers.py | 4 +- .../docs_bzl/scenarios/nested_bundles/BUILD | 15 +++ .../nested_bundles/child/test_one.py | 16 ++++ .../nested_bundles/child/test_two.py | 16 ++++ src/tests/docs_bzl/test_nested_bundles.py | 4 + 10 files changed, 200 insertions(+), 3 deletions(-) create mode 100644 src/tests/docs_bzl/scenarios/nested_bundles/child/test_one.py create mode 100644 src/tests/docs_bzl/scenarios/nested_bundles/child/test_two.py diff --git a/bzl/bundle_rules.bzl b/bzl/bundle_rules.bzl index f4884675b..abdb43056 100644 --- a/bzl/bundle_rules.bzl +++ b/bzl/bundle_rules.bzl @@ -67,6 +67,13 @@ DocsBundleInfo = provider( }, ) +DocsBundleTestsInfo = provider( + doc = "Test targets declared by a documentation bundle.", + fields = { + "tests": "Ordered executable test targets declared by this bundle and its children.", + }, +) + def _parent_index_docname(mount_at): """Choose the page that links to a bundled subtree by default.""" parent = mount_at.rsplit("/", 1)[0] if "/" in mount_at else "" @@ -141,6 +148,16 @@ def _sourcelinks_visible_through(ctx, child): return sourcelinks return [link for link in sourcelinks if link.repository == child_repository] +def _deduplicate_tests(tests): + """Keep the first occurrence of each target while preserving declaration order.""" + deduplicated = [] + seen = {} + for test in tests: + if test.label not in seen: + seen[test.label] = True + deduplicated.append(test) + return deduplicated + def _parse_bundle_declaration(bundle): """Read one nested-bundle declaration and fill in optional values.""" if type(bundle) != "dict": @@ -258,6 +275,83 @@ def create_bundle(name, bundles, srcs = [], sourcelinks = [], strip_prefix = "", ) return ":" + name +def _test_runfiles_path(executable): + """Return the test-runner path to an executable in its runfiles tree.""" + short_path = executable.short_path + if short_path.startswith("../"): + return "$TEST_SRCDIR/" + short_path[3:] + return "$TEST_SRCDIR/$TEST_WORKSPACE/" + short_path + +def _docs_bundle_test_impl(ctx): + """Create the executable for a bundle's hermetic test runner.""" + tests = list(ctx.attr.tests) + for bundle_test in ctx.attr.bundle_tests: + tests.extend(bundle_test[DocsBundleTestsInfo].tests) + tests = _deduplicate_tests(tests) + runfiles = ctx.runfiles() + commands = [ + "#!/usr/bin/env bash", + "set -euo pipefail", + "output_dir=\"$TEST_UNDECLARED_OUTPUTS_DIR/docs_bundle_tests\"", + "mkdir -p \"$output_dir\"", + ] + + for index, test in enumerate(tests): + files_to_run = test[DefaultInfo].files_to_run + executable = files_to_run.executable + if executable == None: + fail("docs bundle test %s is not executable" % test.label) + if test[DefaultInfo].default_runfiles: + runfiles = runfiles.merge(test[DefaultInfo].default_runfiles) + if test[DefaultInfo].data_runfiles: + runfiles = runfiles.merge(test[DefaultInfo].data_runfiles) + commands.extend([ + "echo \"Running docs bundle test %s\"" % test.label, + "XML_OUTPUT_FILE=\"$output_dir/%d.xml\" \"%s\"" % ( + index, + _test_runfiles_path(executable), + ), + "if [[ ! -s \"$output_dir/%d.xml\" ]]; then" % index, + " echo \"docs bundle test %s did not create JUnit XML via XML_OUTPUT_FILE\" >&2" % test.label, + " exit 1", + "fi", + ]) + + executable = ctx.actions.declare_file(ctx.label.name + ".sh") + ctx.actions.write( + output = executable, + content = "\n".join(commands) + "\n", + is_executable = True, + ) + return [ + DefaultInfo( + executable = executable, + runfiles = runfiles, + ), + DocsBundleTestsInfo(tests = tests), + ] + +_docs_bundle_test = rule( + implementation = _docs_bundle_test_impl, + attrs = { + "tests": attr.label_list(), + "bundle_tests": attr.label_list(providers = [DocsBundleTestsInfo]), + }, + test = True, + doc = "Runs the test targets transitively declared by a documentation bundle.", +) + +def create_bundle_tests(name, tests, bundles, visibility = None): + """Create a testonly runner for tests declared by a documentation bundle.""" + parsed_bundles = [_parse_bundle_declaration(declaration) for declaration in bundles] + _docs_bundle_test( + name = name, + tests = tests, + bundle_tests = [str(bundle.bundle) + "_tests" for bundle in parsed_bundles], + visibility = visibility, + ) + return ":" + name + def _external_docs_runfiles_impl(ctx): """Expose external documentation sources needed under ``bazel run``.""" return [DefaultInfo(files = ctx.attr.bundle[DocsBundleInfo].external_runfiles)] diff --git a/docs.bzl b/docs.bzl index 3b92eed59..08c74f878 100644 --- a/docs.bzl +++ b/docs.bzl @@ -52,6 +52,7 @@ load( load( "@score_docs_as_code//:bzl/bundle_rules.bzl", "create_bundle", + "create_bundle_tests", "merge_bundle_sourcelinks", "external_docs_runfiles", ) @@ -60,7 +61,7 @@ load( "create_mounts_manifest", ) -def docs_bundle(name, source_dir = None, entry_doc = "index", bundles = [], scan_code = [], visibility = None, **kwargs): +def docs_bundle(name, source_dir = None, entry_doc = "index", bundles = [], scan_code = [], tests = [], visibility = None, **kwargs): """A docs bundle, optionally composed of others. Args: @@ -78,6 +79,9 @@ def docs_bundle(name, source_dir = None, entry_doc = "index", bundles = [], scan }. scan_code: Source-code targets to scan for source-code links owned by this bundle. + tests: Executable Bazel test targets to run through the generated + `_tests` testonly target. Each target must write JUnit XML + to the `XML_OUTPUT_FILE` environment variable. visibility: Target visibility. **kwargs: Additional attributes forwarded to the underlying rule. """ @@ -105,6 +109,12 @@ def docs_bundle(name, source_dir = None, entry_doc = "index", bundles = [], scan visibility = visibility, **kwargs ) + create_bundle_tests( + name = name + "_tests", + tests = tests, + bundles = bundles, + visibility = visibility, + ) def _missing_requirements(deps): """Add Python hub dependencies if they are missing.""" @@ -142,6 +152,7 @@ def docs( data = [], deps = [], scan_code = [], + tests = [], test_sources = [], known_good = None, metamodel = None, @@ -156,6 +167,8 @@ def docs( data: Additional data files to include in the documentation build. deps: Additional dependencies for the documentation build. scan_code: List of code targets to scan for source code links. + tests: Test targets that are executed through the generated + `docs_bundle_tests` target. test_sources: Optional list of repo-relative directory paths which will be used to filter testcases for documentation generation. When empty (default), all testcases found in `bazel-testlogs` will be used. known_good: Optional label to a "known good" JSON file for source links. @@ -218,6 +231,7 @@ def docs( entry_doc = "index", bundles = bundles, scan_code = scan_code, + tests = tests, visibility = ["//visibility:public"], ) merge_bundle_sourcelinks( diff --git a/docs/how-to/test_to_doc_links.rst b/docs/how-to/test_to_doc_links.rst index 81ec104af..05ada73e5 100644 --- a/docs/how-to/test_to_doc_links.rst +++ b/docs/how-to/test_to_doc_links.rst @@ -114,6 +114,30 @@ See :need:`gd_temp__verification_specification` for code templates. Running Tests and Building Docs ------------------------------- +Bundle-owned verification tests +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For tests that belong to a mountable ``docs_bundle``, declare them in its +``tests`` list and run the generated companion target: + +.. code-block:: python + + docs_bundle( + name = "component_docs", + source_dir = "docs", + tests = [":component_test"], + ) + +.. code-block:: bash + + bazel test :component_docs_tests + +The companion target runs its declared tests, including tests from nested +bundles. Each test executable must write JUnit XML to ``XML_OUTPUT_FILE``. +This verifies the bundle's tests without making the normal documentation +targets ``testonly``. It does not inject test results into Sphinx; the +existing test-link workflow below remains unchanged. + 1. Execute tests so that ``test.xml`` files are generated: .. code-block:: bash diff --git a/docs/reference/bazel_macros.rst b/docs/reference/bazel_macros.rst index d53336a5c..0668a0824 100644 --- a/docs/reference/bazel_macros.rst +++ b/docs/reference/bazel_macros.rst @@ -121,7 +121,7 @@ site). visibility = ["//visibility:public"], ) -Signature: ``docs_bundle(name, source_dir = None, entry_doc = "index", bundles = [], scan_code = [], visibility = None)``. +Signature: ``docs_bundle(name, source_dir = None, entry_doc = "index", bundles = [], scan_code = [], tests = [], visibility = None)``. - ``source_dir`` (string, optional) Directory holding the bundle's own doc sources. It is globbed the same way as @@ -162,6 +162,14 @@ Signature: ``docs_bundle(name, source_dir = None, entry_doc = "index", bundles = same bundle be mounted at different locations by different consumers without changing its canonical entry page. +- ``tests`` (list of Bazel labels, optional) + Executable test targets verified by the automatically created + ``_tests`` target. Run it with ``bazel test :_tests``. Test + targets must create non-empty JUnit XML at the path in + ``XML_OUTPUT_FILE``. The test runner is transitively composed with nested + bundles, but it is separate from the documentation build and does not make + documentation targets ``testonly``. + Edge cases ---------- diff --git a/score_pytest/main.py b/score_pytest/main.py index 8c5afd553..350b2dec0 100644 --- a/score_pytest/main.py +++ b/score_pytest/main.py @@ -10,10 +10,14 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* +import os import sys import pytest if __name__ == "__main__": args = sys.argv[1:] + xml_output = os.environ.get("XML_OUTPUT_FILE") + if xml_output and not any(arg.startswith("--junitxml") for arg in args): + args.insert(0, "--junitxml=" + xml_output) sys.exit(pytest.main(args)) diff --git a/src/tests/docs_bzl/helpers.py b/src/tests/docs_bzl/helpers.py index e4024fe59..0937351b9 100644 --- a/src/tests/docs_bzl/helpers.py +++ b/src/tests/docs_bzl/helpers.py @@ -80,7 +80,9 @@ def run_package( expect_error: bool = False, ) -> RunResult: """Run Bazel against one small public-API test package.""" - assert bazel_cmd in ("build", "run"), "only build and run are supported" + assert bazel_cmd in ("build", "run", "test"), ( + "only build, run, and test are supported" + ) assert target.startswith(":"), "target must be relative to package" package_dir = TEST_ROOT / package diff --git a/src/tests/docs_bzl/scenarios/nested_bundles/BUILD b/src/tests/docs_bzl/scenarios/nested_bundles/BUILD index 4e31b5d08..948a94f25 100644 --- a/src/tests/docs_bzl/scenarios/nested_bundles/BUILD +++ b/src/tests/docs_bzl/scenarios/nested_bundles/BUILD @@ -13,12 +13,17 @@ load("//:bzl/mount_rules.bzl", "create_mounts_manifest") load("//:docs.bzl", "docs", "docs_bundle") +load("//:score_pytest.bzl", "score_pytest") docs_bundle( name = "child", source_dir = "child", entry_doc = "landing", scan_code = [":scan_code"], + tests = [ + ":example_test_one", + ":example_test_two", + ], ) filegroup( @@ -26,6 +31,16 @@ filegroup( srcs = ["child/example.py"], ) +score_pytest( + name = "example_test_one", + srcs = ["child/test_one.py"], +) + +score_pytest( + name = "example_test_two", + srcs = ["child/test_two.py"], +) + # The child is nested below the parent bundle before the parent itself is # mounted into the synthetic host tree below. docs_bundle( diff --git a/src/tests/docs_bzl/scenarios/nested_bundles/child/test_one.py b/src/tests/docs_bzl/scenarios/nested_bundles/child/test_one.py new file mode 100644 index 000000000..176e3924e --- /dev/null +++ b/src/tests/docs_bzl/scenarios/nested_bundles/child/test_one.py @@ -0,0 +1,16 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + + +def test_one() -> None: + assert True diff --git a/src/tests/docs_bzl/scenarios/nested_bundles/child/test_two.py b/src/tests/docs_bzl/scenarios/nested_bundles/child/test_two.py new file mode 100644 index 000000000..0713d4f7a --- /dev/null +++ b/src/tests/docs_bzl/scenarios/nested_bundles/child/test_two.py @@ -0,0 +1,16 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + + +def test_two() -> None: + assert True diff --git a/src/tests/docs_bzl/test_nested_bundles.py b/src/tests/docs_bzl/test_nested_bundles.py index 8936e42ea..2e5202007 100644 --- a/src/tests/docs_bzl/test_nested_bundles.py +++ b/src/tests/docs_bzl/test_nested_bundles.py @@ -69,3 +69,7 @@ def test_nested_bundle_aggregator_preserves_declared_order(): ).read_text(encoding="utf-8") ) assert [mount["mount_at"] for mount in manifest["mounts"]] == ["first", "second"] + + +def test_nested_bundle_runs_transitive_tests(): + run_scenario("test", "nested_bundles", ":parent_tests")