From 10df5875303229b8a497672deef35c9a9ce75695 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 31 Jul 2026 17:23:43 +0200 Subject: [PATCH 1/5] feat: add code targets to docs bundles --- bzl/bundle_rules.bzl | 68 +++++++++++++++++++ docs.bzl | 17 ++++- docs/reference/bazel_macros.rst | 10 ++- scripts_bazel/merge_sourcelinks.py | 7 +- .../tests/generate_sourcelinks_cli_test.py | 24 +++++++ scripts_bazel/tests/merge_sourcelinks_test.py | 32 +++++++++ .../generate_source_code_links_json.py | 6 +- .../docs_bzl/scenarios/nested_bundles/BUILD | 16 ++++- .../scenarios/nested_bundles/child/example.cc | 5 ++ src/tests/docs_bzl/test_nested_bundles.py | 8 +-- 10 files changed, 181 insertions(+), 12 deletions(-) create mode 100644 src/tests/docs_bzl/scenarios/nested_bundles/child/example.cc diff --git a/bzl/bundle_rules.bzl b/bzl/bundle_rules.bzl index f4884675b..541436fe1 100644 --- a/bzl/bundle_rules.bzl +++ b/bzl/bundle_rules.bzl @@ -67,6 +67,30 @@ DocsBundleInfo = provider( }, ) +CodeTargetSourcesInfo = provider( + doc = "Source files declared directly by a code target.", + fields = { + "sources": "Depset of source and header files declared by the target.", + }, +) + +def _collect_code_target_sources_impl(target, ctx): + """Collect direct source attributes without mistaking build outputs for sources.""" + source_files = [] + for attribute in ["srcs", "hdrs", "textual_hdrs"]: + if hasattr(ctx.rule.attr, attribute): + for source in getattr(ctx.rule.attr, attribute): + if type(source) == "File": + source_files.append(source) + else: + source_files.extend(source[DefaultInfo].files.to_list()) + return [CodeTargetSourcesInfo(sources = depset(source_files))] + +_collect_code_target_sources = aspect( + implementation = _collect_code_target_sources_impl, + provides = [CodeTargetSourcesInfo], +) + def _parent_index_docname(mount_at): """Choose the page that links to a bundled subtree by default.""" parent = mount_at.rsplit("/", 1)[0] if "/" in mount_at else "" @@ -321,3 +345,47 @@ def merge_bundle_sourcelinks(name, bundle, known_good = None, visibility = None) known_good = known_good, visibility = visibility, ) + +def _code_targets_sourcelinks_impl(ctx): + """Generate source links from source files declared by code targets.""" + source_files = depset(transitive = [ + target[CodeTargetSourcesInfo].sources + for target in ctx.attr.code_targets + ]) + if not source_files.to_list(): + fail("code_targets must declare source files through srcs, hdrs, or textual_hdrs") + + out = ctx.actions.declare_file(ctx.label.name + ".json") + args = ctx.actions.args() + args.add("--output", out.path) + args.add_all(source_files) + ctx.actions.run( + executable = ctx.executable._generate_sourcelinks, + arguments = [args], + inputs = source_files, + outputs = [out], + mnemonic = "GenerateCodeTargetSourcelinks", + ) + return [DefaultInfo(files = depset([out]))] + +_code_targets_sourcelinks = rule( + implementation = _code_targets_sourcelinks_impl, + attrs = { + "code_targets": attr.label_list(aspects = [_collect_code_target_sources]), + "_generate_sourcelinks": attr.label( + default = Label("//scripts_bazel:generate_sourcelinks"), + cfg = "exec", + executable = True, + ), + }, + doc = "Generates source-code links from source files owned by code targets.", +) + +def generate_code_target_sourcelinks(name, code_targets, visibility = None): + """Create source-code links for source files declared by code targets.""" + _code_targets_sourcelinks( + name = name, + code_targets = code_targets, + visibility = visibility, + ) + return ":" + name diff --git a/docs.bzl b/docs.bzl index 3b92eed59..966510df9 100644 --- a/docs.bzl +++ b/docs.bzl @@ -54,13 +54,14 @@ load( "create_bundle", "merge_bundle_sourcelinks", "external_docs_runfiles", + "generate_code_target_sourcelinks", ) load( "@score_docs_as_code//:bzl/mount_rules.bzl", "create_mounts_manifest", ) -def docs_bundle(name, source_dir = None, entry_doc = "index", bundles = [], scan_code = [], visibility = None, **kwargs): +def docs_bundle(name, source_dir = None, entry_doc = "index", bundles = [], scan_code = [], code_targets = [], visibility = None, **kwargs): """A docs bundle, optionally composed of others. Args: @@ -78,6 +79,9 @@ def docs_bundle(name, source_dir = None, entry_doc = "index", bundles = [], scan }. scan_code: Source-code targets to scan for source-code links owned by this bundle. + code_targets: Implementation targets whose directly declared source files + are scanned for source-code links. This is intended for + targets such as `cc_library` and `py_binary`. visibility: Target visibility. **kwargs: Additional attributes forwarded to the underlying rule. """ @@ -88,6 +92,13 @@ def docs_bundle(name, source_dir = None, entry_doc = "index", bundles = [], scan sourcelinks_name = name + "_sourcelinks_json" _sourcelinks_json(name = sourcelinks_name, srcs = scan_code) sourcelinks = [":" + sourcelinks_name] + if code_targets: + code_targets_sourcelinks_name = name + "_code_targets_sourcelinks_json" + generate_code_target_sourcelinks( + name = code_targets_sourcelinks_name, + code_targets = code_targets, + ) + sourcelinks.append(":" + code_targets_sourcelinks_name) # Store the source directory relative to the workspace so bundle consumers # can locate the original files without copying them. @@ -142,6 +153,7 @@ def docs( data = [], deps = [], scan_code = [], + code_targets = [], test_sources = [], known_good = None, metamodel = None, @@ -156,6 +168,8 @@ def docs( data: Additional data files to include in the documentation build. deps: Additional dependencies for the documentation build. scan_code: List of code targets to scan for source code links. + code_targets: Implementation targets whose directly declared source files + are scanned for source-code links. test_sources: Optional list of repo-relative directory paths which will be used to filter testcases for documentation generation. When empty (default), all testcases found in `bazel-testlogs` will be used. known_good: Optional label to a "known good" JSON file for source links. @@ -218,6 +232,7 @@ def docs( entry_doc = "index", bundles = bundles, scan_code = scan_code, + code_targets = code_targets, visibility = ["//visibility:public"], ) merge_bundle_sourcelinks( diff --git a/docs/reference/bazel_macros.rst b/docs/reference/bazel_macros.rst index d53336a5c..b84c5856d 100644 --- a/docs/reference/bazel_macros.rst +++ b/docs/reference/bazel_macros.rst @@ -121,7 +121,7 @@ site). visibility = ["//visibility:public"], ) -Signature: ``docs_bundle(name, source_dir = None, entry_doc = "index", bundles = [], scan_code = [], visibility = None)``. +Signature: ``docs_bundle(name, source_dir = None, entry_doc = "index", bundles = [], scan_code = [], code_targets = [], visibility = None)``. - ``source_dir`` (string, optional) Directory holding the bundle's own doc sources. It is globbed the same way as @@ -162,6 +162,14 @@ Signature: ``docs_bundle(name, source_dir = None, entry_doc = "index", bundles = same bundle be mounted at different locations by different consumers without changing its canonical entry page. +- ``code_targets`` (list of Bazel labels, optional) + Implementation targets whose directly declared ``srcs``, ``hdrs``, and + ``textual_hdrs`` are scanned for requirement tags. This describes the code + documented by the bundle (for example a ``cc_library`` or ``py_binary``); + it is distinct from ``source_dir``, which holds the documentation files. + Existing ``scan_code`` remains supported for callers that already provide + files or filegroups explicitly. + Edge cases ---------- diff --git a/scripts_bazel/merge_sourcelinks.py b/scripts_bazel/merge_sourcelinks.py index 83b18d1a3..cf08bffdc 100644 --- a/scripts_bazel/merge_sourcelinks.py +++ b/scripts_bazel/merge_sourcelinks.py @@ -52,6 +52,7 @@ def main(): all_files = [x for x in args.files if "known_good.json" not in str(x)] merged = [] + seen = set() for json_file in all_files: with open(json_file) as f: data = json.load(f) @@ -87,7 +88,11 @@ def main(): for d in data[1:]: d.update(metadata) assert isinstance(data, list), repr(data) - merged.extend(data[1:]) + for reference in data[1:]: + key = json.dumps(reference, sort_keys=True) + if key not in seen: + seen.add(key) + merged.append(reference) with open(args.output, "w") as f: json.dump(merged, f, indent=2, ensure_ascii=False) diff --git a/scripts_bazel/tests/generate_sourcelinks_cli_test.py b/scripts_bazel/tests/generate_sourcelinks_cli_test.py index e05fece8c..4aa50ed53 100644 --- a/scripts_bazel/tests/generate_sourcelinks_cli_test.py +++ b/scripts_bazel/tests/generate_sourcelinks_cli_test.py @@ -111,6 +111,30 @@ def some_function(): assert data[1]["need"] == "tool_req__docs_arch_types" +def test_generate_sourcelinks_cli_parses_cpp_traceability_tag( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + test_file = tmp_path / "test_source.cc" + test_file.write_text("// req-Id: tool_req__docs_arch_types\n") + output_file = tmp_path / "output.json" + + monkeypatch.setattr( + sys, + "argv", + [ + str(_MY_PATH.parent / "generate_sourcelinks_cli.py"), + "--output", + str(output_file), + str(test_file), + ], + ) + + assert scripts_bazel.generate_sourcelinks_cli.main() == 0 + data = json.loads(output_file.read_text()) + assert data[1]["tag"] == "// req-Id:" + assert data[1]["need"] == "tool_req__docs_arch_types" + + def test_generate_sourcelinks_cli_parse_external_module( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ): diff --git a/scripts_bazel/tests/merge_sourcelinks_test.py b/scripts_bazel/tests/merge_sourcelinks_test.py index e13d07963..69d9f15fd 100644 --- a/scripts_bazel/tests/merge_sourcelinks_test.py +++ b/scripts_bazel/tests/merge_sourcelinks_test.py @@ -159,6 +159,38 @@ def test_merge_sourcelinks_basic( ) +def test_merge_sourcelinks_deduplicates_identical_references( + create_local_json_files: tuple[Path, Path, Path], monkeypatch: pytest.MonkeyPatch +): + file1, _, output_file = create_local_json_files + + monkeypatch.setattr( + sys, + "argv", + [ + str(_MY_PATH.parent / "merge_sourcelinks.py"), + "--output", + str(output_file), + str(file1), + str(file1), + ], + ) + + assert scripts_bazel.merge_sourcelinks.main() == 0 + assert json.loads(output_file.read_text()) == [ + { + "file": "test1.py", + "line": 10, + "tag": "# req-Id:", + "need": "tool_req__docs_arch_types", + "full_line": "# req-Id: tool_req__docs_arch_types", + "repo_name": "local_repo", + "hash": "", + "url": "", + } + ] + + def test_merge_sourcelinks_with_one_empty_file( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ): diff --git a/src/extensions/score_source_code_linker/generate_source_code_links_json.py b/src/extensions/score_source_code_linker/generate_source_code_links_json.py index 7c48de9ee..582b04796 100644 --- a/src/extensions/score_source_code_linker/generate_source_code_links_json.py +++ b/src/extensions/score_source_code_linker/generate_source_code_links_json.py @@ -30,8 +30,10 @@ LOGGER = get_logger(__name__) TAGS = [ - "# " + "req-traceability:", - "# " + "req-Id:", + "# req-traceability:", + "# req-Id:", + "// req-traceability:", + "// req-Id:", ] diff --git a/src/tests/docs_bzl/scenarios/nested_bundles/BUILD b/src/tests/docs_bzl/scenarios/nested_bundles/BUILD index 4e31b5d08..86c1717f5 100644 --- a/src/tests/docs_bzl/scenarios/nested_bundles/BUILD +++ b/src/tests/docs_bzl/scenarios/nested_bundles/BUILD @@ -13,17 +13,27 @@ load("//:bzl/mount_rules.bzl", "create_mounts_manifest") load("//:docs.bzl", "docs", "docs_bundle") +load("@aspect_rules_py//py:defs.bzl", "py_binary") docs_bundle( name = "child", source_dir = "child", entry_doc = "landing", - scan_code = [":scan_code"], + code_targets = [ + ":example_binary", + ":example_library", + ], ) -filegroup( - name = "scan_code", +py_binary( + name = "example_binary", srcs = ["child/example.py"], + main = "child/example.py", +) + +cc_library( + name = "example_library", + srcs = ["child/example.cc"], ) # The child is nested below the parent bundle before the parent itself is diff --git a/src/tests/docs_bzl/scenarios/nested_bundles/child/example.cc b/src/tests/docs_bzl/scenarios/nested_bundles/child/example.cc new file mode 100644 index 000000000..a8c3055ec --- /dev/null +++ b/src/tests/docs_bzl/scenarios/nested_bundles/child/example.cc @@ -0,0 +1,5 @@ +// req-traceability: REQ_CHILD + +int example() { + return 0; +} diff --git a/src/tests/docs_bzl/test_nested_bundles.py b/src/tests/docs_bzl/test_nested_bundles.py index 8936e42ea..dacbb6420 100644 --- a/src/tests/docs_bzl/test_nested_bundles.py +++ b/src/tests/docs_bzl/test_nested_bundles.py @@ -50,10 +50,10 @@ def test_nested_bundles_render_and_preserve_metadata(): encoding="utf-8" ) ) - assert ( - sourcelinks[0]["file"] - == "src/tests/docs_bzl/scenarios/nested_bundles/child/example.py" - ) + assert {link["file"] for link in sourcelinks} == { + "src/tests/docs_bzl/scenarios/nested_bundles/child/example.py", + "src/tests/docs_bzl/scenarios/nested_bundles/child/example.cc", + } assert (result.build_dir / "concepts" / "example_bundle" / "index.html").is_file() assert ( result.build_dir / "concepts" / "example_bundle" / "child" / "landing.html" From 2c277416933b3255a98f1b549938ff3d865e1c25 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 31 Jul 2026 17:40:13 +0200 Subject: [PATCH 2/5] fix: make code target scans recursive --- bzl/bundle_rules.bzl | 62 ++++++++++++------- docs.bzl | 16 ++--- docs/how-to/dashboards_and_quality_gates.rst | 2 +- docs/how-to/setup.md | 3 +- docs/how-to/source_to_doc_links.rst | 30 ++++++--- .../extensions/source_code_linker.md | 8 ++- docs/reference/bazel_macros.rst | 29 ++++++--- .../generate_source_code_links_json.py | 8 +-- .../docs_bzl/scenarios/nested_bundles/BUILD | 8 ++- .../nested_bundles/child/example_main.cc | 3 + 10 files changed, 111 insertions(+), 58 deletions(-) create mode 100644 src/tests/docs_bzl/scenarios/nested_bundles/child/example_main.cc diff --git a/bzl/bundle_rules.bzl b/bzl/bundle_rules.bzl index 541436fe1..e4dbb12cd 100644 --- a/bzl/bundle_rules.bzl +++ b/bzl/bundle_rules.bzl @@ -68,27 +68,45 @@ DocsBundleInfo = provider( ) CodeTargetSourcesInfo = provider( - doc = "Source files declared directly by a code target.", + doc = "Source files collected from an implementation target and its dependencies.", fields = { - "sources": "Depset of source and header files declared by the target.", + "sources": "Depset of direct and transitive source files.", }, ) -def _collect_code_target_sources_impl(target, ctx): - """Collect direct source attributes without mistaking build outputs for sources.""" +def _source_files_from_attributes(ctx): + """Return files explicitly declared as source or header inputs by one rule.""" source_files = [] - for attribute in ["srcs", "hdrs", "textual_hdrs"]: - if hasattr(ctx.rule.attr, attribute): - for source in getattr(ctx.rule.attr, attribute): - if type(source) == "File": - source_files.append(source) - else: - source_files.extend(source[DefaultInfo].files.to_list()) - return [CodeTargetSourcesInfo(sources = depset(source_files))] + for attribute_name in ["srcs", "hdrs", "textual_hdrs"]: + if not hasattr(ctx.rule.attr, attribute_name): + continue + for source in getattr(ctx.rule.attr, attribute_name): + if type(source) == "File": + source_files.append(source) + else: + source_files.extend(source[DefaultInfo].files.to_list()) + return source_files + +def _collect_code_target_sources_impl(target, ctx): + """Collect source files from an implementation target and its ``deps`` tree.""" + dependency_sources = [] + if hasattr(ctx.rule.attr, "deps"): + dependency_sources = [ + dependency[CodeTargetSourcesInfo].sources + for dependency in ctx.rule.attr.deps + ] + return [CodeTargetSourcesInfo( + sources = depset( + direct = _source_files_from_attributes(ctx), + transitive = dependency_sources, + ), + )] _collect_code_target_sources = aspect( implementation = _collect_code_target_sources_impl, + attr_aspects = ["deps"], provides = [CodeTargetSourcesInfo], + doc = "Collects sources recursively through standard implementation dependencies.", ) def _parent_index_docname(mount_at): @@ -347,7 +365,7 @@ def merge_bundle_sourcelinks(name, bundle, known_good = None, visibility = None) ) def _code_targets_sourcelinks_impl(ctx): - """Generate source links from source files declared by code targets.""" + """Generate one source-link cache for the implementation targets of a bundle.""" source_files = depset(transitive = [ target[CodeTargetSourcesInfo].sources for target in ctx.attr.code_targets @@ -355,18 +373,18 @@ def _code_targets_sourcelinks_impl(ctx): if not source_files.to_list(): fail("code_targets must declare source files through srcs, hdrs, or textual_hdrs") - out = ctx.actions.declare_file(ctx.label.name + ".json") - args = ctx.actions.args() - args.add("--output", out.path) - args.add_all(source_files) + output = ctx.actions.declare_file(ctx.label.name + ".json") + arguments = ctx.actions.args() + arguments.add("--output", output.path) + arguments.add_all(source_files) ctx.actions.run( executable = ctx.executable._generate_sourcelinks, - arguments = [args], + arguments = [arguments], inputs = source_files, - outputs = [out], + outputs = [output], mnemonic = "GenerateCodeTargetSourcelinks", ) - return [DefaultInfo(files = depset([out]))] + return [DefaultInfo(files = depset([output]))] _code_targets_sourcelinks = rule( implementation = _code_targets_sourcelinks_impl, @@ -378,11 +396,11 @@ _code_targets_sourcelinks = rule( executable = True, ), }, - doc = "Generates source-code links from source files owned by code targets.", + doc = "Generates source-code links from implementation target source files.", ) def generate_code_target_sourcelinks(name, code_targets, visibility = None): - """Create source-code links for source files declared by code targets.""" + """Create a cached source-link JSON file for one documentation bundle.""" _code_targets_sourcelinks( name = name, code_targets = code_targets, diff --git a/docs.bzl b/docs.bzl index 966510df9..fda4eb964 100644 --- a/docs.bzl +++ b/docs.bzl @@ -77,11 +77,11 @@ def docs_bundle(name, source_dir = None, entry_doc = "index", bundles = [], scan "mount_at": , "attach_to": }. - scan_code: Source-code targets to scan for source-code links owned by this - bundle. - code_targets: Implementation targets whose directly declared source files - are scanned for source-code links. This is intended for - targets such as `cc_library` and `py_binary`. + scan_code: Deprecated. Explicit source files or filegroups to scan for + source-code links. Use `code_targets` for implementation targets. + code_targets: Implementation targets to scan for source-code links. Their + source files and the source files of their dependencies are + collected recursively. visibility: Target visibility. **kwargs: Additional attributes forwarded to the underlying rule. """ @@ -167,9 +167,9 @@ def docs( source_dir: The source directory containing documentation files. Defaults to "docs". data: Additional data files to include in the documentation build. deps: Additional dependencies for the documentation build. - scan_code: List of code targets to scan for source code links. - code_targets: Implementation targets whose directly declared source files - are scanned for source-code links. + scan_code: Deprecated. Explicit source files or filegroups to scan for source + code links. Use `code_targets` for implementation targets. + code_targets: Implementation targets to scan recursively for source code links. test_sources: Optional list of repo-relative directory paths which will be used to filter testcases for documentation generation. When empty (default), all testcases found in `bazel-testlogs` will be used. known_good: Optional label to a "known good" JSON file for source links. diff --git a/docs/how-to/dashboards_and_quality_gates.rst b/docs/how-to/dashboards_and_quality_gates.rst index f10607c89..9827808a0 100644 --- a/docs/how-to/dashboards_and_quality_gates.rst +++ b/docs/how-to/dashboards_and_quality_gates.rst @@ -145,7 +145,7 @@ Recommended Rollout For a new consumer repository: 1. Start with local-only metrics. -2. Enable ``scan_code`` and verify ``source_code_link`` coverage first. +2. Enable ``code_targets`` and verify ``source_code_link`` coverage first. 3. Add test metadata and verify ``testlink`` coverage. 4. Introduce modest thresholds in CI. 5. Raise thresholds over time as the repository matures. diff --git a/docs/how-to/setup.md b/docs/how-to/setup.md index 644e679ef..f2e537b5e 100644 --- a/docs/how-to/setup.md +++ b/docs/how-to/setup.md @@ -83,7 +83,8 @@ The `docs()` macro accepts the following arguments: | `source_dir` | Directory of documentation source files (RST, MD) | Yes | | `data` | List of `needs_json` targets that should be included in the documentation | No | | `deps` | Additional Bazel Python dependencies | No | -| `scan_code` | Source code targets to scan for traceability tags | No | +| `code_targets` | Implementation targets to scan recursively for traceability tags | No | +| `scan_code` | Deprecated: explicit source files or filegroups to scan | No | | `known_good` | Label to a "known good" JSON file for source links | No | | `metamodel` | Label to a custom `metamodel.yaml` that replaces the default metamodel | No | diff --git a/docs/how-to/source_to_doc_links.rst b/docs/how-to/source_to_doc_links.rst index 29341762f..ae3077a23 100644 --- a/docs/how-to/source_to_doc_links.rst +++ b/docs/how-to/source_to_doc_links.rst @@ -39,21 +39,27 @@ For other languages (C++, Rust, etc.), use the appropriate comment syntax. Scanning Source Code for Links ------------------------------ -In your ``BUILD`` files, you specify which source files to scan -with ``filegroup`` or ``glob`` or whatever Bazel mechanism you prefer. -Finally, pass the scan results to the ``docs`` rule as ``scan_code`` attribute. +In your ``BUILD`` files, pass the implementation targets to the ``docs`` rule +as ``code_targets``. Their ``srcs``, ``hdrs``, and ``textual_hdrs`` are scanned, +including those of their ``deps`` recursively. This means that a +``cc_executable`` also covers source files from the ``cc_library`` targets it +uses. .. code-block:: starlark - :emphasize-lines: 15 + :emphasize-lines: 14 :linenos: - filegroup( - name = "some_sources", + cc_library( + name = "some_library", srcs = [ - "foo.py", "bar.cpp", - "data.yaml", - ] + glob(["subdir/**/*.py"]), + ], + ) + + cc_executable( + name = "some_application", + srcs = ["main.cpp"], + deps = [":some_library"], ) docs( @@ -61,5 +67,9 @@ Finally, pass the scan results to the ``docs`` rule as ``scan_code`` attribute. "@score_process//:needs_json", ], source_dir = "docs", - scan_code = [":some_sources"], + code_targets = [":some_application"], ) + +The older ``scan_code`` parameter remains available for existing configurations +that explicitly provide files or filegroups, but it is deprecated. Prefer +``code_targets`` for new configurations. diff --git a/docs/internals/extensions/source_code_linker.md b/docs/internals/extensions/source_code_linker.md index ec25ef6f3..3a657cb48 100644 --- a/docs/internals/extensions/source_code_linker.md +++ b/docs/internals/extensions/source_code_linker.md @@ -43,8 +43,12 @@ The Bazel parts are responsible for producing the **intermediate caches** that t (step-1-per-repository-cache-generation)= #### Step 1: Per-repository cache generation -All files provided via the `scan_code` attribute to the docs macro will be scanned for the requirement tags. -A *per repository JSON cache* will be generated and saved. +Each `docs_bundle` scans the source files selected by its `code_targets` +attribute. The targets' `srcs`, `hdrs`, and `textual_hdrs`, including their +dependencies, are collected recursively. A *per bundle JSON cache* is then +generated and saved; Bazel reuses it until its source inputs change. +`scan_code` remains available for explicit files or filegroups, but is +deprecated. This script `scripts_bazel/generate_sourcelinks_cli.py` finds all codelinks per file, and gathers them into one JSON cache per repository. It also adds metadata to each needlink that is needed in further steps. diff --git a/docs/reference/bazel_macros.rst b/docs/reference/bazel_macros.rst index b84c5856d..b1e61b3d6 100644 --- a/docs/reference/bazel_macros.rst +++ b/docs/reference/bazel_macros.rst @@ -72,9 +72,17 @@ Minimal example (root ``BUILD``) If you don't provide the necessary Sphinx packages, this function adds its own (but checks for conflicts). -- ``scan_code`` (list of bazel labels) - Source code targets to scan for traceability tags (``req-Id:`` annotations). - Used to generate the source-code-link JSON that maps tags back to source files. +- ``code_targets`` (list of Bazel labels) + Implementation targets to scan for traceability tags (``req-Id:`` annotations). + Their declared ``srcs``, ``hdrs``, and ``textual_hdrs`` are collected through + their ``deps`` recursively. Each documentation bundle produces one cache for + its declared targets; Bazel reuses that cache while its inputs are unchanged. + The generated JSON is supplied to ``live_preview`` just like a normal + documentation build. + +- ``scan_code`` (list of Bazel labels, deprecated) + Explicit source files or filegroups to scan. Use ``code_targets`` for + implementation targets; it follows their dependencies automatically. - ``metamodel`` (bazel label, optional) Path to a custom ``metamodel.yaml`` file. @@ -163,12 +171,15 @@ Signature: ``docs_bundle(name, source_dir = None, entry_doc = "index", bundles = changing its canonical entry page. - ``code_targets`` (list of Bazel labels, optional) - Implementation targets whose directly declared ``srcs``, ``hdrs``, and - ``textual_hdrs`` are scanned for requirement tags. This describes the code - documented by the bundle (for example a ``cc_library`` or ``py_binary``); - it is distinct from ``source_dir``, which holds the documentation files. - Existing ``scan_code`` remains supported for callers that already provide - files or filegroups explicitly. + Implementation targets whose ``srcs``, ``hdrs``, and ``textual_hdrs`` are + scanned for requirement tags. Sources from their ``deps`` are included + recursively, so declaring a ``cc_executable`` also scans the libraries it + uses. The bundle owns one cached scan result; Bazel only regenerates it when + its collected source inputs change. + +- ``scan_code`` (list of Bazel labels, deprecated) + Explicit source files or filegroups to scan. Prefer ``code_targets`` for + implementation targets. Edge cases ---------- diff --git a/src/extensions/score_source_code_linker/generate_source_code_links_json.py b/src/extensions/score_source_code_linker/generate_source_code_links_json.py index 582b04796..bdeddeaf7 100644 --- a/src/extensions/score_source_code_linker/generate_source_code_links_json.py +++ b/src/extensions/score_source_code_linker/generate_source_code_links_json.py @@ -30,10 +30,10 @@ LOGGER = get_logger(__name__) TAGS = [ - "# req-traceability:", - "# req-Id:", - "// req-traceability:", - "// req-Id:", + "# " + "req-traceability:", + "# " + "req-Id:", + "// " + "req-traceability:", + "// " + "req-Id:", ] diff --git a/src/tests/docs_bzl/scenarios/nested_bundles/BUILD b/src/tests/docs_bzl/scenarios/nested_bundles/BUILD index 86c1717f5..9b2f19bd1 100644 --- a/src/tests/docs_bzl/scenarios/nested_bundles/BUILD +++ b/src/tests/docs_bzl/scenarios/nested_bundles/BUILD @@ -21,7 +21,7 @@ docs_bundle( entry_doc = "landing", code_targets = [ ":example_binary", - ":example_library", + ":example_executable", ], ) @@ -36,6 +36,12 @@ cc_library( srcs = ["child/example.cc"], ) +cc_binary( + name = "example_executable", + srcs = ["child/example_main.cc"], + deps = [":example_library"], +) + # The child is nested below the parent bundle before the parent itself is # mounted into the synthetic host tree below. docs_bundle( diff --git a/src/tests/docs_bzl/scenarios/nested_bundles/child/example_main.cc b/src/tests/docs_bzl/scenarios/nested_bundles/child/example_main.cc new file mode 100644 index 000000000..4cce7f667 --- /dev/null +++ b/src/tests/docs_bzl/scenarios/nested_bundles/child/example_main.cc @@ -0,0 +1,3 @@ +int main() { + return 0; +} From fc59863f3b28cea050113f2b9bf6df0703df875e Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 31 Jul 2026 17:58:58 +0200 Subject: [PATCH 3/5] feat: support filegroups in code targets --- bzl/bundle_rules.bzl | 4 +- docs.bzl | 10 +- docs/how-to/source_to_doc_links.rst | 2 +- docs/reference/bazel_macros.rst | 23 ++-- scripts_bazel/merge_sourcelinks.py | 113 +++++++++++------- .../docs_bzl/scenarios/nested_bundles/BUILD | 13 ++ .../nested_bundles/child/filegroup_source.py | 14 +++ src/tests/docs_bzl/test_nested_bundles.py | 1 + 8 files changed, 116 insertions(+), 64 deletions(-) create mode 100644 src/tests/docs_bzl/scenarios/nested_bundles/child/filegroup_source.py diff --git a/bzl/bundle_rules.bzl b/bzl/bundle_rules.bzl index e4dbb12cd..f55cc138a 100644 --- a/bzl/bundle_rules.bzl +++ b/bzl/bundle_rules.bzl @@ -75,7 +75,7 @@ CodeTargetSourcesInfo = provider( ) def _source_files_from_attributes(ctx): - """Return files explicitly declared as source or header inputs by one rule.""" + """Return files explicitly declared through source or header attributes.""" source_files = [] for attribute_name in ["srcs", "hdrs", "textual_hdrs"]: if not hasattr(ctx.rule.attr, attribute_name): @@ -371,7 +371,7 @@ def _code_targets_sourcelinks_impl(ctx): for target in ctx.attr.code_targets ]) if not source_files.to_list(): - fail("code_targets must declare source files through srcs, hdrs, or textual_hdrs") + fail("code_targets must declare source files through filegroups, srcs, hdrs, or textual_hdrs") output = ctx.actions.declare_file(ctx.label.name + ".json") arguments = ctx.actions.args() diff --git a/docs.bzl b/docs.bzl index fda4eb964..a0d7e189e 100644 --- a/docs.bzl +++ b/docs.bzl @@ -79,9 +79,9 @@ def docs_bundle(name, source_dir = None, entry_doc = "index", bundles = [], scan }. scan_code: Deprecated. Explicit source files or filegroups to scan for source-code links. Use `code_targets` for implementation targets. - code_targets: Implementation targets to scan for source-code links. Their - source files and the source files of their dependencies are - collected recursively. + code_targets: Implementation targets or filegroups to scan for source-code + links. Implementation target source files and their dependencies + are collected recursively; filegroups expand to their files. visibility: Target visibility. **kwargs: Additional attributes forwarded to the underlying rule. """ @@ -169,7 +169,9 @@ def docs( deps: Additional dependencies for the documentation build. scan_code: Deprecated. Explicit source files or filegroups to scan for source code links. Use `code_targets` for implementation targets. - code_targets: Implementation targets to scan recursively for source code links. + code_targets: Implementation targets or filegroups to scan for source code + links. Implementation targets are scanned recursively; filegroups + expand to their files. test_sources: Optional list of repo-relative directory paths which will be used to filter testcases for documentation generation. When empty (default), all testcases found in `bazel-testlogs` will be used. known_good: Optional label to a "known good" JSON file for source links. diff --git a/docs/how-to/source_to_doc_links.rst b/docs/how-to/source_to_doc_links.rst index ae3077a23..63d3f5def 100644 --- a/docs/how-to/source_to_doc_links.rst +++ b/docs/how-to/source_to_doc_links.rst @@ -43,7 +43,7 @@ In your ``BUILD`` files, pass the implementation targets to the ``docs`` rule as ``code_targets``. Their ``srcs``, ``hdrs``, and ``textual_hdrs`` are scanned, including those of their ``deps`` recursively. This means that a ``cc_executable`` also covers source files from the ``cc_library`` targets it -uses. +uses. You may also pass filegroups; their files are scanned directly. .. code-block:: starlark :emphasize-lines: 14 diff --git a/docs/reference/bazel_macros.rst b/docs/reference/bazel_macros.rst index b1e61b3d6..18092153e 100644 --- a/docs/reference/bazel_macros.rst +++ b/docs/reference/bazel_macros.rst @@ -73,12 +73,13 @@ Minimal example (root ``BUILD``) this function adds its own (but checks for conflicts). - ``code_targets`` (list of Bazel labels) - Implementation targets to scan for traceability tags (``req-Id:`` annotations). - Their declared ``srcs``, ``hdrs``, and ``textual_hdrs`` are collected through - their ``deps`` recursively. Each documentation bundle produces one cache for - its declared targets; Bazel reuses that cache while its inputs are unchanged. - The generated JSON is supplied to ``live_preview`` just like a normal - documentation build. + Implementation targets or filegroups to scan for traceability tags + (``req-Id:`` annotations). Implementation target ``srcs``, ``hdrs``, and + ``textual_hdrs`` are collected through their ``deps`` recursively; filegroups + expand to their files. Each documentation bundle produces one cache for its + declared targets; Bazel reuses that cache while its inputs are unchanged. The + generated JSON is supplied to ``live_preview`` just like a normal documentation + build. - ``scan_code`` (list of Bazel labels, deprecated) Explicit source files or filegroups to scan. Use ``code_targets`` for @@ -171,11 +172,11 @@ Signature: ``docs_bundle(name, source_dir = None, entry_doc = "index", bundles = changing its canonical entry page. - ``code_targets`` (list of Bazel labels, optional) - Implementation targets whose ``srcs``, ``hdrs``, and ``textual_hdrs`` are - scanned for requirement tags. Sources from their ``deps`` are included - recursively, so declaring a ``cc_executable`` also scans the libraries it - uses. The bundle owns one cached scan result; Bazel only regenerates it when - its collected source inputs change. + Implementation targets or filegroups to scan for requirement tags. + Implementation target ``srcs``, ``hdrs``, and ``textual_hdrs`` are collected + recursively from their ``deps``; filegroups expand to their files. The bundle + owns one cached scan result; Bazel only regenerates it when its collected source + inputs change. - ``scan_code`` (list of Bazel labels, deprecated) Explicit source files or filegroups to scan. Prefer ``code_targets`` for diff --git a/scripts_bazel/merge_sourcelinks.py b/scripts_bazel/merge_sourcelinks.py index cf08bffdc..d12b88f53 100644 --- a/scripts_bazel/merge_sourcelinks.py +++ b/scripts_bazel/merge_sourcelinks.py @@ -20,6 +20,7 @@ import logging import sys from pathlib import Path +from typing import cast from src.extensions.score_source_code_linker.helpers import parse_info_from_known_good @@ -27,6 +28,62 @@ logger = logging.getLogger(__name__) +def _reference_key(reference: dict[str, object]) -> str: + """Return a stable, hashable representation of one source-link reference.""" + # References are dictionaries and therefore cannot be added to ``seen`` directly. + # Sorting their JSON keys makes semantically identical dictionaries produce the + # same key even when their input key order differs. + return json.dumps(reference, sort_keys=True) + + +def _merge_sourcelinks_file( + json_file: Path, + known_good: Path | None, + merged: list[dict[str, object]], + seen: set[str], +) -> None: + """Add the unique references from one sourcelinks JSON file to ``merged``.""" + with open(json_file, encoding="utf-8") as file: + data = cast(list[object], json.load(file)) + if not data: + return + + raw_metadata = data[0] + if not isinstance(raw_metadata, dict) or "repo_name" not in raw_metadata: + logger.warning( + f"Unexpected schema in sourcelinks file '{json_file}': " + "expected first element to be a metadata dict " + "with a 'repo_name' key. " + ) + return + metadata = cast(dict[str, object], raw_metadata) + repo_name = metadata["repo_name"] + if not isinstance(repo_name, str): + logger.warning( + f"Unexpected schema in sourcelinks file '{json_file}': " + "expected metadata 'repo_name' to be a string. " + ) + return + + # A known-good file is optional for standalone builds that include + # documentation from external modules. In that case, keep the metadata + # produced by the individual sourcelinks file. + if known_good and repo_name and repo_name != "local_repo": + hash, repo = parse_info_from_known_good( + known_good_json=known_good, repo_name=repo_name + ) + metadata["hash"] = hash + metadata["url"] = repo + + for raw_reference in data[1:]: + reference = cast(dict[str, object], raw_reference) + reference.update(metadata) + key = _reference_key(reference) + if key not in seen: + seen.add(key) + merged.append(reference) + + def main(): parser = argparse.ArgumentParser( description="Merge multiple sourcelinks JSON files into one" @@ -39,6 +96,7 @@ def main(): ) _ = parser.add_argument( "--known_good", + type=Path, help="Path to a required 'known good' JSON file (provided by Bazel).", ) _ = parser.add_argument( @@ -49,52 +107,15 @@ def main(): ) args = parser.parse_args() - all_files = [x for x in args.files if "known_good.json" not in str(x)] - - merged = [] - seen = set() - for json_file in all_files: - with open(json_file) as f: - data = json.load(f) - # If the file is empty e.g. '[]' there is nothing to parse, we continue - if not data: - continue - metadata = data[0] - if not isinstance(metadata, dict) or "repo_name" not in metadata: - logger.warning( - f"Unexpected schema in sourcelinks file '{json_file}': " - "expected first element to be a metadata dict " - "with a 'repo_name' key. " - ) - # As we can't deal with bad JSON structure we just skip it - continue - # A known-good file is optional for standalone builds that include - # documentation from external modules. In that case, keep the - # metadata produced by the individual sourcelinks file. - if ( - args.known_good - and metadata["repo_name"] - and metadata["repo_name"] != "local_repo" - ): - hash, repo = parse_info_from_known_good( - known_good_json=args.known_good, repo_name=metadata["repo_name"] - ) - metadata["hash"] = hash - metadata["url"] = repo - # In the case that 'metadata[repo_name]' is 'local_module' - # hash & url are already existing and empty inside of 'metadata' - # Therefore all 3 keys will be written to needlinks in each branch - - for d in data[1:]: - d.update(metadata) - assert isinstance(data, list), repr(data) - for reference in data[1:]: - key = json.dumps(reference, sort_keys=True) - if key not in seen: - seen.add(key) - merged.append(reference) - with open(args.output, "w") as f: - json.dump(merged, f, indent=2, ensure_ascii=False) + + merged: list[dict[str, object]] = [] + seen: set[str] = set() + for json_file in args.files: + if "known_good.json" not in str(json_file): + _merge_sourcelinks_file(json_file, args.known_good, merged, seen) + + with open(args.output, "w", encoding="utf-8") as file: + json.dump(merged, file, indent=2, ensure_ascii=False) logger.info(f"Merged {len(args.files)} files into {len(merged)} total references") return 0 diff --git a/src/tests/docs_bzl/scenarios/nested_bundles/BUILD b/src/tests/docs_bzl/scenarios/nested_bundles/BUILD index 9b2f19bd1..57a3a30e0 100644 --- a/src/tests/docs_bzl/scenarios/nested_bundles/BUILD +++ b/src/tests/docs_bzl/scenarios/nested_bundles/BUILD @@ -22,9 +22,22 @@ docs_bundle( code_targets = [ ":example_binary", ":example_executable", + ":nested_filegroup_sources", ], ) +filegroup( + name = "filegroup_sources", + srcs = ["child/filegroup_source.py"], +) + +# Intentionally reference the same file through a second filegroup to verify +# that code_targets resolves nested filegroups without duplicate links. +filegroup( + name = "nested_filegroup_sources", + srcs = [":filegroup_sources"], +) + py_binary( name = "example_binary", srcs = ["child/example.py"], diff --git a/src/tests/docs_bzl/scenarios/nested_bundles/child/filegroup_source.py b/src/tests/docs_bzl/scenarios/nested_bundles/child/filegroup_source.py new file mode 100644 index 000000000..17296fd48 --- /dev/null +++ b/src/tests/docs_bzl/scenarios/nested_bundles/child/filegroup_source.py @@ -0,0 +1,14 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +# req-traceability: REQ_FILEGROUP diff --git a/src/tests/docs_bzl/test_nested_bundles.py b/src/tests/docs_bzl/test_nested_bundles.py index dacbb6420..98390c1a0 100644 --- a/src/tests/docs_bzl/test_nested_bundles.py +++ b/src/tests/docs_bzl/test_nested_bundles.py @@ -53,6 +53,7 @@ def test_nested_bundles_render_and_preserve_metadata(): assert {link["file"] for link in sourcelinks} == { "src/tests/docs_bzl/scenarios/nested_bundles/child/example.py", "src/tests/docs_bzl/scenarios/nested_bundles/child/example.cc", + "src/tests/docs_bzl/scenarios/nested_bundles/child/filegroup_source.py", } assert (result.build_dir / "concepts" / "example_bundle" / "index.html").is_file() assert ( From a81b3d331ebb56a51596ca1f00bc7b739b39fa75 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 31 Jul 2026 18:08:34 +0200 Subject: [PATCH 4/5] fix: use generated source link label --- docs.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs.bzl b/docs.bzl index a0d7e189e..9c6865ae5 100644 --- a/docs.bzl +++ b/docs.bzl @@ -94,11 +94,11 @@ def docs_bundle(name, source_dir = None, entry_doc = "index", bundles = [], scan sourcelinks = [":" + sourcelinks_name] if code_targets: code_targets_sourcelinks_name = name + "_code_targets_sourcelinks_json" - generate_code_target_sourcelinks( + code_targets_sourcelinks = generate_code_target_sourcelinks( name = code_targets_sourcelinks_name, code_targets = code_targets, ) - sourcelinks.append(":" + code_targets_sourcelinks_name) + sourcelinks.append(code_targets_sourcelinks) # Store the source directory relative to the workspace so bundle consumers # can locate the original files without copying them. From 74603acdd4ccb9b048f3a741e453e56c0e1ab401 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 31 Jul 2026 18:10:06 +0200 Subject: [PATCH 5/5] refactor: simplify source link target creation --- docs.bzl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs.bzl b/docs.bzl index 9c6865ae5..0c5d89b7b 100644 --- a/docs.bzl +++ b/docs.bzl @@ -93,9 +93,8 @@ def docs_bundle(name, source_dir = None, entry_doc = "index", bundles = [], scan _sourcelinks_json(name = sourcelinks_name, srcs = scan_code) sourcelinks = [":" + sourcelinks_name] if code_targets: - code_targets_sourcelinks_name = name + "_code_targets_sourcelinks_json" code_targets_sourcelinks = generate_code_target_sourcelinks( - name = code_targets_sourcelinks_name, + name = name + "_code_targets_sourcelinks_json", code_targets = code_targets, ) sourcelinks.append(code_targets_sourcelinks)