diff --git a/BUILD b/BUILD index 586a4595f..38137587b 100644 --- a/BUILD +++ b/BUILD @@ -17,8 +17,8 @@ package(default_visibility = ["//visibility:public"]) exports_files(["pyproject.toml"]) docs( - data = [ - "@score_process//:needs_json", + external_needs = [ + "@score_process//:needs_json_file", ], bundles = [ { diff --git a/docs.bzl b/docs.bzl index 3b92eed59..394351132 100644 --- a/docs.bzl +++ b/docs.bzl @@ -141,6 +141,7 @@ def docs( source_dir = "docs", data = [], deps = [], + external_needs = [], scan_code = [], test_sources = [], known_good = None, @@ -155,6 +156,7 @@ def docs( source_dir: The source directory containing documentation files. Defaults to "docs". data: Additional data files to include in the documentation build. deps: Additional dependencies for the documentation build. + external_needs: List of external needs targets to include in the documentation build. scan_code: List of code targets to scan for source code links. test_sources: Optional list of repo-relative directory paths which will be used to filter testcases for documentation generation. When empty (default), all testcases found in `bazel-testlogs` will be used. @@ -171,6 +173,7 @@ def docs( Note: a bundle label may also point at another module's auto-exposed bundle, e.g. "@score_process//:docs_bundle". """ + # HINT: keep documentation sync docs/reference/bazel_macros.rst source_config = ":" + ("" if source_dir == "." else source_dir + "/") + "conf.py" @@ -205,7 +208,7 @@ def docs( sphinx_build_binary( name = "sphinx_build", visibility = ["//visibility:private"], - data = data + metamodel_label + [":docs_bundle"], + data = data + external_needs + metamodel_label + [":docs_bundle"], deps = deps, ) @@ -236,13 +239,14 @@ def docs( # complete bundle here would add those files to runfiles and could collide # with the executable target name (for example ``docs`` and ``docs/``). # External bundles do need runfiles, so keep only those sources. - docs_data = data + metamodel_label + [":sourcelinks_json", ":_external_docs_runfiles"] + mounts_manifest_label + docs_data = data + external_needs + metamodel_label + [":sourcelinks_json", ":_external_docs_runfiles"] + mounts_manifest_label docs_env = { "SOURCE_DIRECTORY": source_dir, "PACKAGE_DIR": native.package_name(), "TEST_SOURCES": str(test_sources), "DATA": str(data), + "EXTERNAL_NEEDS_FILES": str(external_needs), # `bazel run` starts from a runfiles tree, so this logical path is # resolved by score_mounts through ``RUNFILES_DIR``. "MOUNTS_MANIFEST": "$(rlocationpath :_mounts_manifest)" if bundles else "", @@ -317,7 +321,7 @@ def docs( "-T", # show more details in case of errors "--jobs", "auto", - "--define=external_needs_source=" + str(data), + "--define=external_needs_source=" + str(data + external_needs), "--define=score_sourcelinks_json=$(location :sourcelinks_json)", "--define=score_source_code_linker_plain_links=1", ] + ( @@ -327,7 +331,7 @@ def docs( ) + (["--define=score_metamodel_yaml=$(location " + str(metamodel) + ")"] if metamodel else []), formats = ["needs"], sphinx = ":sphinx_build", - tools = data + metamodel_label + [":sourcelinks_json", ":docs_bundle"] + mounts_manifest_label, + tools = data + external_needs + metamodel_label + [":sourcelinks_json", ":docs_bundle"] + mounts_manifest_label, visibility = ["//visibility:public"], # Persistent workers cause stale symlinks after dependency version # changes, corrupting the Bazel cache. @@ -342,6 +346,16 @@ def docs( visibility = ["//visibility:public"], ) + native.genrule( + # In contrast to the "needs_json" target represents *only* the needs.json file, + # not the whole needs build output. + name = "needs_json_file", + srcs = [":needs_json"], + outs = ["needs.json"], + cmd = "cp $(location :needs_json)/needs.json $@", + visibility = ["//visibility:public"], + ) + native.alias( name = "traceability_gate", actual = Label("//scripts_bazel:traceability_gate"), diff --git a/docs/how-to/other_modules.rst b/docs/how-to/other_modules.rst index 07f40902b..97ab49c3b 100644 --- a/docs/how-to/other_modules.rst +++ b/docs/how-to/other_modules.rst @@ -41,9 +41,10 @@ A minimal example (add or extend the existing `bazel_deps` stanza): 2a) Import the other module's built inventory ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The documentation build in this project is exposed via a Bazel macro/rule that accepts a `data` parameter. -Add the external module's ``:needs_json`` target to that list -to have their needs elements available for cross-referencing. +The documentation build is exposed via a Bazel macro that accepts an ``external_needs`` parameter +for external ``:needs_json_file`` targets. +Use ``external_needs`` instead of ``data`` when the target produces needs JSON — +``data`` is meant for non-needs runfiles (e.g. custom tool outputs). Example `BUILD` snippet (consumer module): @@ -106,3 +107,21 @@ Which results in: See the `Sphinx-Needs documentation `_ for more details on cross-referencing needs. + +Data vs. external_needs +~~~~~~~~~~~~~~~~~~~~~~~ + +It might seem like ``data`` and ``external_needs`` behave nearly the same. + +.. code-block:: starlark + + docs( + data = ["@score_process//:needs_json"], + external_needs = ["@score_process//:needs_json_file"], # same? + ) + +For a ``:docs`` they behave the same, +but for ``:docs_combo`` they do not. +In a combo build, the ``data`` are treated differently and will include the documentation +instead of just link the needs online. +However, ``external_needs`` will always be hyperlinks. diff --git a/docs/how-to/setup.md b/docs/how-to/setup.md index 644e679ef..2f7bca8c3 100644 --- a/docs/how-to/setup.md +++ b/docs/how-to/setup.md @@ -73,19 +73,7 @@ docs( ) ``` - -#### Configuration Options - -The `docs()` macro accepts the following arguments: - -| Parameter | Description | Required | -|-----------|-------------|----------| -| `source_dir` | Directory of documentation source files (RST, MD) | Yes | -| `data` | List of `needs_json` targets that should be included in the documentation | No | -| `deps` | Additional Bazel Python dependencies | No | -| `scan_code` | Source code targets to scan for traceability tags | No | -| `known_good` | Label to a "known good" JSON file for source links | No | -| `metamodel` | Label to a custom `metamodel.yaml` that replaces the default metamodel | No | +For configuration options see {ref}`docs_bazel-macros`. ### 4. Copy conf.py diff --git a/docs/reference/bazel_macros.rst b/docs/reference/bazel_macros.rst index d53336a5c..17a21cb61 100644 --- a/docs/reference/bazel_macros.rst +++ b/docs/reference/bazel_macros.rst @@ -76,6 +76,12 @@ Minimal example (root ``BUILD``) Source code targets to scan for traceability tags (``req-Id:`` annotations). Used to generate the source-code-link JSON that maps tags back to source files. +- ``external_needs`` (list of bazel labels) + External ``:needs_json_file`` targets from other modules/repositories + for referencing their needs. + Users must specify ``:needs_json_file`` explicitly (not ``:needs_json``). + Works the same during ``:docs`` and ``:docs_combo``. + - ``metamodel`` (bazel label, optional) Path to a custom ``metamodel.yaml`` file. When set, the ``score_metamodel`` extension loads **this file instead of** the default metamodel. diff --git a/src/extensions/score_metamodel/external_needs.py b/src/extensions/score_metamodel/external_needs.py index 980dfe1c2..44b7380ea 100644 --- a/src/extensions/score_metamodel/external_needs.py +++ b/src/extensions/score_metamodel/external_needs.py @@ -54,7 +54,7 @@ def _parse_bazel_external_need(s: str) -> ExternalNeedsSource | None: repo, path_to_target = repo_and_path.split("//", 1) repo = repo.lstrip("@") # empty for same-repo `//pkg:needs_json` - if target in ("needs_json", "docs_sources"): + if target in ("needs_json", "needs_json_file", "docs_sources"): return ExternalNeedsSource( bazel_module=repo, path_to_target=path_to_target, @@ -159,12 +159,11 @@ def temp(self: NeedsList): def get_external_needs_source(external_needs_source: str) -> list[ExternalNeedsSource]: if external_needs_source: # Path taken for all invocations via `bazel` - external_needs = parse_external_needs_sources_from_DATA(external_needs_source) + return parse_external_needs_sources_from_DATA(external_needs_source) else: # This is the path taken for anything that doesn't # run via `bazel` e.g. esbonio or other direct executions - external_needs = parse_external_needs_sources_from_bazel_query() # pyright: ignore[reportAny] - return external_needs + return parse_external_needs_sources_from_bazel_query() # pyright: ignore[reportAny] def add_external_needs_json(e: ExternalNeedsSource, config: Config): @@ -182,9 +181,9 @@ def add_external_needs_json(e: ExternalNeedsSource, config: Config): needs_json_data = json.loads(Path(json_file).read_text(encoding="utf-8")) # pyright: ignore[reportAny] except FileNotFoundError: logger.error( - f"Could not find external needs JSON file at {json_file}. " - + "Something went terribly wrong. " - + "Try running `bazel clean --async && rm -rf _build`." + "Could not find external needs JSON file at %s from target %s.", + json_file, + e.target, ) # Attempt to continue, exit code will be non-zero after a logged error anyway. return @@ -227,6 +226,7 @@ def add_external_docs_sources(e: ExternalNeedsSource, config: Config): def connect_external_needs(app: Sphinx, config: Config): extend_needs_json_exporter(config, ["project_url"]) + # Local external needs from DATA (e.g. :needs_json or :docs_sources) external_needs = get_external_needs_source(app.config.external_needs_source) # this sets the default value - required for the needs-config-writer @@ -236,9 +236,41 @@ def connect_external_needs(app: Sphinx, config: Config): for e in external_needs: if e.target == "needs_json": add_external_needs_json(e, app.config) + elif e.target == "needs_json_file": + _add_needs_json_file(e, app.config) elif e.target == "docs_sources": add_external_docs_sources(e, app.config) else: raise ValueError( f"Internal Error. Unknown external needs target: {e.target}" ) + + +def _add_needs_json_file(ext_needs: ExternalNeedsSource, config: Config) -> None: + """Resolve a needs_json_file target from runfiles and register it.""" + json_file_raw = ( + Path(_runfiles_module_dir(ext_needs)) / ext_needs.path_to_target / "needs.json" + ) + r = get_runfiles_dir() + json_file = r / json_file_raw + logger.debug(f"External needs_json_file: {json_file}") + try: + needs_json_data = json.loads( + Path(json_file).read_text(encoding="utf-8") # pyright: ignore[reportAny] + ) + except FileNotFoundError: + logger.error( + "Could not find external needs JSON file at %s from target %s.", + json_file, + ext_needs.target, + ) + return + except json.JSONDecodeError as exc: + logger.error(f"Failed to parse external needs JSON file {json_file}: {exc}") + return + config.needs_external_needs.append( + { # pyright: ignore[reportUnknownMemberType] + "base_url": needs_json_data.get("project_url", "") + "/main", + "json_path": json_file, + } + ) diff --git a/src/extensions/score_metamodel/tests/test_external_needs.py b/src/extensions/score_metamodel/tests/test_external_needs.py index a73ff601f..4c834590c 100644 --- a/src/extensions/score_metamodel/tests/test_external_needs.py +++ b/src/extensions/score_metamodel/tests/test_external_needs.py @@ -24,6 +24,7 @@ import score_metamodel.external_needs as ext_needs from score_metamodel.external_needs import ( ExternalNeedsSource, + _add_needs_json_file, add_external_docs_sources, add_external_needs_json, get_external_needs_source, @@ -202,6 +203,41 @@ def test_add_external_needs_json_appends_entry_local( assert Path(entry["json_path"]) == json_path +def test_add_needs_json_file_appends_entry( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """_add_needs_json_file should load from a :needs_json_file target.""" + # Arrange: create the needs.json at the runfiles path + rel_json = Path("ext_mod+/needs.json") + json_path = tmp_path / rel_json + json_path.parent.mkdir(parents=True, exist_ok=True) + json_path.write_text( + json.dumps({"project_url": "https://example.test/json-file"}), encoding="utf-8" + ) + + runfiles_dir = tmp_path + config = Config() + config.needs_external_needs = [] + + monkeypatch.setattr(ext_needs, "get_runfiles_dir", lambda: runfiles_dir) + + # Act + e = ExternalNeedsSource( + bazel_module="ext_mod", + target="needs_json_file", + path_to_target="", + is_local=False, + ) + _add_needs_json_file(e, config) + + # Assert + assert config.needs_external_needs is not None + assert len(config.needs_external_needs) == 1 + entry = config.needs_external_needs[0] + assert entry["base_url"] == "https://example.test/json-file/main" + assert Path(entry["json_path"]) == json_path + + def test_add_external_needs_json_missing_file_keeps_list_empty( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: diff --git a/src/incremental.py b/src/incremental.py index e52937b03..d1c6bc854 100644 --- a/src/incremental.py +++ b/src/incremental.py @@ -13,6 +13,7 @@ import argparse import hashlib +import json import logging import os import shutil @@ -42,6 +43,17 @@ def get_env(name: str) -> str: return val +def _merged_external_needs() -> str: + """Combine DATA and EXTERNAL_NEEDS_FILES into one JSON label list. + + Both env vars hold JSON lists of Bazel labels; the extension parses the + resulting `external_needs_source` define uniformly. + """ + data = json.loads(get_env("DATA") or "[]") + external = json.loads(os.environ.get("EXTERNAL_NEEDS_FILES", "[]") or "[]") + return json.dumps(data + external) + + def _compute_hash(files: list[Path]) -> str: h = hashlib.sha256() for f in sorted(files, key=str): @@ -142,7 +154,11 @@ def _mounted_watch_dirs( "-T", # show details in case of errors in extensions "--jobs", "auto", - f"--define=external_needs_source={get_env('DATA')}", + # Merge DATA (:needs_json / :docs_sources) with EXTERNAL_NEEDS_FILES + # (:needs_json_file) into a single define. The sphinx_docs rule cannot + # receive per-target env vars, so --define is the only channel that + # works for both the py_binary and the needs_json target. + f"--define=external_needs_source={_merged_external_needs()}", f"--define=testcase_source_dirs={os.environ.get('TEST_SOURCES', '[]')}", # Path to the Bazel-emitted mounts manifest (empty when no mounts are # configured); consumed by the score_mounts extension. diff --git a/src/tests/docs_bzl/scenarios/external_needs/consumer/BUILD b/src/tests/docs_bzl/scenarios/external_needs/consumer/BUILD index a9bcf2ee6..c6e3ad488 100644 --- a/src/tests/docs_bzl/scenarios/external_needs/consumer/BUILD +++ b/src/tests/docs_bzl/scenarios/external_needs/consumer/BUILD @@ -18,6 +18,6 @@ load("//:docs.bzl", "docs") # avoiding the workspace-wide bazel-testlogs scan during the html build. docs( source_dir = "docs", - data = ["//src/tests/docs_bzl/scenarios/external_needs/producer:needs_json"], + external_needs = ["//src/tests/docs_bzl/scenarios/external_needs/producer:needs_json"], test_sources = ["src/tests/docs_bzl/scenarios/external_needs/consumer"], )