feat: record the git revision of each cfg extension in the reports

doc/design_doc.md

@@ -149,6 +149,27 @@ Within any of these files, `{file_dir}` names the directory that holds that file
 `{self_dir}`, by contrast, is the directory of the configuration file that dvsim loaded, at the top of the import tree.
 `{file_dir}` is replaced as each file is loaded, so it can only be used in Hjson files, not in options given on the command line.
 
+A configuration can also be extended from outside the project tree with `--cfg-extend NAME=PATH`, for example to add tests that need a licensed verification IP that the project itself can't hold:
+
+```hjson
+// ext.hjson, merged with: dvsim top_sim_cfgs.hjson --cfg-extend foo=/path/to/ext.hjson ...
+{
+  import_cfgs: ["{file_dir}/vip_tests.hjson"]
+  build_modes: [{name: "vip", build_opts: ["-f {file_dir}/vip.f"]}]
+  overrides: [{name: "foo_setting", value: "with_vip"}]
+}
+```
+
+The Hjson file at `PATH`, with its imports, is merged into the configuration whose `name` is `NAME` once that configuration has loaded all of its own files; this works whether that configuration is run on its own or is a child of a primary configuration.
+`NAME` is matched against the name that the configuration's own files give it, before any `overrides`.
+The usual merge rules apply: the extension's list entries follow the configuration's own, and a conflicting scalar is an error, so an extension changes a scalar with `overrides` (but not one that the configuration already overrides).
+In an extension, `{self_dir}` is still the directory of the configuration file that dvsim loaded, so the extension names its own files with `{file_dir}`.
+Several extensions apply in command-line order.
+An extension whose `NAME` matches no loaded configuration is an error, as is naming one extension twice.
+The sim reports and JSON results list the extensions that each configuration merged, with the git revision of the checkout that holds each extension file: its `origin` (without credentials), its commit, and whether it was dirty, which includes the extension file itself not being tracked.
+Files that an extension imports from elsewhere are not recorded.
+The JSON results of a configuration that merged no extension have no `extensions` key, as before extensions were recorded.
+
 ### Parser stage
 
 The **parser stage** parses all Hjson files in the tree and constructs a **flow manager object**.

src/dvsim/cli/run.py

@@ -37,6 +37,7 @@
     load_config_file,
     proj_root_from_config,
 )
+from dvsim.flow.extension import CfgExtension
 from dvsim.flow.factory import make_cfg
 from dvsim.fusesoc import CONFIG_SECTION as FUSESOC_CONFIG_SECTION
 from dvsim.fusesoc import resolve_options as resolve_fusesoc_options
@@ -185,6 +186,14 @@ def parse_resource(s: str) -> tuple[str, int | None]:
         raise argparse.ArgumentTypeError(msg) from e
 
 
+def parse_cfg_extension(s: str) -> CfgExtension:
+    """Parse a cfg extension (NAME=PATH) from the argparse CLI."""
+    try:
+        return CfgExtension.parse(s)
+    except ValueError as e:
+        raise argparse.ArgumentTypeError(str(e)) from e
+
+
 def resolve_branch(branch, cwd=None):
     """Choose a branch name for output files.
 
@@ -453,6 +462,21 @@ def parse_args(argv: list[str] | None = None):
         ),
     )
 
+    whatg.add_argument(
+        "--cfg-extend",
+        action="append",
+        default=[],
+        type=parse_cfg_extension,
+        metavar="NAME=PATH",
+        help=(
+            "Merge the hjson file PATH (and its imports) into the cfg whose "
+            "'name' is NAME, after the cfg's own files. NAME may be a child of "
+            "a primary config. Lists are appended to; scalars are changed "
+            "with 'overrides'. PATH names its own files with {file_dir}. "
+            "Repeatable; applied in order. It is an error if NAME matches no cfg."
+        ),
+    )
+
     disg = parser.add_argument_group("Dispatch options")
 
     disg.add_argument(

src/dvsim/flow/base.py

@@ -17,11 +17,13 @@
 import hjson
 
 import dvsim.instrumentation.runtime as instrumentation
+from dvsim.flow.extension import CfgExtension, load_cfg_extension
 from dvsim.flow.hjson import set_target_attribute
 from dvsim.fusesoc import rewrite_fusesoc_opts
 from dvsim.job.data import CompletedJobStatus, JobSpec, WorkspaceConfig
 from dvsim.job.status import JobStatus
 from dvsim.logging import log
+from dvsim.report.data import CfgExtensionMeta
 from dvsim.scheduler.resources import UnknownResourcePolicy
 from dvsim.scheduler.runner import (
     build_default_scheduler_backend,
@@ -33,7 +35,7 @@
     rm_path,
     subst_wildcards,
 )
-from dvsim.utils.git import git_commit_hash, git_is_dirty
+from dvsim.utils.git import git_commit_hash, git_is_dirty, git_revision
 
 if TYPE_CHECKING:
     from dvsim.job.deploy import Deploy
@@ -147,7 +149,15 @@ def __init__(self, flow_cfg_file, hjson_data, args, mk_config) -> None:
         # configuration".
         self.is_primary_cfg = "use_cfgs" in hjson_data
 
+        # The cfg extensions (--cfg-extend) merged into this cfg, in order, as
+        # the reports describe them, and the name that they were matched against.
+        self.cfg_extensions: dict[CfgExtension, CfgExtensionMeta] = {}
+        self.cfg_extend_name: str | None = None
+
         if not self.is_primary_cfg:
+            # Extensions merge after everything the cfg loads itself, and
+            # before overrides and wildcards are processed.
+            self._merge_cfg_extensions()
             self.cfgs.append(self)
         else:
             for entry in self.use_cfgs:
@@ -205,6 +215,41 @@ def _merge_hjson(self, hjson_data: Mapping) -> None:
         for key, value in hjson_data.items():
             set_target_attribute(self.flow_cfg_file, self.__dict__, key, value)
 
+    def _merge_cfg_extensions(self) -> None:
+        """Merge in the cfg extensions (--cfg-extend) that name this cfg.
+
+        They apply in command-line order. Their list values follow the cfg's
+        own, and their overrides join the cfg's own.
+        """
+        # The name as the cfg's own files give it. Overrides could still change
+        # it, but those are processed after the extensions merge.
+        name = subst_wildcards(self.name, self.__dict__, ignore_error=True)
+        self.cfg_extend_name = name
+
+        # What load_hjson needs to resolve the names of imported files. As in
+        # the cfg's own files, {self_dir} is the cfg's directory: an extension
+        # names its own files with {file_dir}.
+        initial_values = {"proj_root": self.proj_root, "self_dir": self.self_dir}
+        if self.tool:
+            initial_values["tool"] = self.tool
+        # Only a tool given on the command line can't be changed by the files.
+        arg_keys = {"tool"} if self.args.tool is not None else set()
+
+        for ext in self.args.cfg_extend:
+            if ext.cfg_name != name:
+                continue
+            log.info("[cfg_extend]: %s: %s", name, ext.path)
+            try:
+                ext_data = load_cfg_extension(ext, initial_values, arg_keys)
+                for key, value in ext_data.items():
+                    set_target_attribute(str(ext.path), self.__dict__, key, value)
+            except RuntimeError as err:
+                log.error("Cannot extend cfg %r: %s", name, err)
+                sys.exit(1)
+            self.cfg_extensions[ext] = CfgExtensionMeta(
+                path=str(ext.path), revision=git_revision(ext.path)
+            )
+
     def _expand(self) -> None:
         """Expand wildcards after merging hjson.
 

src/dvsim/flow/extension.py

@@ -0,0 +1,113 @@
+# Copyright lowRISC contributors (OpenTitan project).
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Cfg extensions: hjson files merged into a named flow cfg (--cfg-extend).
+
+See the "DUT configuration" section of doc/design_doc.md.
+"""
+
+import os
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from pathlib import Path
+
+from dvsim.flow.hjson import load_hjson
+
+__all__ = (
+    "CfgExtension",
+    "check_cfg_extensions_used",
+    "load_cfg_extension",
+)
+
+
+@dataclass(frozen=True)
+class CfgExtension:
+    """An extension file, and the name of the cfg it extends."""
+
+    cfg_name: str
+    path: Path
+
+    @classmethod
+    def parse(cls, value: str) -> "CfgExtension":
+        """Parse NAME=PATH, as given to --cfg-extend.
+
+        A relative PATH is taken from the current directory. Symlinks are
+        kept, as they are in {file_dir}.
+        """
+        name, sep, path = value.partition("=")
+        name, path = name.strip(), path.strip()
+        if not sep or not name or not path:
+            msg = f"Invalid cfg extension {value!r}: expected NAME=PATH."
+            raise ValueError(msg)
+        return cls(name, Path(os.path.normpath(Path(path).absolute())))
+
+
+def load_cfg_extension(
+    ext: CfgExtension, initial_values: Mapping[str, object], arg_keys: Iterable[str]
+) -> dict:
+    """Load an extension file and its imports.
+
+    initial_values holds what is needed to resolve the names of imported files,
+    and arg_keys those of its keys that came from the command line (as for
+    load_hjson). The returned data holds only what the extension sets.
+
+    Raises a RuntimeError if the file can't be loaded or isn't allowed.
+    """
+    path = str(ext.path)
+    if not ext.path.is_file():
+        msg = f"{path!r}: Not a file."
+        raise RuntimeError(msg)
+
+    try:
+        data = load_hjson(path, initial_values, set(arg_keys))
+    except ValueError as err:
+        # An import_cfgs entry with a wildcard that can't be resolved.
+        msg = f"{path!r}: Can't resolve the name of an imported file: {err}"
+        raise RuntimeError(msg) from err
+
+    if "use_cfgs" in data:
+        msg = f"{path!r}: An extension can't set use_cfgs."
+        raise RuntimeError(msg)
+
+    # The initial values are the cfg's own. If the extension set one of them,
+    # it set it to the same value (or load_hjson reported a conflict), so
+    # merging it would change nothing.
+    for key in initial_values:
+        data.pop(key, None)
+
+    return data
+
+
+def check_cfg_extensions_used(
+    extensions: Iterable[CfgExtension],
+    used: Iterable[CfgExtension],
+    cfg_names: Iterable[str],
+    primary_name: str | None,
+) -> list[str]:
+    """Check that every extension was used by a loaded cfg.
+
+    used are the extensions that some cfg matched. cfg_names are the names that
+    the cfgs that can be extended had when they matched them (before any
+    overrides); a primary cfg can't be extended, so its name is passed
+    separately to give a clearer message. Returns a list of error messages
+    (empty if all is well).
+    """
+    used = set(used)
+    names = set(cfg_names)
+    errors = []
+    for ext in extensions:
+        if ext in used:
+            continue
+        if ext.cfg_name == primary_name:
+            errors.append(
+                f"Cfg extension {ext.path} names the primary cfg {ext.cfg_name!r}: "
+                "only a cfg that runs jobs (a child, or a cfg run on its own) can be extended."
+            )
+        else:
+            known = ", ".join(sorted(names)) or "none"
+            errors.append(
+                f"Cfg extension {ext.path} names {ext.cfg_name!r}, but no loaded cfg has "
+                f"that name (loaded: {known})."
+            )
+    return errors

src/dvsim/flow/factory.py

@@ -7,6 +7,7 @@
 
 from dvsim.flow.base import FlowCfg
 from dvsim.flow.cdc import CdcCfg
+from dvsim.flow.extension import check_cfg_extensions_used
 from dvsim.flow.formal import FormalCfg
 from dvsim.flow.hjson import load_hjson
 from dvsim.flow.lint import LintCfg
@@ -113,6 +114,13 @@ def make_cfg(path, args, proj_root) -> FlowCfg:
     of the project.
 
     """
+    # An extension named twice would be merged twice.
+    repeated = [ext for ext in dict.fromkeys(args.cfg_extend) if args.cfg_extend.count(ext) > 1]
+    for ext in repeated:
+        log.error("Cfg extension %s=%s is given more than once.", ext.cfg_name, ext.path)
+    if repeated:
+        sys.exit(1)
+
     initial_values = {
         "proj_root": proj_root,
         "self_dir": pathlib.Path(path).parent,
@@ -131,4 +139,19 @@ def factory(child_path):
         child_ivs["flow"] = hjson_data["flow"]
         return _make_child_cfg(child_path, args, child_ivs)
 
-    return cls(path, hjson_data, args, factory)
+    cfg = cls(path, hjson_data, args, factory)
+
+    # Every extension must have found its cfg. A name that matches nothing is
+    # a typo or a cfg that has been renamed, and would otherwise pass silently.
+    errors = check_cfg_extensions_used(
+        args.cfg_extend,
+        [ext for child in cfg.cfgs for ext in child.cfg_extensions],
+        [child.cfg_extend_name for child in cfg.cfgs],
+        cfg.name if cfg.is_primary_cfg else None,
+    )
+    for error in errors:
+        log.error("%s", error)
+    if errors:
+        sys.exit(1)
+
+    return cfg

src/dvsim/flow/hjson.py

@@ -17,7 +17,7 @@
 _FILE_DIR_WILDCARD = "{file_dir}"
 
 
-def load_hjson(path, initial_values):
+def load_hjson(path, initial_values, arg_keys=None):
     """Load an hjson file and any includes.
 
     Combines them all into a single dictionary, which is then returned. This
@@ -29,6 +29,10 @@ def load_hjson(path, initial_values):
     is not modified). It needs to contain values for anything needed to resolve
     include files (typically, this is 'proj_root' and 'tool' (if set)).
 
+    arg_keys are the keys of initial_values that came from the command line,
+    which the files can't change. By default, that is any of them that the
+    command line can set.
+
     """
     worklist = [path]
     seen = {path}
@@ -38,7 +42,8 @@ def load_hjson(path, initial_values):
     # Figure out a list of fields that had a value from the command line. These
     # should have been passed in as part of initial_values and we need to know
     # that we can safely ignore updates.
-    arg_keys = _CMDLINE_FIELDS & initial_values.keys()
+    if arg_keys is None:
+        arg_keys = _CMDLINE_FIELDS & initial_values.keys()
 
     while worklist:
         next_path = worklist.pop()

src/dvsim/report/data.py

@@ -4,9 +4,13 @@
 
 """Report data models."""
 
+from pathlib import Path
+
 from pydantic import BaseModel, ConfigDict
 
 __all__ = (
+    "CfgExtensionMeta",
+    "GitRevisionMeta",
     "IPMeta",
     "ToolMeta",
 )
@@ -50,3 +54,35 @@ class ToolMeta(BaseModel):
     """Name of the tool."""
     version: str
     """Version of the tool."""
+
+
+class CfgExtensionMeta(BaseModel):
+    """A cfg extension (--cfg-extend) merged into a flow's cfg."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    path: str
+    """Path of the extension file."""
+    revision: "GitRevisionMeta | None" = None
+    """The git revision of the checkout that holds the extension file (not of
+    any files it imports from elsewhere), or None if it isn't in git."""
+
+    @property
+    def name(self) -> str:
+        """The name of the extension file."""
+        return Path(self.path).name
+
+
+class GitRevisionMeta(BaseModel):
+    """The git revision of a checkout, when the run started."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    origin: str | None
+    """URL of the ``origin`` remote (credentials stripped), or None."""
+    commit: str
+    """Git commit."""
+    commit_short: str
+    """Shortened git commit."""
+    dirty: bool
+    """Whether the checkout had uncommitted changes, or didn't track the file."""