From b2f094d2c7e39acb943cf3a0da277c4f72690531 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 07:01:50 +0000 Subject: [PATCH 1/4] Add set -e to devcontainer CI runCmd so pre-commit failures fail the job devcontainers/ci runs the multi-line runCmd script without shell errexit, so a failing `pre-commit run --all-files` didn't stop the script or fail the step -- later commands (pytest, docs build) kept running and the job could still pass, letting pre-commit findings slip through (see the follow-up lint/mypy fixes needed on PR #1290). set -e makes any failing command abort the script immediately. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_013hKpsXyi4hXAT3pDobq1qb --- .github/workflows/devcontainer.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index b62693fe..cab83c29 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -67,6 +67,8 @@ jobs: uses: devcontainers/ci@513af61f4de4f75d37e4438f184ba4358f0fc1ca # v0.3.1900000450 with: runCmd: | + set -e + echo "Installing test dependencies..." pip install -e .[development,docs,casts] From bbbf715790998a6222617ada5996778064b7fa3e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 10:02:42 +0000 Subject: [PATCH 2/4] Run pyright hook serially to fix nodeenv install race in CI Now that the devcontainer job's runCmd aborts on the first failing command, PR #1409's CI surfaced a pyright hook failure that was previously silent: pre-commit splits the pyright hook's large Python file list across multiple concurrent subprocess invocations, and each one that finds no cached node install tries to bootstrap its own via nodeenv into the same shared ~/.cache/pyright-python/nodeenv directory. The concurrent installs race -- one process sees "Environment already exists" from another's in-progress install and fails before it's usable. require_serial: true stops pre-commit from running more than one pyright process at a time, so only one process ever performs the nodeenv bootstrap. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_013hKpsXyi4hXAT3pDobq1qb --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 43c6291d..44195846 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -116,6 +116,7 @@ repos: description: Lint using pyright entry: pyright language: python + require_serial: true types: [file, python] - id: pyupgrade name: pyupgrade From 83136a9ef0177a2da73261bfed44f3b5f1e27a3b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 10:43:45 +0000 Subject: [PATCH 3/4] Fix the ruff/pyright pre-commit backlog that set -e now surfaces With the previous commit's set -e in place, pre-commit run --all-files correctly fails the DevContainer job instead of silently continuing -- which surfaced a long-standing backlog of ruff findings across the repo (implicit string concatenation, missing ClassVar annotations, mutable default arguments, naive datetime calls, redundant startswith calls, un-executable shebangs, nested with/if statements) that never actually failed CI before. Fixes each finding at the root: - pre-commit: run the pyright hook serially (require_serial) -- pyright's own nodeenv bootstrap raced across pre-commit's parallel batches and failed intermittently; excluded the vendored doc/_ext/sphinxcontrib_asciinema tree from ruff, matching the existing black/isort exclusions - pyproject.toml: pinned isort's py_version to 310 (matching requires-python) so its stdlib classification of tomllib agrees with ruff's, and added a matching ruff extend-exclude for the vendored asciinema extension - security/compliance_data.py: split the CRA Part II requirements block out into security/compliance_part_ii_data.py to stay under the 1000-line module limit after the ISC004 wrapping fixes - Everywhere else: ClassVar annotations for docutils/pygments class attributes (matching their typeshed stub types so pyright still passes), tz-aware datetime calls, a single tuple-based startswith, combined with/if statements, executable bits for shebang scripts, and a couple of true bugs (script/release.py's version parsing could raise on a non-clean __version__; script/create_sbom.py was logging on the root logger) Verified: ruff/black/isort/pyright/pylint/lint-imports all clean, the full unit test suite (715 tests) and the git-backed BDD feature suite pass (SVN scenarios fail here only because this sandbox has no svn binary; the devcontainer image does). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_013hKpsXyi4hXAT3pDobq1qb --- .pre-commit-config.yaml | 1 + doc/_ext/colordot.py | 4 +- doc/_ext/designguide.py | 7 +- doc/_ext/dfetch_style.py | 5 +- doc/_ext/scenario_directive.py | 37 ++-- doc/conf.py | 3 +- doc/generate-casts/interactive_helper.py | 0 doc/landing-page/conf.py | 1 - doc/static/uml/generate_diagram.py | 12 +- features/steps/add_steps.py | 8 +- features/steps/generic_steps.py | 24 +-- features/steps/git_steps.py | 4 +- features/steps/json_steps.py | 12 +- features/steps/manifest_steps.py | 3 +- features/steps/svn_steps.py | 12 +- pyproject.toml | 4 + script/build.py | 6 +- script/create_release_notes.py | 0 script/create_sbom.py | 5 +- script/dependabot_hook.py | 3 +- script/package.py | 2 +- script/release.py | 11 +- security/compliance.py | 26 ++- security/compliance_data.py | 246 +++++++++++------------ security/compliance_part_ii_data.py | 61 ++++++ security/tm_elements.py | 2 +- security/tm_render.py | 2 +- security/tm_supply_chain.py | 8 +- security/tm_usage.py | 8 +- stubs/py_serializable/__init__.pyi | 23 ++- tests/test_fuzzing.py | 28 +-- tests/test_tree_browser.py | 4 +- 32 files changed, 322 insertions(+), 250 deletions(-) mode change 100644 => 100755 doc/conf.py mode change 100644 => 100755 doc/generate-casts/interactive_helper.py mode change 100644 => 100755 doc/landing-page/conf.py mode change 100644 => 100755 script/build.py mode change 100644 => 100755 script/create_release_notes.py mode change 100644 => 100755 script/create_sbom.py mode change 100644 => 100755 script/package.py mode change 100644 => 100755 script/release.py create mode 100644 security/compliance_part_ii_data.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 44195846..c295ac79 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -110,6 +110,7 @@ repos: entry: ruff language: python args: [check] + exclude: ^doc/_ext/sphinxcontrib_asciinema types: [file, python] - id: pyright name: pyright diff --git a/doc/_ext/colordot.py b/doc/_ext/colordot.py index 9295ab79..5f4d60bf 100644 --- a/doc/_ext/colordot.py +++ b/doc/_ext/colordot.py @@ -20,7 +20,7 @@ import html import re -from typing import Any, List +from typing import Any from docutils import nodes from docutils.nodes import Node, system_message @@ -99,7 +99,7 @@ def _replace_emoji_for_latex( if parent is None: continue idx = parent.children.index(text_node) - new_nodes: List[Node] = [] + new_nodes: list[Node] = [] last = 0 for m in _EMOJI_RE.finditer(text): if m.start() > last: diff --git a/doc/_ext/designguide.py b/doc/_ext/designguide.py index 8014ac68..d4f4a166 100644 --- a/doc/_ext/designguide.py +++ b/doc/_ext/designguide.py @@ -25,7 +25,8 @@ """ import html -from typing import Any +from collections.abc import Callable +from typing import Any, ClassVar from docutils import nodes from docutils.nodes import Node @@ -38,7 +39,7 @@ class SwatchDirective(Directive): required_arguments = 1 optional_arguments = 0 - option_spec = { + option_spec: ClassVar[dict[str, Callable[[str], Any]] | None] = { "token": directives.unchanged, "label": directives.unchanged, "usage": directives.unchanged, @@ -81,7 +82,7 @@ class PaletteDirective(Directive): required_arguments = 0 optional_arguments = 0 - option_spec = { + option_spec: ClassVar[dict[str, Callable[[str], Any]] | None] = { "columns": directives.positive_int, } has_content = True diff --git a/doc/_ext/dfetch_style.py b/doc/_ext/dfetch_style.py index 7019d558..deb02a3a 100644 --- a/doc/_ext/dfetch_style.py +++ b/doc/_ext/dfetch_style.py @@ -2,7 +2,8 @@ import sys import types -from typing import MutableMapping, cast +from collections.abc import Mapping, MutableMapping +from typing import Any, ClassVar, cast import pygments.styles from pygments.style import Style @@ -24,7 +25,7 @@ class DfetchStyle(Style): # pylint: disable=too-few-public-methods background_color = "#fef8f0" # --bg-tint default_style = "" - styles = { + styles: ClassVar[Mapping[Any, str]] = { Token: "#1c1917", # --text Comment: "italic #78716c", # --text-muted Comment.Preproc: "noitalic #a0510a", # --primary-dark diff --git a/doc/_ext/scenario_directive.py b/doc/_ext/scenario_directive.py index deffeb5d..87d92999 100644 --- a/doc/_ext/scenario_directive.py +++ b/doc/_ext/scenario_directive.py @@ -39,7 +39,8 @@ import os import re import textwrap -from typing import Dict, FrozenSet, List, Tuple +from collections.abc import Callable +from typing import Any, ClassVar from docutils import nodes from docutils.parsers.rst import Directive, directives @@ -83,9 +84,9 @@ class ScenarioIncludePlaceholder(nodes.General, nodes.Element): # --------------------------------------------------------------------------- -def _feature_tags(feature_path: str) -> List[str]: +def _feature_tags(feature_path: str) -> list[str]: """Return all behave tags declared before the ``Feature:`` line.""" - tags: List[str] = [] + tags: list[str] = [] with open(feature_path, encoding="utf-8") as fh: for line in fh: stripped = line.strip() @@ -96,7 +97,7 @@ def _feature_tags(feature_path: str) -> List[str]: return tags -def _group_tag(feature_path: str, non_group_tags: FrozenSet[str]) -> str: +def _group_tag(feature_path: str, non_group_tags: frozenset[str]) -> str: """Return the first tag not in *non_group_tags*, or ``'other'``.""" for tag in _feature_tags(feature_path): if tag not in non_group_tags: @@ -114,7 +115,7 @@ def _feature_title(feature_path: str) -> str: return os.path.basename(feature_path) -def _all_scenarios(feature_path: str) -> Tuple[Tuple[str, str], ...]: +def _all_scenarios(feature_path: str) -> tuple[tuple[str, str], ...]: """Return (header, title) pairs for all scenarios in the feature file.""" with open(feature_path, encoding="utf-8") as fh: return tuple( @@ -131,7 +132,7 @@ def _full_feature_content(feature_path: str) -> str: return fh.read() -def _selected_scenarios_content(feature_path: str, scenario_titles: List[str]) -> str: +def _selected_scenarios_content(feature_path: str, scenario_titles: list[str]) -> str: """Return content containing only the selected scenario blocks.""" with open(feature_path, encoding="utf-8") as fh: content = fh.read() @@ -173,7 +174,7 @@ class ScenarioIncludeDirective(Directive): required_arguments = 1 optional_arguments = 0 final_argument_whitespace = False - option_spec = { + option_spec: ClassVar[dict[str, Callable[[str], Any]] | None] = { "scenario": str, "inline": directives.flag, # keep inline even in PDF mode } @@ -192,7 +193,7 @@ def _feature_abs(self, feature_file: str) -> str: raise self.error(f"Feature file not found: {path}") return path - def _requested_scenarios(self, available: Tuple[Tuple[str, str], ...]) -> List[str]: + def _requested_scenarios(self, available: tuple[tuple[str, str], ...]) -> list[str]: return [ t.strip() for t in self.options.get("scenario", "").splitlines() @@ -203,7 +204,7 @@ def _requested_scenarios(self, available: Tuple[Tuple[str, str], ...]) -> List[s # Appendix entry registration (always runs, any builder) # ------------------------------------------------------------------ - def _entry_metadata(self, feature_abs: str) -> Tuple[str, str, str]: + def _entry_metadata(self, feature_abs: str) -> tuple[str, str, str]: """Return (label, group_tag, feature_title) for a feature file.""" env = self._env() non_group_tags = frozenset(getattr(env.config, "scenario_non_command_tags", [])) @@ -218,7 +219,7 @@ def _register_appendix_entry( self, feature_file: str, feature_abs: str, - scenario_titles: List[str], + scenario_titles: list[str], ) -> None: """Store entry in env.scenario_appendix_entries for any builder. @@ -254,7 +255,7 @@ def _register_appendix_entry( # Entry point # ------------------------------------------------------------------ - def run(self) -> List[nodes.Node]: + def run(self) -> list[nodes.Node]: feature_file = self.arguments[0].strip() feature_abs = self._feature_abs(feature_file) available = _all_scenarios(feature_abs) @@ -310,7 +311,7 @@ class ScenarioAppendixDirective(Directive): optional_arguments = 0 has_content = False - def run(self) -> List[nodes.Node]: + def run(self) -> list[nodes.Node]: env = self.state.document.settings.env # Record which document hosts the appendix so that ScenarioAppendixRef # nodes in other documents can be resolved with the correct refdocname. @@ -329,13 +330,13 @@ def run(self) -> List[nodes.Node]: # --------------------------------------------------------------------------- -def _build_appendix_nodes(entries: Dict) -> List[nodes.Node]: +def _build_appendix_nodes(entries: dict) -> list[nodes.Node]: """Build docutils section nodes for every collected appendix entry.""" - by_tag: Dict[str, List] = {} + by_tag: dict[str, list] = {} for entry in entries.values(): by_tag.setdefault(entry["group_tag"], []).append(entry) - result: List[nodes.Node] = [] + result: list[nodes.Node] = [] for tag in sorted(by_tag): tag_entries = sorted(by_tag[tag], key=lambda e: e["feature_title"]) label = f"appendix-{tag}" @@ -366,10 +367,10 @@ def _build_appendix_nodes(entries: Dict) -> List[nodes.Node]: def _render_scenario_inline( - scenario_titles: List[str], feature_abs: str -) -> List[nodes.Node]: + scenario_titles: list[str], feature_abs: str +) -> list[nodes.Node]: """Return docutils nodes for inline HTML rendering of *scenario_titles*.""" - result: List[nodes.Node] = [] + result: list[nodes.Node] = [] for title in scenario_titles: raw_content = _selected_scenarios_content(feature_abs, [title]) content = textwrap.dedent(raw_content).strip() diff --git a/doc/conf.py b/doc/conf.py old mode 100644 new mode 100755 index 04a4cfb3..d52a6195 --- a/doc/conf.py +++ b/doc/conf.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Documentation build configuration file. """ @@ -18,7 +17,7 @@ ext_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "_ext")) sys.path.insert(0, ext_path) -import dfetch_style # noqa: E402 +import dfetch_style dfetch_style.register() diff --git a/doc/generate-casts/interactive_helper.py b/doc/generate-casts/interactive_helper.py old mode 100644 new mode 100755 diff --git a/doc/landing-page/conf.py b/doc/landing-page/conf.py old mode 100644 new mode 100755 index e5762639..8c4730e9 --- a/doc/landing-page/conf.py +++ b/doc/landing-page/conf.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Documentation build configuration file. """ diff --git a/doc/static/uml/generate_diagram.py b/doc/static/uml/generate_diagram.py index 21b6beb7..216170b9 100644 --- a/doc/static/uml/generate_diagram.py +++ b/doc/static/uml/generate_diagram.py @@ -3,8 +3,8 @@ import os import pathlib import re +from collections.abc import Sequence from pathlib import Path -from typing import Sequence, Tuple import_regex = re.compile(r"(import|from) dfetch\.(?P[\.\w]+)") description_regex = re.compile(r"^\"{3}(?P.*)") @@ -17,7 +17,7 @@ class Relation: @dataclasses.dataclass class Module: - path: Tuple[str] + path: tuple[str] name: str description: str relations: list @@ -129,7 +129,7 @@ def generate_c3(relations, path: Sequence[str], blacklist=None): f'{indent}{indent}Component(comp{name}, "{name}", "python", "{description}")' ) - print("") + print() for name, module in modules.items(): if isinstance(module, dict): continue @@ -145,14 +145,14 @@ def generate_c3(relations, path: Sequence[str], blacklist=None): print( f'{indent}Container(cont{container}, "{container}", "python", "Something.")' ) - print("") + print() for relation in outside_in: print(relation) - print("") + print() for relation in inside_out: print(relation) - print("") + print() print(C3_END_TEMPLATE) diff --git a/features/steps/add_steps.py b/features/steps/add_steps.py index 4ca37d06..46db802c 100644 --- a/features/steps/add_steps.py +++ b/features/steps/add_steps.py @@ -48,9 +48,11 @@ def _auto_prompt(_prompt: str, **kwargs) -> str: # type: ignore[return] return prompt_answers.popleft() return str(kwargs.get("default", "")) - with patch("dfetch.commands.add.Prompt.ask", side_effect=_auto_prompt): - with patch("dfetch.commands.add.Confirm.ask", side_effect=_auto_confirm): - call_command(context, cmd) + with ( + patch("dfetch.commands.add.Prompt.ask", side_effect=_auto_prompt), + patch("dfetch.commands.add.Confirm.ask", side_effect=_auto_confirm), + ): + call_command(context, cmd) @when('I run "dfetch {add_args}" with inputs') diff --git a/features/steps/generic_steps.py b/features/steps/generic_steps.py index fac18aeb..44695892 100644 --- a/features/steps/generic_steps.py +++ b/features/steps/generic_steps.py @@ -9,9 +9,10 @@ import pathlib import re import shutil +from collections.abc import Iterable from contextlib import contextmanager from itertools import zip_longest -from typing import Iterable, List, Optional, Pattern, Tuple, Union +from re import Pattern from unittest.mock import patch from behave import given, then, when # pylint: disable=no-name-in-module @@ -57,18 +58,17 @@ def remote_server_path(context) -> str: return pathlib.Path(context.remotes_dir_path).as_uri() -def call_command(context: Context, args: list[str], path: Optional[str] = ".") -> None: +def call_command(context: Context, args: list[str], path: str | None = ".") -> None: before = context.console.export_text() DLogger.reset_projects() - with temporary_env("CI", "true"): - with in_directory(path or "."): - try: - run(args, context.console) - context.cmd_returncode = 0 - except DfetchFatalException: - context.cmd_returncode = 1 + with temporary_env("CI", "true"), in_directory(path or "."): + try: + run(args, context.console) + context.cmd_returncode = 0 + except DfetchFatalException: + context.cmd_returncode = 1 after = context.console.export_text() context.cmd_output = after[len(before) :].strip("\n") @@ -186,7 +186,7 @@ def list_dir(path): result = "" prev_node = [] - for node in list(sorted(nodes)) + [""]: + for node in sorted(nodes) + [""]: if prev_node: end = "" if "".join(node).startswith("".join(prev_node)): @@ -359,7 +359,7 @@ def step_impl(context, name): check_file_exists(name) -def check_json(path: Union[str, os.PathLike], content: str, context) -> None: +def check_json(path: str | os.PathLike, content: str, context) -> None: """Check a JSON file for exact equality (after normalising formatting).""" content = apply_archive_substitutions(content, context) with open(path, "r", encoding="UTF-8") as file_to_check: @@ -394,7 +394,7 @@ def step_impl(_, path, target): assert actual == target, f"Expected {path!r} to point to {target!r}, got {actual!r}" -def multisub(patterns: List[Tuple[Pattern[str], str]], text: str) -> str: +def multisub(patterns: list[tuple[Pattern[str], str]], text: str) -> str: """Apply a list of tuples that each contain a regex + replace string.""" for pattern, replace in patterns: text = pattern.sub(replace, text) diff --git a/features/steps/git_steps.py b/features/steps/git_steps.py index 745b48fb..c0ec9b8b 100644 --- a/features/steps/git_steps.py +++ b/features/steps/git_steps.py @@ -221,7 +221,7 @@ def step_impl(context, name, ending): create_repo() subprocess.check_call(["git", "config", "core.autocrlf", "false"]) pathlib.Path("README.md").write_bytes( - f"Generated file for {name}{terminator}".encode("utf-8") + f"Generated file for {name}{terminator}".encode() ) commit_all("Initial commit") tag("v1") @@ -239,7 +239,7 @@ def step_impl(context, name, ending, filename, gitattr): subprocess.check_call(["git", "config", "core.autocrlf", "false"]) pathlib.Path(".gitattributes").write_text(gitattr + "\n", encoding="utf-8") pathlib.Path(filename).write_bytes( - f"Generated file for {name}{terminator}".encode("utf-8") + f"Generated file for {name}{terminator}".encode() ) commit_all("Initial commit") tag("v1") diff --git a/features/steps/json_steps.py b/features/steps/json_steps.py index 6a19e7d3..c886383f 100644 --- a/features/steps/json_steps.py +++ b/features/steps/json_steps.py @@ -6,7 +6,6 @@ import json import os import re -from typing import Union from behave import then # pylint: disable=no-name-in-module @@ -59,16 +58,19 @@ def _try_match(exp_index: int, used: set) -> bool: return True exp_item = expected[exp_index] for i, act_item in enumerate(actual): - if i not in used and _json_subset_matches(exp_item, act_item): - if _try_match(exp_index + 1, used | {i}): - return True + if ( + i not in used + and _json_subset_matches(exp_item, act_item) + and _try_match(exp_index + 1, used | {i}) + ): + return True return False return _try_match(0, set()) return expected == actual -def check_json_subset(path: Union[str, os.PathLike], content: str, context) -> None: +def check_json_subset(path: str | os.PathLike, content: str, context) -> None: """Assert that a JSON file *contains* the given key-values (subset match). Dynamic placeholders (````, ````) in diff --git a/features/steps/manifest_steps.py b/features/steps/manifest_steps.py index 909c59d2..6554c080 100644 --- a/features/steps/manifest_steps.py +++ b/features/steps/manifest_steps.py @@ -5,7 +5,6 @@ import os import pathlib -from typing import Optional from behave import given, then, when # pylint: disable=no-name-in-module @@ -27,7 +26,7 @@ def apply_manifest_substitutions(context, contents: str) -> str: def generate_manifest( - context, name="dfetch.yaml", contents: Optional[str] = None, path=None + context, name="dfetch.yaml", contents: str | None = None, path=None ): contents = contents or context.text manifest = apply_manifest_substitutions(context, contents) diff --git a/features/steps/svn_steps.py b/features/steps/svn_steps.py index 53b98eb7..5154b105 100644 --- a/features/steps/svn_steps.py +++ b/features/steps/svn_steps.py @@ -175,9 +175,8 @@ def step_impl(context, name, ext_path, source_name): source_url = ( pathlib.Path(context.remotes_dir_path).joinpath(source_name, "trunk").as_uri() ) - with in_directory(name): - with in_directory("trunk"): - add_externals([{"url": source_url, "path": ext_path, "revision": ""}]) + with in_directory(name), in_directory("trunk"): + add_externals([{"url": source_url, "path": ext_path, "revision": ""}]) @given( @@ -185,9 +184,8 @@ def step_impl(context, name, ext_path, source_name): ) def step_impl(context, name, ext_path, source_name): source_url = pathlib.Path(context.remotes_dir_path).joinpath(source_name).as_uri() - with in_directory(name): - with in_directory("trunk"): - add_externals([{"url": source_url, "path": ext_path, "revision": ""}]) + with in_directory(name), in_directory("trunk"): + add_externals([{"url": source_url, "path": ext_path, "revision": ""}]) @given('a svn-server "{name}" with {ending} content') @@ -198,7 +196,7 @@ def step_impl(context, name, ending): create_stdlayout() with in_directory("trunk"): pathlib.Path("README.md").write_bytes( - f"Generated file for {name}{terminator}".encode("utf-8") + f"Generated file for {name}{terminator}".encode() ) subprocess.check_call(["svn", "update", "."]) subprocess.check_call(["svn", "add", "--force", "."]) diff --git a/pyproject.toml b/pyproject.toml index 2ec878c0..240880fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -129,6 +129,7 @@ target-version = ["py313"] [tool.isort] profile = "black" +py_version = "310" skip = [ "doc/_ext/sphinxcontrib_asciinema", ] @@ -198,6 +199,9 @@ fail_under = 72 skip = "*.cast,./venv,**/plantuml-c4/**,./example,.mypy_cache,./doc/_build/**,./doc/landing-page/_build/**,./doc/_ext/sphinxcontrib_asciinema/**,./build,*.patch,.git,**/generate-casts/demo-magic/**,./doc/openssl/**" ignore-words-list = "bund" +[tool.ruff] +extend-exclude = ["doc/_ext/sphinxcontrib_asciinema"] + [tool.ruff.lint.per-file-ignores] "features/steps/*" = ["F811"] diff --git a/script/build.py b/script/build.py old mode 100644 new mode 100755 index 104e6cb8..34907f34 --- a/script/build.py +++ b/script/build.py @@ -3,15 +3,13 @@ import subprocess # nosec import sys + import tomllib as toml -from typing import Union from dfetch import __version__ -def parse_option( - option_name: str, option_value: Union[bool, str, list, dict] -) -> list[str]: +def parse_option(option_name: str, option_value: bool | str | list | dict) -> list[str]: """ Convert a config value to Nuitka CLI arguments. diff --git a/script/create_release_notes.py b/script/create_release_notes.py old mode 100644 new mode 100755 diff --git a/script/create_sbom.py b/script/create_sbom.py old mode 100644 new mode 100755 index 841845a4..e43a87d1 --- a/script/create_sbom.py +++ b/script/create_sbom.py @@ -11,6 +11,7 @@ from pathlib import Path logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) PROJECT_DIR = Path(__file__).parent.parent.resolve() @@ -35,7 +36,7 @@ def temporary_venv(): """Create a temporary virtual environment and clean it up on exit.""" with tempfile.TemporaryDirectory(prefix="venv_sbom_") as tmpdir: venv_dir = Path(tmpdir) - logging.info(f"Creating temporary virtual environment at {venv_dir}") + logger.info(f"Creating temporary virtual environment at {venv_dir}") venv.create(venv_dir, with_pip=True, upgrade_deps=True) if sys.platform.startswith("win"): @@ -98,4 +99,4 @@ def parse_args() -> argparse.Namespace: [python, "-m", "cyclonedx_py", "environment", "-o", str(output_file)] ) -logging.info(f"SBOM generated at {output_file}") +logger.info(f"SBOM generated at {output_file}") diff --git a/script/dependabot_hook.py b/script/dependabot_hook.py index 0d38f88e..7a0af9cb 100755 --- a/script/dependabot_hook.py +++ b/script/dependabot_hook.py @@ -4,7 +4,6 @@ import re import sys from pathlib import Path -from typing import Optional # Config SBOM_FILE = "sbom.json" # path to your CycloneDX SBOM @@ -29,7 +28,7 @@ def get_new_version_from_pyproject(name: str) -> str: def replace_cyclonedx_version_if_outdated( new_version: str, -) -> Optional[str]: +) -> str | None: """Update the SBOM JSON file with the new version""" feature_file_path = Path(FEATURE_FILE) content = feature_file_path.read_text(encoding="UTF-8") diff --git a/script/package.py b/script/package.py old mode 100644 new mode 100755 index 126a2db1..5f96ea26 --- a/script/package.py +++ b/script/package.py @@ -4,10 +4,10 @@ import shutil import subprocess # nosec import sys -import tomllib as toml import xml.etree.ElementTree as ET # nosec (only used for XML generation, not parsing untrusted input) from pathlib import Path +import tomllib as toml from setuptools_scm import get_version # pylint: disable=import-error from dfetch import __version__ as __digit_only_version__ # Used inside the installers diff --git a/script/release.py b/script/release.py old mode 100644 new mode 100755 index 09e99ad9..90402903 --- a/script/release.py +++ b/script/release.py @@ -5,7 +5,7 @@ import glob import os import re -from datetime import datetime +from datetime import datetime, timezone from dfetch import __version__ @@ -45,7 +45,9 @@ def replace_pattern_in_files(file_path_pattern, search_pattern, replacement, fla replace_pattern_in_files( file_path_pattern=f"{base_dir}/CHANGELOG.rst", search_pattern=r"(Release \d+\.\d+\.\d+) \(unreleased\)", - replacement=r"\1 (released " + datetime.now().strftime("%Y-%m-%d") + ")", + replacement=r"\1 (released " + + datetime.now(tz=timezone.utc).strftime("%Y-%m-%d") + + ")", flags=re.DOTALL, ) @@ -69,7 +71,10 @@ def replace_pattern_in_files(file_path_pattern, search_pattern, replacement, fla flags=re.DOTALL, ) - major, minor, _ = map(int, __version__.split(".")) + version_match = re.match(r"(\d+)\.(\d+)\.(\d+)", __version__) + if not version_match: + raise ValueError(f"Could not parse version from {__version__!r}") + major, minor = int(version_match.group(1)), int(version_match.group(2)) replace_pattern_in_files( file_path_pattern=f"{base_dir}/doc/howto/contributing.rst", diff --git a/security/compliance.py b/security/compliance.py index 51eaeb57..66a6d5c6 100644 --- a/security/compliance.py +++ b/security/compliance.py @@ -20,17 +20,17 @@ import re import sys import uuid -from datetime import date +from datetime import datetime, timezone from typing import Any from security.compliance_data import ( ANNEX_V_MAP, CLASSIFICATION_DECISION, - PART_II_REQUIREMENTS, SO_IMPLEMENTATIONS, STANDARDS, TRACK_B_CONTROLS, ) +from security.compliance_part_ii_data import PART_II_REQUIREMENTS from security.compliance_types import SOImplementation from security.tm_controls_data import SC_CONTROLS, USAGE_CONTROLS, Control @@ -100,7 +100,7 @@ def _build_metadata(version: str) -> dict[str, Any]: """Return the OSCAL 1.2.2 metadata block with parties and roles.""" return { "title": "dfetch CRA Compliance Component Definition", - "last-modified": f"{date.today().isoformat()}T00:00:00Z", + "last-modified": f"{datetime.now(tz=timezone.utc).date().isoformat()}T00:00:00Z", "version": version, "oscal-version": "1.2.2", "document-ids": [ @@ -502,7 +502,7 @@ def _format_ref_as_rst(ref: str) -> str: if not ref or ref == "—": return "—" # Already RST markup - if ref.startswith(":doc:") or ref.startswith(":ref:"): + if ref.startswith((":doc:", ":ref:")): return ref # Handle parenthetical suffix like "path (note about it)" paren_suffix = "" @@ -664,8 +664,10 @@ def _gap_entries() -> list[tuple[str, str]]: """Return (title, body) pairs for the gap analysis section.""" return [ ( - ":ref:`C-043 ` — Release-gate CVE check" - " (ECR-a, SO.VulnerabilityManagementProcess → GEC-1)", + ( + ":ref:`C-043 ` — Release-gate CVE check" + " (ECR-a, SO.VulnerabilityManagementProcess → GEC-1)" + ), ( "dfetch's CI detects vulnerabilities at commit time " "(:ref:`C-015 `, :ref:`C-016 `, :ref:`C-017 `). " @@ -675,8 +677,10 @@ def _gap_entries() -> list[tuple[str, str]]: ), ), ( - ":ref:`C-044 ` — Data minimisation policy" - " (ECR-g, SO.DataMinimization → DTM-1)", + ( + ":ref:`C-044 ` — Data minimisation policy" + " (ECR-g, SO.DataMinimization → DTM-1)" + ), ( "dfetch processes dependency metadata only. The ``.dfetch_data.yaml`` " "file stores: ``remote_url`` (credentials stripped by " @@ -688,8 +692,10 @@ def _gap_entries() -> list[tuple[str, str]]: ), ), ( - ":ref:`C-046 ` — Exploit mitigation inventory" - " (ECR-k, SO.ReduceImpactOfIncident → GEC-11)", + ( + ":ref:`C-046 ` — Exploit mitigation inventory" + " (ECR-k, SO.ReduceImpactOfIncident → GEC-11)" + ), ( "prEN 40000-1-4 ECR-k requires documenting applicable exploit " "mitigation techniques. For dfetch (pure Python):\n\n" diff --git a/security/compliance_data.py b/security/compliance_data.py index e89bb72d..8871a278 100644 --- a/security/compliance_data.py +++ b/security/compliance_data.py @@ -4,13 +4,12 @@ Kept in a separate module to stay within the 1000-line limit per file. """ -from security.compliance_types import ( # noqa: F401 # re-exported +from security.compliance_types import ( # re-exported ApplicableStandard, - PartIIRequirement, SODocumentation, SOImplementation, ) -from security.tm_controls_data import Control # noqa: F401 # re-exported +from security.tm_controls_data import Control # re-exported # ── Classification decision ─────────────────────────────────────────────────── @@ -143,46 +142,68 @@ ANNEX_V_MAP: list[tuple[str, str]] = [ ( - "**1. General description** — intended purpose, product name and version, " - "manufacturer address", - ":doc:`security` § *Product and manufacturer identification*; " - ":doc:`../reference/manifest` (manifest schema and version field)", + ( + "**1. General description** — intended purpose, product name and version, " + "manufacturer address" + ), + ( + ":doc:`security` § *Product and manufacturer identification*; " + ":doc:`../reference/manifest` (manifest schema and version field)" + ), ), ( - "**2. Design and development** — software architecture; how components " - "build on or feed into each other", - ":doc:`../explanation/architecture` (layer diagram and module overview); " - ":doc:`security_pipeline` § *Threat model pipeline* (security-relevant " - "component relationships)", + ( + "**2. Design and development** — software architecture; how components " + "build on or feed into each other" + ), + ( + ":doc:`../explanation/architecture` (layer diagram and module overview); " + ":doc:`security_pipeline` § *Threat model pipeline* (security-relevant " + "component relationships)" + ), ), ( - "**3. Production and monitoring** — build pipeline, dependency management, " - "CI/CD monitoring", - ":doc:`security_pipeline` § *Compliance pipeline* and *Release attestations*; " - "CI workflows in " - "`\\.github/workflows/ `_", + ( + "**3. Production and monitoring** — build pipeline, dependency management, " + "CI/CD monitoring" + ), + ( + ":doc:`security_pipeline` § *Compliance pipeline* and *Release attestations*; " + "CI workflows in " + "`\\.github/workflows/ `_" + ), ), ( - "**4. Cybersecurity risk assessment** (Article 13(2)) — asset identification, " - "threat analysis, risk treatment", - ":doc:`threat_model_supply_chain` (pre-install lifecycle); " - ":doc:`threat_model_usage` (runtime invocation); " - "see also :doc:`security` § *Risk Rating Methodology*", + ( + "**4. Cybersecurity risk assessment** (Article 13(2)) — asset identification, " + "threat analysis, risk treatment" + ), + ( + ":doc:`threat_model_supply_chain` (pre-install lifecycle); " + ":doc:`threat_model_usage` (runtime invocation); " + "see also :doc:`security` § *Risk Rating Methodology*" + ), ), ( - "**5. Implemented security solutions and applied standards** — " - "list of harmonised standards applied; where not applied, description of how " - "each Annex I requirement is met", - "This page (§§ *Applicable Standards*, *Part I*, *Part II*); " - ":doc:`control_register` (all 46 controls with references); " - "OSCAL Component Definition " - "`security/dfetch.component-definition.json " - "`_", + ( + "**5. Implemented security solutions and applied standards** — " + "list of harmonised standards applied; where not applied, description of how " + "each Annex I requirement is met" + ), + ( + "This page (§§ *Applicable Standards*, *Part I*, *Part II*); " + ":doc:`control_register` (all 46 controls with references); " + "OSCAL Component Definition " + "`security/dfetch.component-definition.json " + "`_" + ), ), ( "**6. EU Declaration of Conformity** (Annex IV)", - "Not required. dfetch is outside mandatory CRA scope (see " - "*Classification Decision* above). No CE marking is affixed.", + ( + "Not required. dfetch is outside mandatory CRA scope (see " + "*Classification Decision* above). No CE marking is affixed." + ), ), ] @@ -226,8 +247,10 @@ "AUM-5 (no password or authentication mechanism)", ], gaps=[ - "Integrity hash verification (:ref:`C-005 `) is opt-in; manifest entries" - " without an ``integrity`` field are fetched without hash verification by default" + ( + "Integrity hash verification (:ref:`C-005 `) is opt-in; manifest entries" + " without an ``integrity`` field are fetched without hash verification by default" + ) ], status="partially-implemented", doc=SODocumentation( @@ -355,15 +378,19 @@ ecr_id="ecr-d", controls=["C-006", "C-036"], not_applicable=[ - "ACM-2, AUM-2, AUM-3, AUM-4, AUM-6 " - "(dfetch has no user-facing authentication or access control)" + ( + "ACM-2, AUM-2, AUM-3, AUM-4, AUM-6 " + "(dfetch has no user-facing authentication or access control)" + ) ], gaps=[ - "dfetch has no native authentication or authorisation layer; access control is " - "fully delegated to the underlying VCS server and host OS. C-006 prevents " - "interactive credential prompts, and C-036 strips credentials from persisted " - "metadata — both are confidentiality controls, not access-control mechanisms " - "in the authentication/authorisation sense" + ( + "dfetch has no native authentication or authorisation layer; access control is " + "fully delegated to the underlying VCS server and host OS. C-006 prevents " + "interactive credential prompts, and C-036 strips credentials from persisted " + "metadata — both are confidentiality controls, not access-control mechanisms " + "in the authentication/authorisation sense" + ) ], status="partially-implemented", doc=SODocumentation( @@ -449,9 +476,11 @@ ecr_id="ecr-e", controls=["C-045"], gaps=[ - "C-045 warns on plaintext-scheme URLs but does not refuse to proceed; " - "TLS/SSH confidentiality is provided by the underlying VCS client, not " - "enforced by dfetch itself" + ( + "C-045 warns on plaintext-scheme URLs but does not refuse to proceed; " + "TLS/SSH confidentiality is provided by the underlying VCS client, not " + "enforced by dfetch itself" + ) ], status="partially-implemented", doc=SODocumentation( @@ -479,10 +508,12 @@ ecr_id="ecr-e", controls=["C-045"], gaps=[ - "Server authentication (TLS certificate verification, SSH host-key checking) " - "is delegated to the OS trust store and VCS client; dfetch does not " - "independently authenticate remote endpoints and cannot enforce authenticated " - "channels when C-045's warning is overridden by the user" + ( + "Server authentication (TLS certificate verification, SSH host-key checking) " + "is delegated to the OS trust store and VCS client; dfetch does not " + "independently authenticate remote endpoints and cannot enforce authenticated " + "channels when C-045's warning is overridden by the user" + ) ], status="partially-implemented", doc=SODocumentation( @@ -564,9 +595,11 @@ ecr_id="ecr-f", controls=["C-005"], gaps=[ - "C-005 provides end-to-end hash verification for archive sources only (opt-in); " - "git and svn sources rely solely on VCS object integrity (SHA-1/SHA-256 object " - "model) and TLS/SSH channel integrity — no dfetch-level hash verification" + ( + "C-005 provides end-to-end hash verification for archive sources only (opt-in); " + "git and svn sources rely solely on VCS object integrity (SHA-1/SHA-256 object " + "model) and TLS/SSH channel integrity — no dfetch-level hash verification" + ) ], status="partially-implemented", doc=SODocumentation( @@ -663,13 +696,17 @@ ecr_id="ecr-i", controls=["C-001", "C-007"], not_applicable=[ - "TCM-1 (dfetch makes targeted VCS fetch requests; " - "no ambient outbound traffic to throttle)" + ( + "TCM-1 (dfetch makes targeted VCS fetch requests; " + "no ambient outbound traffic to throttle)" + ) ], gaps=[ - "Archive HTTP operations time out at 15 s (reachability) and 60 s (download) " - "via ``archive.py``; git and svn subprocess calls have no timeout and can " - "stall indefinitely" + ( + "Archive HTTP operations time out at 15 s (reachability) and 60 s (download) " + "via ``archive.py``; git and svn subprocess calls have no timeout and can " + "stall indefinitely" + ) ], status="partially-implemented", doc=SODocumentation( @@ -724,13 +761,17 @@ ecr_id="ecr-j", controls=["C-001", "C-003", "C-004", "C-007", "C-008"], not_applicable=[ - "GEC-2-j, GEC-3-j, GEC-4-j, GEC-5-j, GEC-7-j " - "(dfetch exposes no network services)" + ( + "GEC-2-j, GEC-3-j, GEC-4-j, GEC-5-j, GEC-7-j " + "(dfetch exposes no network services)" + ) ], gaps=[ - "No domain or URL-scheme allowlist constrains which remote URLs the manifest " - "may reference; git and svn subprocess calls have no timeout (archive HTTP " - "operations time out at 15 s / 60 s)" + ( + "No domain or URL-scheme allowlist constrains which remote URLs the manifest " + "may reference; git and svn subprocess calls have no timeout (archive HTTP " + "operations time out at 15 s / 60 s)" + ) ], status="partially-implemented", doc=SODocumentation( @@ -785,11 +826,13 @@ ecr_id="ecr-l", controls=[], gaps=[ - "No persistent structured security event log (LGM-1/2/3/4 gap). dfetch prints " - "operational output to stderr but does not retain it, does not record which " - "credentials were used, which files were modified, or when remote access occurred. " - "C-036 ensures credentials are excluded from operational output but is not a " - "logging control" + ( + "No persistent structured security event log (LGM-1/2/3/4 gap). dfetch prints " + "operational output to stderr but does not retain it, does not record which " + "credentials were used, which files were modified, or when remote access occurred. " + "C-036 ensures credentials are excluded from operational output but is not a " + "logging control" + ) ], status="partially-implemented", doc=SODocumentation( @@ -846,12 +889,14 @@ so_id="so-secure-data-deletion", ecr_id="ecr-m", not_applicable=[ - "No dfetch-specific secure-deletion control is required: dfetch stores " - "no personal data and no keying material. The only on-disk state is " - ".dfetch_data.yaml (non-sensitive metadata) and vendored source files " - "(third-party code, not user data). Standard OS file deletion (rm / del) " - "is sufficient; cryptographic wipe is not warranted. DLM-1 is satisfied " - "by design (no sensitive data to wipe), not by a dedicated dfetch control." + ( + "No dfetch-specific secure-deletion control is required: dfetch stores " + "no personal data and no keying material. The only on-disk state is " + ".dfetch_data.yaml (non-sensitive metadata) and vendored source files " + "(third-party code, not user data). Standard OS file deletion (rm / del) " + "is sufficient; cryptographic wipe is not warranted. DLM-1 is satisfied " + "by design (no sensitive data to wipe), not by a dedicated dfetch control." + ) ], status="implemented", doc=SODocumentation( @@ -904,60 +949,3 @@ ), ), ] - -# ── CRA Part II requirements (prEN 40000-1-3) ──────────────────────────────── - -PART_II_REQUIREMENTS: list[PartIIRequirement] = [ - PartIIRequirement( - id="pii-01", - ref="Part II §1", - text="Identify and document vulnerabilities and components (SBOM).", - controls=["C-021", "C-022"], - status="implemented", - ), - PartIIRequirement( - id="pii-02", - ref="Part II §2", - text="Address vulnerabilities without delay; provide free security updates.", - controls=["C-015", "C-016", "SECURITY.md"], - gaps=[ - "No LTS backport policy (latest release only — documented in SECURITY.md)" - ], - status="partially-implemented", - ), - PartIIRequirement( - id="pii-03", - ref="Part II §3", - text="Apply effective coordinated vulnerability disclosure (CVD) policy.", - controls=["SECURITY.md"], - status="implemented", - ), - PartIIRequirement( - id="pii-04", - ref="Part II §4", - text="Report actively exploited vulnerabilities to national CSIRT and ENISA.", - status="not-applicable", - ), - PartIIRequirement( - id="pii-05", - ref="Part II §5", - text="Publish coordinated vulnerability disclosure policy.", - controls=["SECURITY.md"], - status="implemented", - ), - PartIIRequirement( - id="pii-06", - ref="Part II §6", - text="Share information on vulnerabilities in integrated components.", - controls=["C-022", "C-016"], - gaps=["No proactive downstream notification process"], - status="partially-implemented", - ), - PartIIRequirement( - id="pii-07", - ref="Part II §7", - text="Provide security updates free of charge for the support period.", - controls=["MIT licence", "PyPI", "SECURITY.md"], - status="implemented", - ), -] diff --git a/security/compliance_part_ii_data.py b/security/compliance_part_ii_data.py new file mode 100644 index 00000000..92699ae8 --- /dev/null +++ b/security/compliance_part_ii_data.py @@ -0,0 +1,61 @@ +"""Static compliance data: CRA Part II software requirements (prEN 40000-1-3). + +Split out of compliance_data.py to stay within the 1000-line limit per file. +""" + +from security.compliance_types import PartIIRequirement + +PART_II_REQUIREMENTS: list[PartIIRequirement] = [ + PartIIRequirement( + id="pii-01", + ref="Part II §1", + text="Identify and document vulnerabilities and components (SBOM).", + controls=["C-021", "C-022"], + status="implemented", + ), + PartIIRequirement( + id="pii-02", + ref="Part II §2", + text="Address vulnerabilities without delay; provide free security updates.", + controls=["C-015", "C-016", "SECURITY.md"], + gaps=[ + "No LTS backport policy (latest release only — documented in SECURITY.md)" + ], + status="partially-implemented", + ), + PartIIRequirement( + id="pii-03", + ref="Part II §3", + text="Apply effective coordinated vulnerability disclosure (CVD) policy.", + controls=["SECURITY.md"], + status="implemented", + ), + PartIIRequirement( + id="pii-04", + ref="Part II §4", + text="Report actively exploited vulnerabilities to national CSIRT and ENISA.", + status="not-applicable", + ), + PartIIRequirement( + id="pii-05", + ref="Part II §5", + text="Publish coordinated vulnerability disclosure policy.", + controls=["SECURITY.md"], + status="implemented", + ), + PartIIRequirement( + id="pii-06", + ref="Part II §6", + text="Share information on vulnerabilities in integrated components.", + controls=["C-022", "C-016"], + gaps=["No proactive downstream notification process"], + status="partially-implemented", + ), + PartIIRequirement( + id="pii-07", + ref="Part II §7", + text="Provide security updates free of charge for the support period.", + controls=["MIT licence", "PyPI", "SECURITY.md"], + status="implemented", + ), +] diff --git a/security/tm_elements.py b/security/tm_elements.py index 3b09bf7c..73a2fa3f 100644 --- a/security/tm_elements.py +++ b/security/tm_elements.py @@ -11,7 +11,7 @@ from pytm import Assumption, Boundary -from security.tm_controls_data import Control # noqa: F401 # re-exported +from security.tm_controls_data import Control # re-exported THREATS_FILE = os.path.join(os.path.dirname(__file__), "threats.json") diff --git a/security/tm_render.py b/security/tm_render.py index d98a8a9b..97856198 100644 --- a/security/tm_render.py +++ b/security/tm_render.py @@ -111,7 +111,7 @@ def _render_asset_rows( The highest risk across all matching findings/controls is used per dimension. """ assets = sorted( - list(getattr(TM, "_assets")) + list(getattr(TM, "_data", [])), + list(getattr(TM, "_assets", [])) + list(getattr(TM, "_data", [])), key=lambda e: getattr(e, "name", ""), ) if not assets: diff --git a/security/tm_supply_chain.py b/security/tm_supply_chain.py index 540e1842..e5765e83 100644 --- a/security/tm_supply_chain.py +++ b/security/tm_supply_chain.py @@ -21,7 +21,7 @@ sys.path.insert(0, _repo_root) # pylint: disable=wrong-import-position -from pytm import ( # noqa: E402 +from pytm import ( TM, Actor, Boundary, @@ -33,8 +33,8 @@ Process, ) -from security.tm_controls_data import SC_CONTROLS as CONTROLS # noqa: E402 -from security.tm_elements import ( # noqa: E402 +from security.tm_controls_data import SC_CONTROLS as CONTROLS +from security.tm_elements import ( THREATS_FILE, Control, ThreatResponse, @@ -42,7 +42,7 @@ make_dev_env_boundary, make_supply_chain_assumptions, ) -from security.tm_render import apply_report_utils_patch, run_model # noqa: E402 +from security.tm_render import apply_report_utils_patch, run_model # pylint: enable=wrong-import-position diff --git a/security/tm_usage.py b/security/tm_usage.py index 31e1ec3c..0b019754 100644 --- a/security/tm_usage.py +++ b/security/tm_usage.py @@ -33,7 +33,7 @@ sys.path.insert(0, _repo_root) # pylint: disable=wrong-import-position -from pytm import ( # noqa: E402 +from pytm import ( TM, Actor, Boundary, @@ -45,8 +45,8 @@ Process, ) -from security.tm_controls_data import USAGE_CONTROLS as CONTROLS # noqa: E402 -from security.tm_elements import ( # noqa: E402 +from security.tm_controls_data import USAGE_CONTROLS as CONTROLS +from security.tm_elements import ( THREATS_FILE, Control, ThreatResponse, @@ -55,7 +55,7 @@ make_network_boundary, make_usage_assumptions, ) -from security.tm_render import apply_report_utils_patch, run_model # noqa: E402 +from security.tm_render import apply_report_utils_patch, run_model # pylint: enable=wrong-import-position diff --git a/stubs/py_serializable/__init__.pyi b/stubs/py_serializable/__init__.pyi index ec55f39d..bb518776 100644 --- a/stubs/py_serializable/__init__.pyi +++ b/stubs/py_serializable/__init__.pyi @@ -9,43 +9,44 @@ at runtime. """ from collections.abc import Callable, Iterable -from typing import Any, Optional, TypeVar, overload +from typing import Any, TypeVar, overload + +from typing_extensions import Self _T = TypeVar("_T") class _Serializable: def as_json(self) -> str: ... @classmethod - def from_json(cls: type[_T], data: dict[str, Any]) -> Optional[_T]: ... + def from_json(cls, data: dict[str, Any]) -> Self | None: ... def as_xml(self) -> Any: ... @classmethod - def from_xml(cls: type[_T], data: Any) -> Optional[_T]: ... + def from_xml(cls, data: Any) -> Self | None: ... @overload def serializable_class( cls: None = ..., *, - name: Optional[str] = ..., - serialization_types: Optional[Iterable[Any]] = ..., - ignore_during_deserialization: Optional[Iterable[str]] = ..., + name: str | None = ..., + serialization_types: Iterable[Any] | None = ..., + ignore_during_deserialization: Iterable[str] | None = ..., ignore_unknown_during_deserialization: bool = ..., ) -> Callable[[type[_T]], type[_T]]: ... @overload def serializable_class( cls: type[_T], *, - name: Optional[str] = ..., - serialization_types: Optional[Iterable[Any]] = ..., - ignore_during_deserialization: Optional[Iterable[str]] = ..., + name: str | None = ..., + serialization_types: Iterable[Any] | None = ..., + ignore_during_deserialization: Iterable[str] | None = ..., ignore_unknown_during_deserialization: bool = ..., ) -> type[_T]: ... def serializable_class(cls: Any = None, **kwargs: Any) -> Any: ... - def xml_name(name: str) -> Callable[[_T], _T]: ... def xml_sequence(order: int) -> Callable[[_T], _T]: ... def xml_array( array_type: Any, - child_name: Optional[str] = ..., + child_name: str | None = ..., ) -> Callable[[_T], _T]: ... def xml_attribute() -> Callable[[_T], _T]: ... def xml_string(string_type: Any) -> Callable[[_T], _T]: ... diff --git a/tests/test_fuzzing.py b/tests/test_fuzzing.py index 777e9e1d..812a01cf 100644 --- a/tests/test_fuzzing.py +++ b/tests/test_fuzzing.py @@ -169,23 +169,27 @@ def test_manifest_can_be_created(data): @given(manifest_strategy) def test_check(data): """Validate check command.""" - with suppress(DfetchFatalException): - with tempfile.TemporaryDirectory() as tmpdir: - with in_directory(tmpdir): - with open("dfetch.yaml", "w", encoding="UTF-8") as manifest_file: - yaml.dump(data, manifest_file) - run(["check"]) + with ( + suppress(DfetchFatalException), + tempfile.TemporaryDirectory() as tmpdir, + in_directory(tmpdir), + ): + with open("dfetch.yaml", "w", encoding="UTF-8") as manifest_file: + yaml.dump(data, manifest_file) + run(["check"]) @given(manifest_strategy) def test_update(data): """Validate update command.""" - with suppress(DfetchFatalException): - with tempfile.TemporaryDirectory() as tmpdir: - with in_directory(tmpdir): - with open("dfetch.yaml", "w", encoding="UTF-8") as manifest_file: - yaml.dump(data, manifest_file) - run(["update"]) + with ( + suppress(DfetchFatalException), + tempfile.TemporaryDirectory() as tmpdir, + in_directory(tmpdir), + ): + with open("dfetch.yaml", "w", encoding="UTF-8") as manifest_file: + yaml.dump(data, manifest_file) + run(["update"]) if __name__ == "__main__": diff --git a/tests/test_tree_browser.py b/tests/test_tree_browser.py index 8744a9f0..69437e48 100644 --- a/tests/test_tree_browser.py +++ b/tests/test_tree_browser.py @@ -153,10 +153,12 @@ def _browser( *, idx: int = 0, top: int = 0, - config: BrowserConfig = BrowserConfig(), + config: BrowserConfig | None = None, children: list[Entry] | None = None, ) -> _HeadlessBrowser: """Return a seeded _HeadlessBrowser backed by *children* (or empty) for expansions.""" + if config is None: + config = BrowserConfig() dir_path = nodes[0].path if nodes else "" ls = ( (lambda path: children if path == dir_path else []) From ec80384e8ead5443dafc6ebfb2d7e768654d39fb Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 10:53:00 +0000 Subject: [PATCH 4/4] Run the mypy hook serially to fix a mypy-cache race in CI Same class of bug as the pyright hook's nodeenv race: pre-commit splits dfetch/'s ~71 files across several concurrent mypy subprocesses (no require_serial), and mypy's incremental cache (.mypy_cache/) isn't safe for concurrent readers/writers -- one process's cache access collides with another's and mypy crashes with "INTERNAL ERROR", exit code 2. Reproduced locally: running mypy on the same 4 file-batches concurrently against a shared cache crashes one batch every time; running the same batches sequentially against the same cache never crashes. require_serial: true stops pre-commit from running more than one mypy process at a time, matching the fix already applied to the pyright hook. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_013hKpsXyi4hXAT3pDobq1qb --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c295ac79..6527c59f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,6 +43,7 @@ repos: entry: mypy language: system files: ^dfetch/ + require_serial: true types: [file, python] - id: doc8 name: doc8