diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index b62693fe..cab83c29 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -67,6 +67,8 @@ jobs: uses: devcontainers/ci@513af61f4de4f75d37e4438f184ba4358f0fc1ca # v0.3.1900000450 with: runCmd: | + set -e + echo "Installing test dependencies..." pip install -e .[development,docs,casts] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 43c6291d..6527c59f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,6 +43,7 @@ repos: entry: mypy language: system files: ^dfetch/ + require_serial: true types: [file, python] - id: doc8 name: doc8 @@ -110,12 +111,14 @@ repos: entry: ruff language: python args: [check] + exclude: ^doc/_ext/sphinxcontrib_asciinema types: [file, python] - id: pyright name: pyright description: Lint using pyright entry: pyright language: python + require_serial: true types: [file, python] - id: pyupgrade name: pyupgrade diff --git a/doc/_ext/colordot.py b/doc/_ext/colordot.py index 9295ab79..5f4d60bf 100644 --- a/doc/_ext/colordot.py +++ b/doc/_ext/colordot.py @@ -20,7 +20,7 @@ import html import re -from typing import Any, List +from typing import Any from docutils import nodes from docutils.nodes import Node, system_message @@ -99,7 +99,7 @@ def _replace_emoji_for_latex( if parent is None: continue idx = parent.children.index(text_node) - new_nodes: List[Node] = [] + new_nodes: list[Node] = [] last = 0 for m in _EMOJI_RE.finditer(text): if m.start() > last: diff --git a/doc/_ext/designguide.py b/doc/_ext/designguide.py index 8014ac68..d4f4a166 100644 --- a/doc/_ext/designguide.py +++ b/doc/_ext/designguide.py @@ -25,7 +25,8 @@ """ import html -from typing import Any +from collections.abc import Callable +from typing import Any, ClassVar from docutils import nodes from docutils.nodes import Node @@ -38,7 +39,7 @@ class SwatchDirective(Directive): required_arguments = 1 optional_arguments = 0 - option_spec = { + option_spec: ClassVar[dict[str, Callable[[str], Any]] | None] = { "token": directives.unchanged, "label": directives.unchanged, "usage": directives.unchanged, @@ -81,7 +82,7 @@ class PaletteDirective(Directive): required_arguments = 0 optional_arguments = 0 - option_spec = { + option_spec: ClassVar[dict[str, Callable[[str], Any]] | None] = { "columns": directives.positive_int, } has_content = True diff --git a/doc/_ext/dfetch_style.py b/doc/_ext/dfetch_style.py index 7019d558..deb02a3a 100644 --- a/doc/_ext/dfetch_style.py +++ b/doc/_ext/dfetch_style.py @@ -2,7 +2,8 @@ import sys import types -from typing import MutableMapping, cast +from collections.abc import Mapping, MutableMapping +from typing import Any, ClassVar, cast import pygments.styles from pygments.style import Style @@ -24,7 +25,7 @@ class DfetchStyle(Style): # pylint: disable=too-few-public-methods background_color = "#fef8f0" # --bg-tint default_style = "" - styles = { + styles: ClassVar[Mapping[Any, str]] = { Token: "#1c1917", # --text Comment: "italic #78716c", # --text-muted Comment.Preproc: "noitalic #a0510a", # --primary-dark diff --git a/doc/_ext/scenario_directive.py b/doc/_ext/scenario_directive.py index deffeb5d..87d92999 100644 --- a/doc/_ext/scenario_directive.py +++ b/doc/_ext/scenario_directive.py @@ -39,7 +39,8 @@ import os import re import textwrap -from typing import Dict, FrozenSet, List, Tuple +from collections.abc import Callable +from typing import Any, ClassVar from docutils import nodes from docutils.parsers.rst import Directive, directives @@ -83,9 +84,9 @@ class ScenarioIncludePlaceholder(nodes.General, nodes.Element): # --------------------------------------------------------------------------- -def _feature_tags(feature_path: str) -> List[str]: +def _feature_tags(feature_path: str) -> list[str]: """Return all behave tags declared before the ``Feature:`` line.""" - tags: List[str] = [] + tags: list[str] = [] with open(feature_path, encoding="utf-8") as fh: for line in fh: stripped = line.strip() @@ -96,7 +97,7 @@ def _feature_tags(feature_path: str) -> List[str]: return tags -def _group_tag(feature_path: str, non_group_tags: FrozenSet[str]) -> str: +def _group_tag(feature_path: str, non_group_tags: frozenset[str]) -> str: """Return the first tag not in *non_group_tags*, or ``'other'``.""" for tag in _feature_tags(feature_path): if tag not in non_group_tags: @@ -114,7 +115,7 @@ def _feature_title(feature_path: str) -> str: return os.path.basename(feature_path) -def _all_scenarios(feature_path: str) -> Tuple[Tuple[str, str], ...]: +def _all_scenarios(feature_path: str) -> tuple[tuple[str, str], ...]: """Return (header, title) pairs for all scenarios in the feature file.""" with open(feature_path, encoding="utf-8") as fh: return tuple( @@ -131,7 +132,7 @@ def _full_feature_content(feature_path: str) -> str: return fh.read() -def _selected_scenarios_content(feature_path: str, scenario_titles: List[str]) -> str: +def _selected_scenarios_content(feature_path: str, scenario_titles: list[str]) -> str: """Return content containing only the selected scenario blocks.""" with open(feature_path, encoding="utf-8") as fh: content = fh.read() @@ -173,7 +174,7 @@ class ScenarioIncludeDirective(Directive): required_arguments = 1 optional_arguments = 0 final_argument_whitespace = False - option_spec = { + option_spec: ClassVar[dict[str, Callable[[str], Any]] | None] = { "scenario": str, "inline": directives.flag, # keep inline even in PDF mode } @@ -192,7 +193,7 @@ def _feature_abs(self, feature_file: str) -> str: raise self.error(f"Feature file not found: {path}") return path - def _requested_scenarios(self, available: Tuple[Tuple[str, str], ...]) -> List[str]: + def _requested_scenarios(self, available: tuple[tuple[str, str], ...]) -> list[str]: return [ t.strip() for t in self.options.get("scenario", "").splitlines() @@ -203,7 +204,7 @@ def _requested_scenarios(self, available: Tuple[Tuple[str, str], ...]) -> List[s # Appendix entry registration (always runs, any builder) # ------------------------------------------------------------------ - def _entry_metadata(self, feature_abs: str) -> Tuple[str, str, str]: + def _entry_metadata(self, feature_abs: str) -> tuple[str, str, str]: """Return (label, group_tag, feature_title) for a feature file.""" env = self._env() non_group_tags = frozenset(getattr(env.config, "scenario_non_command_tags", [])) @@ -218,7 +219,7 @@ def _register_appendix_entry( self, feature_file: str, feature_abs: str, - scenario_titles: List[str], + scenario_titles: list[str], ) -> None: """Store entry in env.scenario_appendix_entries for any builder. @@ -254,7 +255,7 @@ def _register_appendix_entry( # Entry point # ------------------------------------------------------------------ - def run(self) -> List[nodes.Node]: + def run(self) -> list[nodes.Node]: feature_file = self.arguments[0].strip() feature_abs = self._feature_abs(feature_file) available = _all_scenarios(feature_abs) @@ -310,7 +311,7 @@ class ScenarioAppendixDirective(Directive): optional_arguments = 0 has_content = False - def run(self) -> List[nodes.Node]: + def run(self) -> list[nodes.Node]: env = self.state.document.settings.env # Record which document hosts the appendix so that ScenarioAppendixRef # nodes in other documents can be resolved with the correct refdocname. @@ -329,13 +330,13 @@ def run(self) -> List[nodes.Node]: # --------------------------------------------------------------------------- -def _build_appendix_nodes(entries: Dict) -> List[nodes.Node]: +def _build_appendix_nodes(entries: dict) -> list[nodes.Node]: """Build docutils section nodes for every collected appendix entry.""" - by_tag: Dict[str, List] = {} + by_tag: dict[str, list] = {} for entry in entries.values(): by_tag.setdefault(entry["group_tag"], []).append(entry) - result: List[nodes.Node] = [] + result: list[nodes.Node] = [] for tag in sorted(by_tag): tag_entries = sorted(by_tag[tag], key=lambda e: e["feature_title"]) label = f"appendix-{tag}" @@ -366,10 +367,10 @@ def _build_appendix_nodes(entries: Dict) -> List[nodes.Node]: def _render_scenario_inline( - scenario_titles: List[str], feature_abs: str -) -> List[nodes.Node]: + scenario_titles: list[str], feature_abs: str +) -> list[nodes.Node]: """Return docutils nodes for inline HTML rendering of *scenario_titles*.""" - result: List[nodes.Node] = [] + result: list[nodes.Node] = [] for title in scenario_titles: raw_content = _selected_scenarios_content(feature_abs, [title]) content = textwrap.dedent(raw_content).strip() diff --git a/doc/conf.py b/doc/conf.py old mode 100644 new mode 100755 index 04a4cfb3..d52a6195 --- a/doc/conf.py +++ b/doc/conf.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Documentation build configuration file. """ @@ -18,7 +17,7 @@ ext_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "_ext")) sys.path.insert(0, ext_path) -import dfetch_style # noqa: E402 +import dfetch_style dfetch_style.register() diff --git a/doc/generate-casts/interactive_helper.py b/doc/generate-casts/interactive_helper.py old mode 100644 new mode 100755 diff --git a/doc/landing-page/conf.py b/doc/landing-page/conf.py old mode 100644 new mode 100755 index e5762639..8c4730e9 --- a/doc/landing-page/conf.py +++ b/doc/landing-page/conf.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Documentation build configuration file. """ diff --git a/doc/static/uml/generate_diagram.py b/doc/static/uml/generate_diagram.py index 21b6beb7..216170b9 100644 --- a/doc/static/uml/generate_diagram.py +++ b/doc/static/uml/generate_diagram.py @@ -3,8 +3,8 @@ import os import pathlib import re +from collections.abc import Sequence from pathlib import Path -from typing import Sequence, Tuple import_regex = re.compile(r"(import|from) dfetch\.(?P[\.\w]+)") description_regex = re.compile(r"^\"{3}(?P.*)") @@ -17,7 +17,7 @@ class Relation: @dataclasses.dataclass class Module: - path: Tuple[str] + path: tuple[str] name: str description: str relations: list @@ -129,7 +129,7 @@ def generate_c3(relations, path: Sequence[str], blacklist=None): f'{indent}{indent}Component(comp{name}, "{name}", "python", "{description}")' ) - print("") + print() for name, module in modules.items(): if isinstance(module, dict): continue @@ -145,14 +145,14 @@ def generate_c3(relations, path: Sequence[str], blacklist=None): print( f'{indent}Container(cont{container}, "{container}", "python", "Something.")' ) - print("") + print() for relation in outside_in: print(relation) - print("") + print() for relation in inside_out: print(relation) - print("") + print() print(C3_END_TEMPLATE) diff --git a/features/steps/add_steps.py b/features/steps/add_steps.py index 4ca37d06..46db802c 100644 --- a/features/steps/add_steps.py +++ b/features/steps/add_steps.py @@ -48,9 +48,11 @@ def _auto_prompt(_prompt: str, **kwargs) -> str: # type: ignore[return] return prompt_answers.popleft() return str(kwargs.get("default", "")) - with patch("dfetch.commands.add.Prompt.ask", side_effect=_auto_prompt): - with patch("dfetch.commands.add.Confirm.ask", side_effect=_auto_confirm): - call_command(context, cmd) + with ( + patch("dfetch.commands.add.Prompt.ask", side_effect=_auto_prompt), + patch("dfetch.commands.add.Confirm.ask", side_effect=_auto_confirm), + ): + call_command(context, cmd) @when('I run "dfetch {add_args}" with inputs') diff --git a/features/steps/generic_steps.py b/features/steps/generic_steps.py index fac18aeb..44695892 100644 --- a/features/steps/generic_steps.py +++ b/features/steps/generic_steps.py @@ -9,9 +9,10 @@ import pathlib import re import shutil +from collections.abc import Iterable from contextlib import contextmanager from itertools import zip_longest -from typing import Iterable, List, Optional, Pattern, Tuple, Union +from re import Pattern from unittest.mock import patch from behave import given, then, when # pylint: disable=no-name-in-module @@ -57,18 +58,17 @@ def remote_server_path(context) -> str: return pathlib.Path(context.remotes_dir_path).as_uri() -def call_command(context: Context, args: list[str], path: Optional[str] = ".") -> None: +def call_command(context: Context, args: list[str], path: str | None = ".") -> None: before = context.console.export_text() DLogger.reset_projects() - with temporary_env("CI", "true"): - with in_directory(path or "."): - try: - run(args, context.console) - context.cmd_returncode = 0 - except DfetchFatalException: - context.cmd_returncode = 1 + with temporary_env("CI", "true"), in_directory(path or "."): + try: + run(args, context.console) + context.cmd_returncode = 0 + except DfetchFatalException: + context.cmd_returncode = 1 after = context.console.export_text() context.cmd_output = after[len(before) :].strip("\n") @@ -186,7 +186,7 @@ def list_dir(path): result = "" prev_node = [] - for node in list(sorted(nodes)) + [""]: + for node in sorted(nodes) + [""]: if prev_node: end = "" if "".join(node).startswith("".join(prev_node)): @@ -359,7 +359,7 @@ def step_impl(context, name): check_file_exists(name) -def check_json(path: Union[str, os.PathLike], content: str, context) -> None: +def check_json(path: str | os.PathLike, content: str, context) -> None: """Check a JSON file for exact equality (after normalising formatting).""" content = apply_archive_substitutions(content, context) with open(path, "r", encoding="UTF-8") as file_to_check: @@ -394,7 +394,7 @@ def step_impl(_, path, target): assert actual == target, f"Expected {path!r} to point to {target!r}, got {actual!r}" -def multisub(patterns: List[Tuple[Pattern[str], str]], text: str) -> str: +def multisub(patterns: list[tuple[Pattern[str], str]], text: str) -> str: """Apply a list of tuples that each contain a regex + replace string.""" for pattern, replace in patterns: text = pattern.sub(replace, text) diff --git a/features/steps/git_steps.py b/features/steps/git_steps.py index 745b48fb..c0ec9b8b 100644 --- a/features/steps/git_steps.py +++ b/features/steps/git_steps.py @@ -221,7 +221,7 @@ def step_impl(context, name, ending): create_repo() subprocess.check_call(["git", "config", "core.autocrlf", "false"]) pathlib.Path("README.md").write_bytes( - f"Generated file for {name}{terminator}".encode("utf-8") + f"Generated file for {name}{terminator}".encode() ) commit_all("Initial commit") tag("v1") @@ -239,7 +239,7 @@ def step_impl(context, name, ending, filename, gitattr): subprocess.check_call(["git", "config", "core.autocrlf", "false"]) pathlib.Path(".gitattributes").write_text(gitattr + "\n", encoding="utf-8") pathlib.Path(filename).write_bytes( - f"Generated file for {name}{terminator}".encode("utf-8") + f"Generated file for {name}{terminator}".encode() ) commit_all("Initial commit") tag("v1") diff --git a/features/steps/json_steps.py b/features/steps/json_steps.py index 6a19e7d3..c886383f 100644 --- a/features/steps/json_steps.py +++ b/features/steps/json_steps.py @@ -6,7 +6,6 @@ import json import os import re -from typing import Union from behave import then # pylint: disable=no-name-in-module @@ -59,16 +58,19 @@ def _try_match(exp_index: int, used: set) -> bool: return True exp_item = expected[exp_index] for i, act_item in enumerate(actual): - if i not in used and _json_subset_matches(exp_item, act_item): - if _try_match(exp_index + 1, used | {i}): - return True + if ( + i not in used + and _json_subset_matches(exp_item, act_item) + and _try_match(exp_index + 1, used | {i}) + ): + return True return False return _try_match(0, set()) return expected == actual -def check_json_subset(path: Union[str, os.PathLike], content: str, context) -> None: +def check_json_subset(path: str | os.PathLike, content: str, context) -> None: """Assert that a JSON file *contains* the given key-values (subset match). Dynamic placeholders (````, ````) in diff --git a/features/steps/manifest_steps.py b/features/steps/manifest_steps.py index 909c59d2..6554c080 100644 --- a/features/steps/manifest_steps.py +++ b/features/steps/manifest_steps.py @@ -5,7 +5,6 @@ import os import pathlib -from typing import Optional from behave import given, then, when # pylint: disable=no-name-in-module @@ -27,7 +26,7 @@ def apply_manifest_substitutions(context, contents: str) -> str: def generate_manifest( - context, name="dfetch.yaml", contents: Optional[str] = None, path=None + context, name="dfetch.yaml", contents: str | None = None, path=None ): contents = contents or context.text manifest = apply_manifest_substitutions(context, contents) diff --git a/features/steps/svn_steps.py b/features/steps/svn_steps.py index 53b98eb7..5154b105 100644 --- a/features/steps/svn_steps.py +++ b/features/steps/svn_steps.py @@ -175,9 +175,8 @@ def step_impl(context, name, ext_path, source_name): source_url = ( pathlib.Path(context.remotes_dir_path).joinpath(source_name, "trunk").as_uri() ) - with in_directory(name): - with in_directory("trunk"): - add_externals([{"url": source_url, "path": ext_path, "revision": ""}]) + with in_directory(name), in_directory("trunk"): + add_externals([{"url": source_url, "path": ext_path, "revision": ""}]) @given( @@ -185,9 +184,8 @@ def step_impl(context, name, ext_path, source_name): ) def step_impl(context, name, ext_path, source_name): source_url = pathlib.Path(context.remotes_dir_path).joinpath(source_name).as_uri() - with in_directory(name): - with in_directory("trunk"): - add_externals([{"url": source_url, "path": ext_path, "revision": ""}]) + with in_directory(name), in_directory("trunk"): + add_externals([{"url": source_url, "path": ext_path, "revision": ""}]) @given('a svn-server "{name}" with {ending} content') @@ -198,7 +196,7 @@ def step_impl(context, name, ending): create_stdlayout() with in_directory("trunk"): pathlib.Path("README.md").write_bytes( - f"Generated file for {name}{terminator}".encode("utf-8") + f"Generated file for {name}{terminator}".encode() ) subprocess.check_call(["svn", "update", "."]) subprocess.check_call(["svn", "add", "--force", "."]) diff --git a/pyproject.toml b/pyproject.toml index 2ec878c0..240880fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -129,6 +129,7 @@ target-version = ["py313"] [tool.isort] profile = "black" +py_version = "310" skip = [ "doc/_ext/sphinxcontrib_asciinema", ] @@ -198,6 +199,9 @@ fail_under = 72 skip = "*.cast,./venv,**/plantuml-c4/**,./example,.mypy_cache,./doc/_build/**,./doc/landing-page/_build/**,./doc/_ext/sphinxcontrib_asciinema/**,./build,*.patch,.git,**/generate-casts/demo-magic/**,./doc/openssl/**" ignore-words-list = "bund" +[tool.ruff] +extend-exclude = ["doc/_ext/sphinxcontrib_asciinema"] + [tool.ruff.lint.per-file-ignores] "features/steps/*" = ["F811"] diff --git a/script/build.py b/script/build.py old mode 100644 new mode 100755 index 104e6cb8..34907f34 --- a/script/build.py +++ b/script/build.py @@ -3,15 +3,13 @@ import subprocess # nosec import sys + import tomllib as toml -from typing import Union from dfetch import __version__ -def parse_option( - option_name: str, option_value: Union[bool, str, list, dict] -) -> list[str]: +def parse_option(option_name: str, option_value: bool | str | list | dict) -> list[str]: """ Convert a config value to Nuitka CLI arguments. diff --git a/script/create_release_notes.py b/script/create_release_notes.py old mode 100644 new mode 100755 diff --git a/script/create_sbom.py b/script/create_sbom.py old mode 100644 new mode 100755 index 841845a4..e43a87d1 --- a/script/create_sbom.py +++ b/script/create_sbom.py @@ -11,6 +11,7 @@ from pathlib import Path logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) PROJECT_DIR = Path(__file__).parent.parent.resolve() @@ -35,7 +36,7 @@ def temporary_venv(): """Create a temporary virtual environment and clean it up on exit.""" with tempfile.TemporaryDirectory(prefix="venv_sbom_") as tmpdir: venv_dir = Path(tmpdir) - logging.info(f"Creating temporary virtual environment at {venv_dir}") + logger.info(f"Creating temporary virtual environment at {venv_dir}") venv.create(venv_dir, with_pip=True, upgrade_deps=True) if sys.platform.startswith("win"): @@ -98,4 +99,4 @@ def parse_args() -> argparse.Namespace: [python, "-m", "cyclonedx_py", "environment", "-o", str(output_file)] ) -logging.info(f"SBOM generated at {output_file}") +logger.info(f"SBOM generated at {output_file}") diff --git a/script/dependabot_hook.py b/script/dependabot_hook.py index 0d38f88e..7a0af9cb 100755 --- a/script/dependabot_hook.py +++ b/script/dependabot_hook.py @@ -4,7 +4,6 @@ import re import sys from pathlib import Path -from typing import Optional # Config SBOM_FILE = "sbom.json" # path to your CycloneDX SBOM @@ -29,7 +28,7 @@ def get_new_version_from_pyproject(name: str) -> str: def replace_cyclonedx_version_if_outdated( new_version: str, -) -> Optional[str]: +) -> str | None: """Update the SBOM JSON file with the new version""" feature_file_path = Path(FEATURE_FILE) content = feature_file_path.read_text(encoding="UTF-8") diff --git a/script/package.py b/script/package.py old mode 100644 new mode 100755 index 126a2db1..5f96ea26 --- a/script/package.py +++ b/script/package.py @@ -4,10 +4,10 @@ import shutil import subprocess # nosec import sys -import tomllib as toml import xml.etree.ElementTree as ET # nosec (only used for XML generation, not parsing untrusted input) from pathlib import Path +import tomllib as toml from setuptools_scm import get_version # pylint: disable=import-error from dfetch import __version__ as __digit_only_version__ # Used inside the installers diff --git a/script/release.py b/script/release.py old mode 100644 new mode 100755 index 09e99ad9..90402903 --- a/script/release.py +++ b/script/release.py @@ -5,7 +5,7 @@ import glob import os import re -from datetime import datetime +from datetime import datetime, timezone from dfetch import __version__ @@ -45,7 +45,9 @@ def replace_pattern_in_files(file_path_pattern, search_pattern, replacement, fla replace_pattern_in_files( file_path_pattern=f"{base_dir}/CHANGELOG.rst", search_pattern=r"(Release \d+\.\d+\.\d+) \(unreleased\)", - replacement=r"\1 (released " + datetime.now().strftime("%Y-%m-%d") + ")", + replacement=r"\1 (released " + + datetime.now(tz=timezone.utc).strftime("%Y-%m-%d") + + ")", flags=re.DOTALL, ) @@ -69,7 +71,10 @@ def replace_pattern_in_files(file_path_pattern, search_pattern, replacement, fla flags=re.DOTALL, ) - major, minor, _ = map(int, __version__.split(".")) + version_match = re.match(r"(\d+)\.(\d+)\.(\d+)", __version__) + if not version_match: + raise ValueError(f"Could not parse version from {__version__!r}") + major, minor = int(version_match.group(1)), int(version_match.group(2)) replace_pattern_in_files( file_path_pattern=f"{base_dir}/doc/howto/contributing.rst", diff --git a/security/compliance.py b/security/compliance.py index 51eaeb57..66a6d5c6 100644 --- a/security/compliance.py +++ b/security/compliance.py @@ -20,17 +20,17 @@ import re import sys import uuid -from datetime import date +from datetime import datetime, timezone from typing import Any from security.compliance_data import ( ANNEX_V_MAP, CLASSIFICATION_DECISION, - PART_II_REQUIREMENTS, SO_IMPLEMENTATIONS, STANDARDS, TRACK_B_CONTROLS, ) +from security.compliance_part_ii_data import PART_II_REQUIREMENTS from security.compliance_types import SOImplementation from security.tm_controls_data import SC_CONTROLS, USAGE_CONTROLS, Control @@ -100,7 +100,7 @@ def _build_metadata(version: str) -> dict[str, Any]: """Return the OSCAL 1.2.2 metadata block with parties and roles.""" return { "title": "dfetch CRA Compliance Component Definition", - "last-modified": f"{date.today().isoformat()}T00:00:00Z", + "last-modified": f"{datetime.now(tz=timezone.utc).date().isoformat()}T00:00:00Z", "version": version, "oscal-version": "1.2.2", "document-ids": [ @@ -502,7 +502,7 @@ def _format_ref_as_rst(ref: str) -> str: if not ref or ref == "—": return "—" # Already RST markup - if ref.startswith(":doc:") or ref.startswith(":ref:"): + if ref.startswith((":doc:", ":ref:")): return ref # Handle parenthetical suffix like "path (note about it)" paren_suffix = "" @@ -664,8 +664,10 @@ def _gap_entries() -> list[tuple[str, str]]: """Return (title, body) pairs for the gap analysis section.""" return [ ( - ":ref:`C-043 ` — Release-gate CVE check" - " (ECR-a, SO.VulnerabilityManagementProcess → GEC-1)", + ( + ":ref:`C-043 ` — Release-gate CVE check" + " (ECR-a, SO.VulnerabilityManagementProcess → GEC-1)" + ), ( "dfetch's CI detects vulnerabilities at commit time " "(:ref:`C-015 `, :ref:`C-016 `, :ref:`C-017 `). " @@ -675,8 +677,10 @@ def _gap_entries() -> list[tuple[str, str]]: ), ), ( - ":ref:`C-044 ` — Data minimisation policy" - " (ECR-g, SO.DataMinimization → DTM-1)", + ( + ":ref:`C-044 ` — Data minimisation policy" + " (ECR-g, SO.DataMinimization → DTM-1)" + ), ( "dfetch processes dependency metadata only. The ``.dfetch_data.yaml`` " "file stores: ``remote_url`` (credentials stripped by " @@ -688,8 +692,10 @@ def _gap_entries() -> list[tuple[str, str]]: ), ), ( - ":ref:`C-046 ` — Exploit mitigation inventory" - " (ECR-k, SO.ReduceImpactOfIncident → GEC-11)", + ( + ":ref:`C-046 ` — Exploit mitigation inventory" + " (ECR-k, SO.ReduceImpactOfIncident → GEC-11)" + ), ( "prEN 40000-1-4 ECR-k requires documenting applicable exploit " "mitigation techniques. For dfetch (pure Python):\n\n" diff --git a/security/compliance_data.py b/security/compliance_data.py index e89bb72d..8871a278 100644 --- a/security/compliance_data.py +++ b/security/compliance_data.py @@ -4,13 +4,12 @@ Kept in a separate module to stay within the 1000-line limit per file. """ -from security.compliance_types import ( # noqa: F401 # re-exported +from security.compliance_types import ( # re-exported ApplicableStandard, - PartIIRequirement, SODocumentation, SOImplementation, ) -from security.tm_controls_data import Control # noqa: F401 # re-exported +from security.tm_controls_data import Control # re-exported # ── Classification decision ─────────────────────────────────────────────────── @@ -143,46 +142,68 @@ ANNEX_V_MAP: list[tuple[str, str]] = [ ( - "**1. General description** — intended purpose, product name and version, " - "manufacturer address", - ":doc:`security` § *Product and manufacturer identification*; " - ":doc:`../reference/manifest` (manifest schema and version field)", + ( + "**1. General description** — intended purpose, product name and version, " + "manufacturer address" + ), + ( + ":doc:`security` § *Product and manufacturer identification*; " + ":doc:`../reference/manifest` (manifest schema and version field)" + ), ), ( - "**2. Design and development** — software architecture; how components " - "build on or feed into each other", - ":doc:`../explanation/architecture` (layer diagram and module overview); " - ":doc:`security_pipeline` § *Threat model pipeline* (security-relevant " - "component relationships)", + ( + "**2. Design and development** — software architecture; how components " + "build on or feed into each other" + ), + ( + ":doc:`../explanation/architecture` (layer diagram and module overview); " + ":doc:`security_pipeline` § *Threat model pipeline* (security-relevant " + "component relationships)" + ), ), ( - "**3. Production and monitoring** — build pipeline, dependency management, " - "CI/CD monitoring", - ":doc:`security_pipeline` § *Compliance pipeline* and *Release attestations*; " - "CI workflows in " - "`\\.github/workflows/ `_", + ( + "**3. Production and monitoring** — build pipeline, dependency management, " + "CI/CD monitoring" + ), + ( + ":doc:`security_pipeline` § *Compliance pipeline* and *Release attestations*; " + "CI workflows in " + "`\\.github/workflows/ `_" + ), ), ( - "**4. Cybersecurity risk assessment** (Article 13(2)) — asset identification, " - "threat analysis, risk treatment", - ":doc:`threat_model_supply_chain` (pre-install lifecycle); " - ":doc:`threat_model_usage` (runtime invocation); " - "see also :doc:`security` § *Risk Rating Methodology*", + ( + "**4. Cybersecurity risk assessment** (Article 13(2)) — asset identification, " + "threat analysis, risk treatment" + ), + ( + ":doc:`threat_model_supply_chain` (pre-install lifecycle); " + ":doc:`threat_model_usage` (runtime invocation); " + "see also :doc:`security` § *Risk Rating Methodology*" + ), ), ( - "**5. Implemented security solutions and applied standards** — " - "list of harmonised standards applied; where not applied, description of how " - "each Annex I requirement is met", - "This page (§§ *Applicable Standards*, *Part I*, *Part II*); " - ":doc:`control_register` (all 46 controls with references); " - "OSCAL Component Definition " - "`security/dfetch.component-definition.json " - "`_", + ( + "**5. Implemented security solutions and applied standards** — " + "list of harmonised standards applied; where not applied, description of how " + "each Annex I requirement is met" + ), + ( + "This page (§§ *Applicable Standards*, *Part I*, *Part II*); " + ":doc:`control_register` (all 46 controls with references); " + "OSCAL Component Definition " + "`security/dfetch.component-definition.json " + "`_" + ), ), ( "**6. EU Declaration of Conformity** (Annex IV)", - "Not required. dfetch is outside mandatory CRA scope (see " - "*Classification Decision* above). No CE marking is affixed.", + ( + "Not required. dfetch is outside mandatory CRA scope (see " + "*Classification Decision* above). No CE marking is affixed." + ), ), ] @@ -226,8 +247,10 @@ "AUM-5 (no password or authentication mechanism)", ], gaps=[ - "Integrity hash verification (:ref:`C-005 `) is opt-in; manifest entries" - " without an ``integrity`` field are fetched without hash verification by default" + ( + "Integrity hash verification (:ref:`C-005 `) is opt-in; manifest entries" + " without an ``integrity`` field are fetched without hash verification by default" + ) ], status="partially-implemented", doc=SODocumentation( @@ -355,15 +378,19 @@ ecr_id="ecr-d", controls=["C-006", "C-036"], not_applicable=[ - "ACM-2, AUM-2, AUM-3, AUM-4, AUM-6 " - "(dfetch has no user-facing authentication or access control)" + ( + "ACM-2, AUM-2, AUM-3, AUM-4, AUM-6 " + "(dfetch has no user-facing authentication or access control)" + ) ], gaps=[ - "dfetch has no native authentication or authorisation layer; access control is " - "fully delegated to the underlying VCS server and host OS. C-006 prevents " - "interactive credential prompts, and C-036 strips credentials from persisted " - "metadata — both are confidentiality controls, not access-control mechanisms " - "in the authentication/authorisation sense" + ( + "dfetch has no native authentication or authorisation layer; access control is " + "fully delegated to the underlying VCS server and host OS. C-006 prevents " + "interactive credential prompts, and C-036 strips credentials from persisted " + "metadata — both are confidentiality controls, not access-control mechanisms " + "in the authentication/authorisation sense" + ) ], status="partially-implemented", doc=SODocumentation( @@ -449,9 +476,11 @@ ecr_id="ecr-e", controls=["C-045"], gaps=[ - "C-045 warns on plaintext-scheme URLs but does not refuse to proceed; " - "TLS/SSH confidentiality is provided by the underlying VCS client, not " - "enforced by dfetch itself" + ( + "C-045 warns on plaintext-scheme URLs but does not refuse to proceed; " + "TLS/SSH confidentiality is provided by the underlying VCS client, not " + "enforced by dfetch itself" + ) ], status="partially-implemented", doc=SODocumentation( @@ -479,10 +508,12 @@ ecr_id="ecr-e", controls=["C-045"], gaps=[ - "Server authentication (TLS certificate verification, SSH host-key checking) " - "is delegated to the OS trust store and VCS client; dfetch does not " - "independently authenticate remote endpoints and cannot enforce authenticated " - "channels when C-045's warning is overridden by the user" + ( + "Server authentication (TLS certificate verification, SSH host-key checking) " + "is delegated to the OS trust store and VCS client; dfetch does not " + "independently authenticate remote endpoints and cannot enforce authenticated " + "channels when C-045's warning is overridden by the user" + ) ], status="partially-implemented", doc=SODocumentation( @@ -564,9 +595,11 @@ ecr_id="ecr-f", controls=["C-005"], gaps=[ - "C-005 provides end-to-end hash verification for archive sources only (opt-in); " - "git and svn sources rely solely on VCS object integrity (SHA-1/SHA-256 object " - "model) and TLS/SSH channel integrity — no dfetch-level hash verification" + ( + "C-005 provides end-to-end hash verification for archive sources only (opt-in); " + "git and svn sources rely solely on VCS object integrity (SHA-1/SHA-256 object " + "model) and TLS/SSH channel integrity — no dfetch-level hash verification" + ) ], status="partially-implemented", doc=SODocumentation( @@ -663,13 +696,17 @@ ecr_id="ecr-i", controls=["C-001", "C-007"], not_applicable=[ - "TCM-1 (dfetch makes targeted VCS fetch requests; " - "no ambient outbound traffic to throttle)" + ( + "TCM-1 (dfetch makes targeted VCS fetch requests; " + "no ambient outbound traffic to throttle)" + ) ], gaps=[ - "Archive HTTP operations time out at 15 s (reachability) and 60 s (download) " - "via ``archive.py``; git and svn subprocess calls have no timeout and can " - "stall indefinitely" + ( + "Archive HTTP operations time out at 15 s (reachability) and 60 s (download) " + "via ``archive.py``; git and svn subprocess calls have no timeout and can " + "stall indefinitely" + ) ], status="partially-implemented", doc=SODocumentation( @@ -724,13 +761,17 @@ ecr_id="ecr-j", controls=["C-001", "C-003", "C-004", "C-007", "C-008"], not_applicable=[ - "GEC-2-j, GEC-3-j, GEC-4-j, GEC-5-j, GEC-7-j " - "(dfetch exposes no network services)" + ( + "GEC-2-j, GEC-3-j, GEC-4-j, GEC-5-j, GEC-7-j " + "(dfetch exposes no network services)" + ) ], gaps=[ - "No domain or URL-scheme allowlist constrains which remote URLs the manifest " - "may reference; git and svn subprocess calls have no timeout (archive HTTP " - "operations time out at 15 s / 60 s)" + ( + "No domain or URL-scheme allowlist constrains which remote URLs the manifest " + "may reference; git and svn subprocess calls have no timeout (archive HTTP " + "operations time out at 15 s / 60 s)" + ) ], status="partially-implemented", doc=SODocumentation( @@ -785,11 +826,13 @@ ecr_id="ecr-l", controls=[], gaps=[ - "No persistent structured security event log (LGM-1/2/3/4 gap). dfetch prints " - "operational output to stderr but does not retain it, does not record which " - "credentials were used, which files were modified, or when remote access occurred. " - "C-036 ensures credentials are excluded from operational output but is not a " - "logging control" + ( + "No persistent structured security event log (LGM-1/2/3/4 gap). dfetch prints " + "operational output to stderr but does not retain it, does not record which " + "credentials were used, which files were modified, or when remote access occurred. " + "C-036 ensures credentials are excluded from operational output but is not a " + "logging control" + ) ], status="partially-implemented", doc=SODocumentation( @@ -846,12 +889,14 @@ so_id="so-secure-data-deletion", ecr_id="ecr-m", not_applicable=[ - "No dfetch-specific secure-deletion control is required: dfetch stores " - "no personal data and no keying material. The only on-disk state is " - ".dfetch_data.yaml (non-sensitive metadata) and vendored source files " - "(third-party code, not user data). Standard OS file deletion (rm / del) " - "is sufficient; cryptographic wipe is not warranted. DLM-1 is satisfied " - "by design (no sensitive data to wipe), not by a dedicated dfetch control." + ( + "No dfetch-specific secure-deletion control is required: dfetch stores " + "no personal data and no keying material. The only on-disk state is " + ".dfetch_data.yaml (non-sensitive metadata) and vendored source files " + "(third-party code, not user data). Standard OS file deletion (rm / del) " + "is sufficient; cryptographic wipe is not warranted. DLM-1 is satisfied " + "by design (no sensitive data to wipe), not by a dedicated dfetch control." + ) ], status="implemented", doc=SODocumentation( @@ -904,60 +949,3 @@ ), ), ] - -# ── CRA Part II requirements (prEN 40000-1-3) ──────────────────────────────── - -PART_II_REQUIREMENTS: list[PartIIRequirement] = [ - PartIIRequirement( - id="pii-01", - ref="Part II §1", - text="Identify and document vulnerabilities and components (SBOM).", - controls=["C-021", "C-022"], - status="implemented", - ), - PartIIRequirement( - id="pii-02", - ref="Part II §2", - text="Address vulnerabilities without delay; provide free security updates.", - controls=["C-015", "C-016", "SECURITY.md"], - gaps=[ - "No LTS backport policy (latest release only — documented in SECURITY.md)" - ], - status="partially-implemented", - ), - PartIIRequirement( - id="pii-03", - ref="Part II §3", - text="Apply effective coordinated vulnerability disclosure (CVD) policy.", - controls=["SECURITY.md"], - status="implemented", - ), - PartIIRequirement( - id="pii-04", - ref="Part II §4", - text="Report actively exploited vulnerabilities to national CSIRT and ENISA.", - status="not-applicable", - ), - PartIIRequirement( - id="pii-05", - ref="Part II §5", - text="Publish coordinated vulnerability disclosure policy.", - controls=["SECURITY.md"], - status="implemented", - ), - PartIIRequirement( - id="pii-06", - ref="Part II §6", - text="Share information on vulnerabilities in integrated components.", - controls=["C-022", "C-016"], - gaps=["No proactive downstream notification process"], - status="partially-implemented", - ), - PartIIRequirement( - id="pii-07", - ref="Part II §7", - text="Provide security updates free of charge for the support period.", - controls=["MIT licence", "PyPI", "SECURITY.md"], - status="implemented", - ), -] diff --git a/security/compliance_part_ii_data.py b/security/compliance_part_ii_data.py new file mode 100644 index 00000000..92699ae8 --- /dev/null +++ b/security/compliance_part_ii_data.py @@ -0,0 +1,61 @@ +"""Static compliance data: CRA Part II software requirements (prEN 40000-1-3). + +Split out of compliance_data.py to stay within the 1000-line limit per file. +""" + +from security.compliance_types import PartIIRequirement + +PART_II_REQUIREMENTS: list[PartIIRequirement] = [ + PartIIRequirement( + id="pii-01", + ref="Part II §1", + text="Identify and document vulnerabilities and components (SBOM).", + controls=["C-021", "C-022"], + status="implemented", + ), + PartIIRequirement( + id="pii-02", + ref="Part II §2", + text="Address vulnerabilities without delay; provide free security updates.", + controls=["C-015", "C-016", "SECURITY.md"], + gaps=[ + "No LTS backport policy (latest release only — documented in SECURITY.md)" + ], + status="partially-implemented", + ), + PartIIRequirement( + id="pii-03", + ref="Part II §3", + text="Apply effective coordinated vulnerability disclosure (CVD) policy.", + controls=["SECURITY.md"], + status="implemented", + ), + PartIIRequirement( + id="pii-04", + ref="Part II §4", + text="Report actively exploited vulnerabilities to national CSIRT and ENISA.", + status="not-applicable", + ), + PartIIRequirement( + id="pii-05", + ref="Part II §5", + text="Publish coordinated vulnerability disclosure policy.", + controls=["SECURITY.md"], + status="implemented", + ), + PartIIRequirement( + id="pii-06", + ref="Part II §6", + text="Share information on vulnerabilities in integrated components.", + controls=["C-022", "C-016"], + gaps=["No proactive downstream notification process"], + status="partially-implemented", + ), + PartIIRequirement( + id="pii-07", + ref="Part II §7", + text="Provide security updates free of charge for the support period.", + controls=["MIT licence", "PyPI", "SECURITY.md"], + status="implemented", + ), +] diff --git a/security/tm_elements.py b/security/tm_elements.py index 3b09bf7c..73a2fa3f 100644 --- a/security/tm_elements.py +++ b/security/tm_elements.py @@ -11,7 +11,7 @@ from pytm import Assumption, Boundary -from security.tm_controls_data import Control # noqa: F401 # re-exported +from security.tm_controls_data import Control # re-exported THREATS_FILE = os.path.join(os.path.dirname(__file__), "threats.json") diff --git a/security/tm_render.py b/security/tm_render.py index d98a8a9b..97856198 100644 --- a/security/tm_render.py +++ b/security/tm_render.py @@ -111,7 +111,7 @@ def _render_asset_rows( The highest risk across all matching findings/controls is used per dimension. """ assets = sorted( - list(getattr(TM, "_assets")) + list(getattr(TM, "_data", [])), + list(getattr(TM, "_assets", [])) + list(getattr(TM, "_data", [])), key=lambda e: getattr(e, "name", ""), ) if not assets: diff --git a/security/tm_supply_chain.py b/security/tm_supply_chain.py index 540e1842..e5765e83 100644 --- a/security/tm_supply_chain.py +++ b/security/tm_supply_chain.py @@ -21,7 +21,7 @@ sys.path.insert(0, _repo_root) # pylint: disable=wrong-import-position -from pytm import ( # noqa: E402 +from pytm import ( TM, Actor, Boundary, @@ -33,8 +33,8 @@ Process, ) -from security.tm_controls_data import SC_CONTROLS as CONTROLS # noqa: E402 -from security.tm_elements import ( # noqa: E402 +from security.tm_controls_data import SC_CONTROLS as CONTROLS +from security.tm_elements import ( THREATS_FILE, Control, ThreatResponse, @@ -42,7 +42,7 @@ make_dev_env_boundary, make_supply_chain_assumptions, ) -from security.tm_render import apply_report_utils_patch, run_model # noqa: E402 +from security.tm_render import apply_report_utils_patch, run_model # pylint: enable=wrong-import-position diff --git a/security/tm_usage.py b/security/tm_usage.py index 31e1ec3c..0b019754 100644 --- a/security/tm_usage.py +++ b/security/tm_usage.py @@ -33,7 +33,7 @@ sys.path.insert(0, _repo_root) # pylint: disable=wrong-import-position -from pytm import ( # noqa: E402 +from pytm import ( TM, Actor, Boundary, @@ -45,8 +45,8 @@ Process, ) -from security.tm_controls_data import USAGE_CONTROLS as CONTROLS # noqa: E402 -from security.tm_elements import ( # noqa: E402 +from security.tm_controls_data import USAGE_CONTROLS as CONTROLS +from security.tm_elements import ( THREATS_FILE, Control, ThreatResponse, @@ -55,7 +55,7 @@ make_network_boundary, make_usage_assumptions, ) -from security.tm_render import apply_report_utils_patch, run_model # noqa: E402 +from security.tm_render import apply_report_utils_patch, run_model # pylint: enable=wrong-import-position diff --git a/stubs/py_serializable/__init__.pyi b/stubs/py_serializable/__init__.pyi index ec55f39d..bb518776 100644 --- a/stubs/py_serializable/__init__.pyi +++ b/stubs/py_serializable/__init__.pyi @@ -9,43 +9,44 @@ at runtime. """ from collections.abc import Callable, Iterable -from typing import Any, Optional, TypeVar, overload +from typing import Any, TypeVar, overload + +from typing_extensions import Self _T = TypeVar("_T") class _Serializable: def as_json(self) -> str: ... @classmethod - def from_json(cls: type[_T], data: dict[str, Any]) -> Optional[_T]: ... + def from_json(cls, data: dict[str, Any]) -> Self | None: ... def as_xml(self) -> Any: ... @classmethod - def from_xml(cls: type[_T], data: Any) -> Optional[_T]: ... + def from_xml(cls, data: Any) -> Self | None: ... @overload def serializable_class( cls: None = ..., *, - name: Optional[str] = ..., - serialization_types: Optional[Iterable[Any]] = ..., - ignore_during_deserialization: Optional[Iterable[str]] = ..., + name: str | None = ..., + serialization_types: Iterable[Any] | None = ..., + ignore_during_deserialization: Iterable[str] | None = ..., ignore_unknown_during_deserialization: bool = ..., ) -> Callable[[type[_T]], type[_T]]: ... @overload def serializable_class( cls: type[_T], *, - name: Optional[str] = ..., - serialization_types: Optional[Iterable[Any]] = ..., - ignore_during_deserialization: Optional[Iterable[str]] = ..., + name: str | None = ..., + serialization_types: Iterable[Any] | None = ..., + ignore_during_deserialization: Iterable[str] | None = ..., ignore_unknown_during_deserialization: bool = ..., ) -> type[_T]: ... def serializable_class(cls: Any = None, **kwargs: Any) -> Any: ... - def xml_name(name: str) -> Callable[[_T], _T]: ... def xml_sequence(order: int) -> Callable[[_T], _T]: ... def xml_array( array_type: Any, - child_name: Optional[str] = ..., + child_name: str | None = ..., ) -> Callable[[_T], _T]: ... def xml_attribute() -> Callable[[_T], _T]: ... def xml_string(string_type: Any) -> Callable[[_T], _T]: ... diff --git a/tests/test_fuzzing.py b/tests/test_fuzzing.py index 777e9e1d..812a01cf 100644 --- a/tests/test_fuzzing.py +++ b/tests/test_fuzzing.py @@ -169,23 +169,27 @@ def test_manifest_can_be_created(data): @given(manifest_strategy) def test_check(data): """Validate check command.""" - with suppress(DfetchFatalException): - with tempfile.TemporaryDirectory() as tmpdir: - with in_directory(tmpdir): - with open("dfetch.yaml", "w", encoding="UTF-8") as manifest_file: - yaml.dump(data, manifest_file) - run(["check"]) + with ( + suppress(DfetchFatalException), + tempfile.TemporaryDirectory() as tmpdir, + in_directory(tmpdir), + ): + with open("dfetch.yaml", "w", encoding="UTF-8") as manifest_file: + yaml.dump(data, manifest_file) + run(["check"]) @given(manifest_strategy) def test_update(data): """Validate update command.""" - with suppress(DfetchFatalException): - with tempfile.TemporaryDirectory() as tmpdir: - with in_directory(tmpdir): - with open("dfetch.yaml", "w", encoding="UTF-8") as manifest_file: - yaml.dump(data, manifest_file) - run(["update"]) + with ( + suppress(DfetchFatalException), + tempfile.TemporaryDirectory() as tmpdir, + in_directory(tmpdir), + ): + with open("dfetch.yaml", "w", encoding="UTF-8") as manifest_file: + yaml.dump(data, manifest_file) + run(["update"]) if __name__ == "__main__": diff --git a/tests/test_tree_browser.py b/tests/test_tree_browser.py index 8744a9f0..69437e48 100644 --- a/tests/test_tree_browser.py +++ b/tests/test_tree_browser.py @@ -153,10 +153,12 @@ def _browser( *, idx: int = 0, top: int = 0, - config: BrowserConfig = BrowserConfig(), + config: BrowserConfig | None = None, children: list[Entry] | None = None, ) -> _HeadlessBrowser: """Return a seeded _HeadlessBrowser backed by *children* (or empty) for expansions.""" + if config is None: + config = BrowserConfig() dir_path = nodes[0].path if nodes else "" ls = ( (lambda path: children if path == dir_path else [])