From 7a832d8dd665c6befc2886db8bcc80c8f38ce4a0 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 9 Jul 2026 10:48:39 +0200 Subject: [PATCH 01/24] Fix typos in changelogs --- doc/changes/changes_10.0.0.md | 4 ++-- doc/changes/changes_10.2.0.md | 2 +- doc/changes/changes_10.2.1.md | 2 -- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/doc/changes/changes_10.0.0.md b/doc/changes/changes_10.0.0.md index 216a0a2ed..d8ba3cd22 100644 --- a/doc/changes/changes_10.0.0.md +++ b/doc/changes/changes_10.0.0.md @@ -48,7 +48,7 @@ This release fixes vulnerabilities by updating dependencies: * #699: Added `all-extras` support to the Python environment GitHub action * #875: Added `name` attribute to generated workflow jobs using `-extension.yml` workflows -## Bug +## Bug Fix * #744: Updated nox DB-version handling to use `BaseConfig.minimum_exasol_version` instead hardcoded `7.1.9` @@ -80,4 +80,4 @@ This release fixes vulnerabilities by updating dependencies: * Updated dependency `pip-audit:2.10.0` to `2.10.1` * Updated dependency `pylint:4.0.5` to `4.0.6` * Updated dependency `pytest:9.0.3` to `9.1.1` -* Updated dependency `zizmor:1.25.2` to `1.26.1` \ No newline at end of file +* Updated dependency `zizmor:1.25.2` to `1.26.1` diff --git a/doc/changes/changes_10.2.0.md b/doc/changes/changes_10.2.0.md index a8848b691..e00721c99 100644 --- a/doc/changes/changes_10.2.0.md +++ b/doc/changes/changes_10.2.0.md @@ -6,7 +6,7 @@ This minor release adds automated vulnerability updates through Nox session `vulnerabilities:update` and improves the `dependency-update.yml`. It also includes a few workflow-related bug fixes and documentation updates. -## Bug +## Bug Fix * #909: Updated `cd.yml` workflow so that `cd-extension.yml` workflow depends on `build-and-publish`. This ensures that the custom release workflow only runs when the PyPi release was successful. * #910: Added `gh-pages.yml` to be ignored when `has_documentation=False` in the `PROJECT_CONFIG` diff --git a/doc/changes/changes_10.2.1.md b/doc/changes/changes_10.2.1.md index 98cdcb506..c29ae2b39 100644 --- a/doc/changes/changes_10.2.1.md +++ b/doc/changes/changes_10.2.1.md @@ -1,7 +1,5 @@ # 10.2.1 - 2026-07-08 -## Summary - ## Bug Fix * #920: Ensured extracted secrets are unique and alphabetically sorted from the custom workflows From c37d96c14555b69c336a006c4b5d6c0ebc4ebfeb Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 9 Jul 2026 11:20:27 +0200 Subject: [PATCH 02/24] Make permissions block present in workflows that call the custom workflows --- exasol/toolbox/templates/github/workflows/cd.yml | 6 +++++- exasol/toolbox/templates/github/workflows/ci.yml | 6 +++++- .../templates/github/workflows/fast-tests.yml | 6 +++++- .../templates/github/workflows/merge-gate.yml | 12 ++++++++++-- .../github/workflows/periodic-validation.yml | 6 +++++- 5 files changed, 30 insertions(+), 6 deletions(-) diff --git a/exasol/toolbox/templates/github/workflows/cd.yml b/exasol/toolbox/templates/github/workflows/cd.yml index 58eceed90..a856759cf 100644 --- a/exasol/toolbox/templates/github/workflows/cd.yml +++ b/exasol/toolbox/templates/github/workflows/cd.yml @@ -38,8 +38,12 @@ jobs: (( secret_name )): ${{ secrets.(( secret_name )) }} (% endfor %) (% endif %) + (% if custom_workflows["cd-extension"].permissions %) permissions: - contents: write + (% for permission_name, permission_level in custom_workflows["cd-extension"].permissions.items() %) + (( permission_name )): (( permission_level )) + (% endfor %) + (% endif %) (% endif %) (% if has_documentation %) diff --git a/exasol/toolbox/templates/github/workflows/ci.yml b/exasol/toolbox/templates/github/workflows/ci.yml index c3c095328..da524c3a7 100644 --- a/exasol/toolbox/templates/github/workflows/ci.yml +++ b/exasol/toolbox/templates/github/workflows/ci.yml @@ -15,8 +15,12 @@ jobs: (( secret_name )): ${{ secrets.(( secret_name )) }} (% endfor %) (% endif %) + (% if custom_workflows["merge-gate"].permissions %) permissions: - contents: read + (% for permission_name, permission_level in custom_workflows["merge-gate"].permissions.items() %) + (( permission_name )): (( permission_level )) + (% endfor %) + (% endif %) report: name: Report diff --git a/exasol/toolbox/templates/github/workflows/fast-tests.yml b/exasol/toolbox/templates/github/workflows/fast-tests.yml index 0d4156809..c0151676a 100644 --- a/exasol/toolbox/templates/github/workflows/fast-tests.yml +++ b/exasol/toolbox/templates/github/workflows/fast-tests.yml @@ -52,6 +52,10 @@ jobs: (( secret_name )): ${{ secrets.(( secret_name )) }} (% endfor %) (% endif %) + (% if custom_workflows["fast-tests-extension"].permissions %) permissions: - contents: read + (% for permission_name, permission_level in custom_workflows["fast-tests-extension"].permissions.items() %) + (( permission_name )): (( permission_level )) + (% endfor %) + (% endif %) (% endif %) diff --git a/exasol/toolbox/templates/github/workflows/merge-gate.yml b/exasol/toolbox/templates/github/workflows/merge-gate.yml index d68adf12c..8ee19db55 100644 --- a/exasol/toolbox/templates/github/workflows/merge-gate.yml +++ b/exasol/toolbox/templates/github/workflows/merge-gate.yml @@ -59,8 +59,12 @@ jobs: (( secret_name )): ${{ secrets.(( secret_name )) }} (% endfor %) (% endif %) + (% if custom_workflows["slow-checks"].permissions %) permissions: - contents: read + (% for permission_name, permission_level in custom_workflows["slow-checks"].permissions.items() %) + (( permission_name )): (( permission_level )) + (% endfor %) + (% endif %) (% if custom_workflows["merge-gate-extension"].exists %) merge-gate-extension: @@ -72,8 +76,12 @@ jobs: (( secret_name )): ${{ secrets.(( secret_name )) }} (% endfor %) (% endif %) + (% if custom_workflows["merge-gate-extension"].permissions %) permissions: - contents: read + (% for permission_name, permission_level in custom_workflows["merge-gate-extension"].permissions.items() %) + (( permission_name )): (( permission_level )) + (% endfor %) + (% endif %) (% endif %) # This job ensures inputs have been executed successfully. diff --git a/exasol/toolbox/templates/github/workflows/periodic-validation.yml b/exasol/toolbox/templates/github/workflows/periodic-validation.yml index 017f1c0fb..ae3adaa43 100644 --- a/exasol/toolbox/templates/github/workflows/periodic-validation.yml +++ b/exasol/toolbox/templates/github/workflows/periodic-validation.yml @@ -51,8 +51,12 @@ jobs: (( secret_name )): ${{ secrets.(( secret_name )) }} (% endfor %) (% endif %) + (% if custom_workflows["slow-checks"].permissions %) permissions: - contents: read + (% for permission_name, permission_level in custom_workflows["slow-checks"].permissions.items() %) + (( permission_name )): (( permission_level )) + (% endfor %) + (% endif %) report: name: Report From 5d7f0ed6090655fff96483c047fc6f6ecd8736c0 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 9 Jul 2026 11:23:25 +0200 Subject: [PATCH 03/24] Add documentation for new feature --- .../github_workflows/workflow_variables.rst | 48 +++++++++++++++++-- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/doc/user_guide/features/github_workflows/workflow_variables.rst b/doc/user_guide/features/github_workflows/workflow_variables.rst index 01f0ff384..fb6b8e8e5 100644 --- a/doc/user_guide/features/github_workflows/workflow_variables.rst +++ b/doc/user_guide/features/github_workflows/workflow_variables.rst @@ -19,10 +19,20 @@ standardized baseline that can be overridden in individual projects. :start-at: github_template_dict :end-before: @computed_field +.. _custom_workflow_metadata: + +Custom Workflow Metadata +^^^^^^^^^^^^^^^^^^^^^^^^ + +The PTB extracts metadata from reusable custom workflow files and exposes it +through :py:attr:`exasol.toolbox.config.BaseConfig.github_template_dict` under the +``custom_workflows`` entry. PTB-controlled workflow templates use that metadata +when they call reusable workflows. + .. _custom_workflow_secrets: -Custom Workflow Secrets -^^^^^^^^^^^^^^^^^^^^^^^ +Secrets +------- The PTB extracts secret names from reusable custom workflow files and exposes them through :py:attr:`exasol.toolbox.config.BaseConfig.github_template_dict` under the @@ -50,6 +60,38 @@ For example, ``slow-checks.yml`` can define its reusable workflow header like th Those extracted secret names are then made available to the PTB templates that reference the custom workflow. +.. _custom_workflow_permissions: + +Permissions +----------- + +The PTB extracts the effective GitHub permissions required by reusable custom +workflow files. It scans every job in the workflow, reads each job's +``permissions`` block, and combines the results into a single ordered mapping. + +When multiple jobs request the same permission, the most permissive level wins +while the first-seen key order is preserved. For example, if one job requests +``contents: read`` and another later job requests ``contents: write``, the final +mapping keeps ``contents`` once, with ``write`` as the level. + +Please only configure the minimum required permissions and granting the least required +access. In practice, ``contents: read`` is the most common baseline for workflows, and +other permissions should only be added when a particular step truly requires them. + +For example, a custom workflow can declare permissions like this: + +.. code-block:: yaml + + name: Slow-Checks + + on: + workflow_call: + + jobs: + run-integration-tests: + permissions: + contents: read + .. _workflow_matrix: Matrix Combinations @@ -64,7 +106,7 @@ Extending the Matrix ^^^^^^^^^^^^^^^^^^^^ If you need to expose additional values via the ``matrix.yml``, you can extend -:class:`exasol.toolbox.config.BaseConfig`. +:class:`exasol.toolbox.config.BaseConfig`. The example adds two additional matrix dimensions: A declared one `extra_matrix_value` and a computed one `computed_matrix_value`. Each of them From 620fde3e54e5e01e412b2de4f60a8e7b0b14a88e Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 9 Jul 2026 11:46:04 +0200 Subject: [PATCH 04/24] Add jinja test --- test/unit/util/workflows/render_yaml_test.py | 88 ++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/test/unit/util/workflows/render_yaml_test.py b/test/unit/util/workflows/render_yaml_test.py index 7f0606f6f..61611c05e 100644 --- a/test/unit/util/workflows/render_yaml_test.py +++ b/test/unit/util/workflows/render_yaml_test.py @@ -521,6 +521,94 @@ def test_includes_extension_with_multiple_secrets(test_yml, project_config): yaml_dict = yaml_renderer.get_yaml_dict() assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) + @staticmethod + def test_includes_extension_with_custom_permissions(test_yml, project_config): + input_yaml = """ + jobs: + run-unit-tests: + name: Unit Tests (Python-${{ matrix.python-versions }}) + runs-on: "(( os_version ))" + permissions: + contents: read + strategy: + fail-fast: false + matrix: + python-versions: (( python_versions | tojson )) + + steps: + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + + (% if custom_workflows["fast-tests-extension"].exists %) + fast-tests-extension: + name: Extension + uses: ./.github/workflows/fast-tests-extension.yml + (% if custom_workflows["fast-tests-extension"].permissions %) + permissions: + (% for permission_name, permission_level in custom_workflows["fast-tests-extension"].permissions.items() %) + (( permission_name )): (( permission_level )) + (% endfor %) + (% endif %) + (% endif %) + + """ + expected_yaml = """ + jobs: + run-unit-tests: + name: Unit Tests (Python-${{ matrix.python-versions }}) + runs-on: "ubuntu-24.04" + permissions: + contents: read + strategy: + fail-fast: false + matrix: + python-versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] + + steps: + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + + fast-tests-extension: + name: Extension + uses: ./.github/workflows/fast-tests-extension.yml + permissions: + contents: write + packages: read + id-token: write + + """ + workflow_directory = project_config.github_workflow_directory + workflow_directory.mkdir(parents=True, exist_ok=True) + (workflow_directory / "fast-tests-extension.yml").write_text(cleandoc(""" + name: Fast-Tests-Extension + + on: + workflow_call: + + jobs: + build-artifacts: + permissions: + contents: read + packages: read + publish-artifacts: + permissions: + contents: write + id-token: write + """)) + + content = cleandoc(input_yaml) + test_yml.write_text(content) + + yaml_renderer = YamlRenderer( + github_template_dict=project_config.github_template_dict, + file_path=test_yml, + ) + + yaml_dict = yaml_renderer.get_yaml_dict() + assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) + @staticmethod def test_jinja_variable_unknown(test_yml, yaml_renderer): input_yaml = """ From 24cfb7bbbd5301c4e0bdd3fd24698f32282e070d Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 9 Jul 2026 11:47:46 +0200 Subject: [PATCH 05/24] Prepare config test --- test/unit/config_test.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/test/unit/config_test.py b/test/unit/config_test.py index 580b816cb..5fa06445b 100644 --- a/test/unit/config_test.py +++ b/test/unit/config_test.py @@ -51,11 +51,31 @@ def test_works_as_defined(tmp_path, test_project_config_factory): "github_workflow_patcher_yaml": None, "github_template_dict": { "custom_workflows": { - "cd-extension": {"exists": False, "secrets": ()}, - "fast-tests-extension": {"exists": False, "secrets": ()}, - "merge-gate-extension": {"exists": False, "secrets": ()}, - "slow-checks": {"exists": False, "secrets": ()}, - "merge-gate": {"exists": True, "secrets": ("SONAR_TOKEN",)}, + "cd-extension": { + "exists": False, + "secrets": (), + "permissions": {}, + }, + "fast-tests-extension": { + "exists": False, + "secrets": (), + "permissions": {}, + }, + "merge-gate-extension": { + "exists": False, + "secrets": (), + "permissions": {}, + }, + "slow-checks": { + "exists": False, + "secrets": (), + "permissions": {}, + }, + "merge-gate": { + "exists": True, + "secrets": ("SONAR_TOKEN",), + "permissions": {"contents": "read"}, + }, }, "dependency_manager_version": "2.3.0", "has_documentation": True, From bea075f236bf2d2bd6b4f5cdc4e59c14168d7487 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 9 Jul 2026 11:48:47 +0200 Subject: [PATCH 06/24] Add extract_permissions --- doc/changes/unreleased.md | 4 + .../toolbox/util/workflows/custom_workflow.py | 59 ++++++++- .../workflows/custom_workflow_extractor.py | 28 ++++- .../custom_workflow_extractor_test.py | 116 +++++++++++++++++- .../util/workflows/custom_workflow_test.py | 72 +++++++++++ 5 files changed, 271 insertions(+), 8 deletions(-) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index fb4737052..89e85ab78 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -1,3 +1,7 @@ # Unreleased ## Summary + +## Feature + +* #922: Extended `custom_workflows` of `github_template_dict` for automatic custom workflow permissions extraction diff --git a/exasol/toolbox/util/workflows/custom_workflow.py b/exasol/toolbox/util/workflows/custom_workflow.py index 49c375b61..941142e0e 100644 --- a/exasol/toolbox/util/workflows/custom_workflow.py +++ b/exasol/toolbox/util/workflows/custom_workflow.py @@ -1,5 +1,6 @@ from __future__ import annotations +from enum import IntEnum from pathlib import Path from pydantic import ( @@ -13,12 +14,43 @@ from exasol.toolbox.util.workflows.render_yaml import parse_yaml_text +class PermissionRank(IntEnum): + """GitHub permission levels ranked from least to most permissive.""" + + NONE = 0 + READ = 1 + WRITE = 2 + + @classmethod + def _missing_(cls, value: object) -> PermissionRank: + """Convert a GitHub permission string to its rank.""" + if isinstance(value, str): + for rank in cls: + if rank.name.lower() == value: + return rank + raise ValueError(f"Unknown GitHub permission level: {value!r}") + + +def merge_permissions(permission_maps: list[dict[str, str]]) -> dict[str, str]: + """Merge permission maps to keep the greater permission.""" + merged_permissions: dict[str, str] = {} + for permission_map in permission_maps: + for permission_name, requested_level in permission_map.items(): + current_level = merged_permissions.get(permission_name, None) + if current_level is None: + merged_permissions[permission_name] = requested_level + continue + if PermissionRank(requested_level) > PermissionRank(current_level): # type: ignore[arg-type] + merged_permissions[permission_name] = requested_level + return merged_permissions + + class CustomWorkflow(BaseModel): """A project-owned workflow used for seeded workflows and extensions. These workflows are seeded by the PTB or extend PTB-provided workflows, but they are maintained by the project itself rather than the PTB. See - `Custom Workflows `__. + `Custom Workflows `__. """ model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) @@ -53,3 +85,28 @@ def extract_secrets(self) -> tuple[str, ...]: if secrets := workflow_call.get("secrets", {}): return tuple(secrets.keys()) return () + + def extract_permissions(self) -> dict[str, str]: + """Return the effective job permissions required by the workflow. + + The extractor scans all jobs and merges their ``permissions`` blocks into a + single mapping. When the same permission appears multiple times, the more + permissive level wins while preserving the first-seen order of the keys. + + For example, a custom workflow can declare permissions like this: + + .. code-block:: yaml + + name: Slow-Checks + + on: + workflow_call: + + jobs: + run-integration-tests: + permissions: + contents: read + """ + jobs = self.yaml_content.get("jobs", {}) + permission_maps = [job.get("permissions", {}) for job in jobs.values()] + return merge_permissions(permission_maps) diff --git a/exasol/toolbox/util/workflows/custom_workflow_extractor.py b/exasol/toolbox/util/workflows/custom_workflow_extractor.py index aeab01b38..8637fd7e3 100644 --- a/exasol/toolbox/util/workflows/custom_workflow_extractor.py +++ b/exasol/toolbox/util/workflows/custom_workflow_extractor.py @@ -8,7 +8,12 @@ field_validator, ) -from exasol.toolbox.util.workflows.custom_workflow import CustomWorkflow +from exasol.toolbox.util.workflows.custom_workflow import ( + CustomWorkflow, + merge_permissions, +) + +MINIMUM_GITHUB_PERMISSIONS: dict[str, str] = {"contents": "read"} class CustomWorkflowEntry(BaseModel): @@ -16,6 +21,7 @@ class CustomWorkflowEntry(BaseModel): exists: bool secrets: tuple[str, ...] + permissions: dict[str, str] @field_validator("secrets", mode="before") @classmethod @@ -23,6 +29,14 @@ def _normalize_secrets(cls, secrets: tuple[str, ...]) -> list[str]: """Return unique secret names in alphabetical order.""" return sorted(set(secrets)) + @field_validator("permissions", mode="before") + @classmethod + def _normalize_permissions( + cls, permissions: list[dict[str, str]] + ) -> dict[str, str]: + """Merge permission maps while preserving first-seen order.""" + return merge_permissions(permissions) + class CustomWorkflowExtractor(BaseModel): model_config = ConfigDict(frozen=True) @@ -43,13 +57,18 @@ def _build_custom_workflow_entry( file_path = self.github_workflow_directory / f"{workflow}.yml" secrets: tuple[str, ...] = () + permissions: list[dict[str, str]] = [] if file_path.is_file(): custom_workflow = CustomWorkflow.load_from_file(file_path=file_path) secrets = custom_workflow.extract_secrets() - + permissions = [ + MINIMUM_GITHUB_PERMISSIONS, + custom_workflow.extract_permissions(), + ] return CustomWorkflowEntry( exists=file_path.exists(), secrets=secrets, + permissions=permissions, ) def _build_merge_gate_entry( @@ -61,6 +80,11 @@ def _build_merge_gate_entry( + custom_workflows_dict["slow-checks"].secrets # from the `report.yml` + (self.sonar_token_name,), + permissions=[ + MINIMUM_GITHUB_PERMISSIONS, + custom_workflows_dict["merge-gate-extension"].permissions, + custom_workflows_dict["slow-checks"].permissions, + ], ) def build_custom_workflow_dict( diff --git a/test/unit/util/workflows/custom_workflow_extractor_test.py b/test/unit/util/workflows/custom_workflow_extractor_test.py index 608fa15c3..7295c1508 100644 --- a/test/unit/util/workflows/custom_workflow_extractor_test.py +++ b/test/unit/util/workflows/custom_workflow_extractor_test.py @@ -29,11 +29,13 @@ def test_secrets_are_normalized_on_creation(): custom_workflow_entry = CustomWorkflowEntry( exists=True, secrets=("ZETA_SECRET", "ALPHA_SECRET", "ALPHA_SECRET"), + permissions=[{"contents": "read"}], ) assert custom_workflow_entry == CustomWorkflowEntry( exists=True, secrets=("ALPHA_SECRET", "ZETA_SECRET"), + permissions=[{"contents": "read"}], ) @@ -47,6 +49,7 @@ def test_file_does_not_exist(custom_workflow_extractor): assert custom_workflow_entry == CustomWorkflowEntry( exists=False, secrets=(), + permissions=[], ) @staticmethod @@ -69,6 +72,7 @@ def test_file_does_not_contain_secrets( assert custom_workflow_entry == CustomWorkflowEntry( exists=True, secrets=(), + permissions=[{"contents": "read"}], ) @staticmethod @@ -92,6 +96,44 @@ def test_file_contains_secret(custom_workflow_extractor, workflow_directory): assert custom_workflow_entry == CustomWorkflowEntry( exists=True, secrets=("SLOW_CHECK_SECRET",), + permissions=[{"contents": "read"}], + ) + + @staticmethod + def test_file_contains_permissions(custom_workflow_extractor, workflow_directory): + workflow = "slow-checks" + workflow_path = workflow_directory / f"{workflow}.yml" + workflow_path.write_text(cleandoc(""" + name: Slow-Checks + + on: + workflow_call: + + jobs: + run-integration-tests: + permissions: + contents: read + packages: write + publish-results: + permissions: + contents: write + id-token: write + """)) + + custom_workflow_entry = custom_workflow_extractor._build_custom_workflow_entry( + workflow=workflow + ) + + assert custom_workflow_entry == CustomWorkflowEntry( + exists=True, + secrets=(), + permissions=[ + { + "contents": "write", + "packages": "write", + "id-token": "write", + } + ], ) @@ -102,10 +144,22 @@ def test_build_merge_gate_entry(custom_workflow_extractor): "merge-gate-extension": CustomWorkflowEntry( exists=True, secrets=("EXT_SECRET",), + permissions=[ + { + "contents": "read", + "packages": "read", + } + ], ), "slow-checks": CustomWorkflowEntry( exists=True, secrets=("SLOW_SECRET",), + permissions=[ + { + "packages": "write", + "id-token": "write", + } + ], ), } @@ -116,16 +170,31 @@ def test_build_merge_gate_entry(custom_workflow_extractor): assert merge_gate_entry == CustomWorkflowEntry( exists=True, secrets=("EXT_SECRET", "SLOW_SECRET", "SONAR_TOKEN"), + permissions=[ + { + "contents": "read", + "packages": "write", + "id-token": "write", + } + ], ) class TestBuildCustomWorkflowDict: default_custom_workflow_dict = { - "cd-extension": CustomWorkflowEntry(exists=False, secrets=()), - "fast-tests-extension": CustomWorkflowEntry(exists=False, secrets=()), - "merge-gate-extension": CustomWorkflowEntry(exists=False, secrets=()), - "slow-checks": CustomWorkflowEntry(exists=False, secrets=()), - "merge-gate": CustomWorkflowEntry(exists=True, secrets=("SONAR_TOKEN",)), + "cd-extension": CustomWorkflowEntry(exists=False, secrets=(), permissions=[]), + "fast-tests-extension": CustomWorkflowEntry( + exists=False, secrets=(), permissions=[] + ), + "merge-gate-extension": CustomWorkflowEntry( + exists=False, secrets=(), permissions=[] + ), + "slow-checks": CustomWorkflowEntry(exists=False, secrets=(), permissions=[]), + "merge-gate": CustomWorkflowEntry( + exists=True, + secrets=("SONAR_TOKEN",), + permissions=[{"contents": "read"}], + ), } def test_no_custom_workflows_exist(self, custom_workflow_extractor): @@ -166,6 +235,7 @@ def test_custom_workflow_written_to( expected_custom_workflow_dict[workflow] = CustomWorkflowEntry( exists=True, secrets=(secret,), + permissions=[{"contents": "read"}], ) if workflow in ("merge-gate-extension", "slow-checks"): expected_custom_workflow_dict["merge-gate"] = CustomWorkflowEntry( @@ -174,6 +244,42 @@ def test_custom_workflow_written_to( secret, "SONAR_TOKEN", ), + permissions=[{"contents": "read"}], ) assert custom_workflow_dict == expected_custom_workflow_dict + + @staticmethod + def test_merge_gate_permissions_are_aggregated_in_order(custom_workflow_extractor): + custom_workflows_dict = { + "merge-gate-extension": CustomWorkflowEntry( + exists=True, + secrets=(), + permissions=[ + { + "contents": "read", + "packages": "read", + } + ], + ), + "slow-checks": CustomWorkflowEntry( + exists=True, + secrets=(), + permissions=[ + { + "packages": "write", + "id-token": "write", + } + ], + ), + } + + merge_gate_entry = custom_workflow_extractor._build_merge_gate_entry( + custom_workflows_dict + ) + + assert list(merge_gate_entry.permissions.items()) == [ + ("contents", "read"), + ("packages", "write"), + ("id-token", "write"), + ] diff --git a/test/unit/util/workflows/custom_workflow_test.py b/test/unit/util/workflows/custom_workflow_test.py index 14fc74b53..27e2578d3 100644 --- a/test/unit/util/workflows/custom_workflow_test.py +++ b/test/unit/util/workflows/custom_workflow_test.py @@ -3,6 +3,7 @@ import pytest +from exasol.toolbox.util.workflows.custom_workflow import merge_permissions from exasol.toolbox.util.workflows.custom_workflow_extractor import CustomWorkflow @@ -24,6 +25,23 @@ class TestCustomWorkflow: required: true """) + yaml_with_permissions = cleandoc(""" + name: Build & Publish + + on: + workflow_call: + + jobs: + build: + permissions: + contents: read + packages: read + publish: + permissions: + contents: write + id-token: write + """) + yaml_without_secrets = cleandoc(""" name: Build & Publish @@ -31,6 +49,17 @@ class TestCustomWorkflow: workflow_call: """) + yaml_with_job_without_permissions = cleandoc(""" + name: Build & Publish + + on: + workflow_call: + + jobs: + build: + runs-on: ubuntu-latest + """) + def test_load_from_file(self, test_yml): test_yml.write_text(self.yaml_with_secrets) @@ -53,3 +82,46 @@ def test_extract_secrets_when_no_secrets_present(self, test_yml): secrets = custom_workflow.extract_secrets() assert secrets == () + + def test_extract_permissions_when_present(self, test_yml): + test_yml.write_text(self.yaml_with_permissions) + + custom_workflow = CustomWorkflow.load_from_file(file_path=test_yml) + permissions = custom_workflow.extract_permissions() + + assert list(permissions.items()) == [ + ("contents", "write"), + ("packages", "read"), + ("id-token", "write"), + ] + + def test_extract_permissions_when_no_jobs_present(self, test_yml): + test_yml.write_text(self.yaml_without_secrets) + + custom_workflow = CustomWorkflow.load_from_file(file_path=test_yml) + permissions = custom_workflow.extract_permissions() + + assert permissions == {} + + def test_extract_permissions_when_job_has_no_permissions(self, test_yml): + test_yml.write_text(self.yaml_with_job_without_permissions) + + custom_workflow = CustomWorkflow.load_from_file(file_path=test_yml) + permissions = custom_workflow.extract_permissions() + + assert permissions == {} + + def test_merge_permissions_keeps_most_permissive_level(self): + permissions = merge_permissions( + [ + {"contents": "read", "packages": "read"}, + {"contents": "write", "issues": "read"}, + {"packages": "read", "issues": "write"}, + ] + ) + + assert list(permissions.items()) == [ + ("contents", "write"), + ("packages", "read"), + ("issues", "write"), + ] From 92a1ae40a000f9cbac7e47228e6d148448464920 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 10 Jul 2026 09:56:16 +0200 Subject: [PATCH 07/24] Remove not-maintained workflows from get_workflow_templates - need to be handled separately --- exasol/toolbox/util/workflows/templates.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/exasol/toolbox/util/workflows/templates.py b/exasol/toolbox/util/workflows/templates.py index 521d1e587..2d713e83e 100644 --- a/exasol/toolbox/util/workflows/templates.py +++ b/exasol/toolbox/util/workflows/templates.py @@ -21,9 +21,12 @@ def get_workflow_templates() -> Mapping[str, Path]: """ package_resources = resources.files(WORKFLOW_TEMPLATES_DIRECTORY) return { - workflow_path.name.removesuffix(".yml"): Path(str(workflow_path)) + workflow_name: Path(str(workflow_path)) for workflow_path in package_resources.iterdir() - if workflow_path.is_file() and workflow_path.name.endswith(".yml") + if workflow_path.is_file() + and workflow_path.name.endswith(".yml") + and (workflow_name := workflow_path.name.removesuffix(".yml")) + not in NOT_MAINTAINED_WORKFLOW_NAMES } From c3d2872d926380bb65ce9d9ce88cddbb66ec4213 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 10 Jul 2026 10:17:17 +0200 Subject: [PATCH 08/24] Remove now unneeded custom exception as not maintained workflows not in ALL anymore --- exasol/toolbox/util/workflows/exceptions.py | 12 ------- exasol/toolbox/util/workflows/templates.py | 7 +--- .../util/workflows/workflow_orchestrator.py | 16 ++------- test/unit/nox/_workflow_test.py | 10 +++--- .../util/workflows/patch_workflow_test.py | 33 +------------------ test/unit/util/workflows/templates_test.py | 11 +------ .../workflows/workflow_orchestrator_test.py | 31 +---------------- 7 files changed, 11 insertions(+), 109 deletions(-) diff --git a/exasol/toolbox/util/workflows/exceptions.py b/exasol/toolbox/util/workflows/exceptions.py index 1ec4b5479..a87e131eb 100644 --- a/exasol/toolbox/util/workflows/exceptions.py +++ b/exasol/toolbox/util/workflows/exceptions.py @@ -99,18 +99,6 @@ def __init__(self, workflow_name: str, valid_workflows): ) -class NotMaintainedWorkflowError(ValueError): - """ - Raised when a PTB-seeded workflow is requested in an existing project. - """ - - def __init__(self, workflow_name: str): - super().__init__( - f"Workflow '{workflow_name}' is a PTB-seeded workflow that is " - "originally provided by the PTB and can only be seeded for a new project." - ) - - class YamlKeyError(Exception): """ Base exception for when a specified value cannot be found in a YAML. diff --git a/exasol/toolbox/util/workflows/templates.py b/exasol/toolbox/util/workflows/templates.py index 2d713e83e..860f35395 100644 --- a/exasol/toolbox/util/workflows/templates.py +++ b/exasol/toolbox/util/workflows/templates.py @@ -4,10 +4,7 @@ import importlib_resources as resources -from exasol.toolbox.util.workflows.exceptions import ( - InvalidWorkflowNameError, - NotMaintainedWorkflowError, -) +from exasol.toolbox.util.workflows.exceptions import InvalidWorkflowNameError WORKFLOW_TEMPLATES_DIRECTORY = "exasol.toolbox.templates.github.workflows" NOT_MAINTAINED_WORKFLOW_NAMES: Final[list[str]] = ["slow-checks"] @@ -36,8 +33,6 @@ def validate_workflow_name(workflow_name: str) -> str: """ if workflow_name not in WORKFLOW_TEMPLATE_OPTIONS: raise InvalidWorkflowNameError(workflow_name, WORKFLOW_TEMPLATE_OPTIONS.keys()) - if workflow_name in NOT_MAINTAINED_WORKFLOW_NAMES: - raise NotMaintainedWorkflowError(workflow_name) return workflow_name diff --git a/exasol/toolbox/util/workflows/workflow_orchestrator.py b/exasol/toolbox/util/workflows/workflow_orchestrator.py index 646653aac..679769bb3 100644 --- a/exasol/toolbox/util/workflows/workflow_orchestrator.py +++ b/exasol/toolbox/util/workflows/workflow_orchestrator.py @@ -17,7 +17,6 @@ from exasol.toolbox.util.workflows import logger from exasol.toolbox.util.workflows.exceptions import ( InvalidWorkflowPatcherEntryError, - NotMaintainedWorkflowError, YamlKeyError, ) from exasol.toolbox.util.workflows.patch_workflow import ( @@ -81,11 +80,10 @@ def _is_new_project(self) -> bool: def _iter_workflows(self) -> Iterator[Workflow]: logger.info(f"Selected workflow(s) to update: {list(self.templates.keys())}") - is_new_project = self._is_new_project() for workflow_name, template_path in self.templates.items(): patch_yaml = self._extract_workflow_patch(workflow_name=workflow_name) - if self._skip_workflow(workflow_name, is_new_project): + if self._skip_workflow(workflow_name): continue yield self._load_workflow( @@ -108,20 +106,12 @@ def _load_workflow( entry=ex.entry, ) from ex - def _skip_workflow(self, workflow_name: str, is_new_project: bool) -> bool: + def _skip_workflow(self, workflow_name: str) -> bool: """ Return ``True`` if the workflow should be skipped because it is not maintained by the PTB or not applicable to the current project, otherwise return ``False``. """ - try: - validate_workflow_name(workflow_name) - except NotMaintainedWorkflowError: - if not is_new_project: - logger.debug( - "Skipping not-maintained workflow in older project: %s", - workflow_name, - ) - return True + validate_workflow_name(workflow_name) if ( workflow_name in DOCUMENTATION_ONLY_WORKFLOW_NAMES diff --git a/test/unit/nox/_workflow_test.py b/test/unit/nox/_workflow_test.py index b69d7f3ed..ade8112bf 100644 --- a/test/unit/nox/_workflow_test.py +++ b/test/unit/nox/_workflow_test.py @@ -182,7 +182,7 @@ def test_raises_session_quit_when_workflows_are_out_of_date( check_workflow(nox_session) assert str(exc.value) == ( - "\n14 workflows are out of date:\n" + "\n13 workflows are out of date:\n" "- build-and-publish\n" "- cd\n" "- check-release-tag\n" @@ -195,8 +195,7 @@ def test_raises_session_quit_when_workflows_are_out_of_date( "- merge-gate\n" "- periodic-validation\n" "- pr-merge\n" - "- report\n" - "- slow-checks" + "- report" ) @staticmethod @@ -221,7 +220,7 @@ def test_raises_session_quit_without_documentation_workflows( check_workflow(nox_session) assert str(exc.value) == ( - "\n12 workflows are out of date:\n" + "\n11 workflows are out of date:\n" "- build-and-publish\n" "- cd\n" "- check-release-tag\n" @@ -232,8 +231,7 @@ def test_raises_session_quit_without_documentation_workflows( "- matrix\n" "- merge-gate\n" "- periodic-validation\n" - "- report\n" - "- slow-checks" + "- report" ) diff --git a/test/unit/util/workflows/patch_workflow_test.py b/test/unit/util/workflows/patch_workflow_test.py index e5b69618e..f8aa21978 100644 --- a/test/unit/util/workflows/patch_workflow_test.py +++ b/test/unit/util/workflows/patch_workflow_test.py @@ -6,15 +6,11 @@ from pydantic import ValidationError from ruamel.yaml import CommentedMap -from exasol.toolbox.util.workflows.exceptions import ( - InvalidWorkflowPatcherYamlError, - NotMaintainedWorkflowError, -) +from exasol.toolbox.util.workflows.exceptions import InvalidWorkflowPatcherYamlError from exasol.toolbox.util.workflows.patch_workflow import ( ActionType, WorkflowPatcher, ) -from exasol.toolbox.util.workflows.templates import NOT_MAINTAINED_WORKFLOW_NAMES @pytest.fixture @@ -139,30 +135,3 @@ def test_raises_error_for_unknown_workflow_name( assert "Invalid workflow: unknown-workflow. Must be one of dict_keys([" in str( underlying_error ) - - @staticmethod - def test_rejects_not_maintained_workflow_name( - workflow_patcher_yaml, workflow_patcher - ): - content = f""" - workflows: - - name: "{NOT_MAINTAINED_WORKFLOW_NAMES[0]}" - remove_jobs: - - build-documentation-and-check-links - """ - workflow_patcher_yaml.write_text(cleandoc(content)) - - with pytest.raises( - InvalidWorkflowPatcherYamlError, - match="is malformed; it failed Pydantic validation", - ) as ex: - workflow_patcher.content - - underlying_error = ex.value.__cause__ - assert isinstance(underlying_error, ValidationError) - assert "workflows.0.name" in str(underlying_error) - assert "PTB-seeded workflow" in str(underlying_error) - assert isinstance( - ex.value.validation_error.errors()[0]["ctx"]["error"], - NotMaintainedWorkflowError, - ) diff --git a/test/unit/util/workflows/templates_test.py b/test/unit/util/workflows/templates_test.py index 8969ad658..ab2679203 100644 --- a/test/unit/util/workflows/templates_test.py +++ b/test/unit/util/workflows/templates_test.py @@ -1,9 +1,6 @@ import pytest -from exasol.toolbox.util.workflows.exceptions import ( - InvalidWorkflowNameError, - NotMaintainedWorkflowError, -) +from exasol.toolbox.util.workflows.exceptions import InvalidWorkflowNameError from exasol.toolbox.util.workflows.templates import ( NOT_MAINTAINED_WORKFLOW_NAMES, WORKFLOW_TEMPLATE_OPTIONS, @@ -30,7 +27,6 @@ def test_get_workflow_templates(project_config): "periodic-validation", "pr-merge", "report", - "slow-checks", } # check only one path, as all formatted the same by convention assert ( @@ -57,8 +53,3 @@ def test_returns_valid_maintained_names(workflow_name): def test_rejects_unknown_workflow(): with pytest.raises(InvalidWorkflowNameError, match="Invalid workflow: unknown"): validate_workflow_name("unknown") - - @staticmethod - def test_rejects_not_maintained_workflow(): - with pytest.raises(NotMaintainedWorkflowError, match="PTB-seeded workflow"): - validate_workflow_name(NOT_MAINTAINED_WORKFLOW_NAMES[0]) diff --git a/test/unit/util/workflows/workflow_orchestrator_test.py b/test/unit/util/workflows/workflow_orchestrator_test.py index 63316dd5e..ec83f4072 100644 --- a/test/unit/util/workflows/workflow_orchestrator_test.py +++ b/test/unit/util/workflows/workflow_orchestrator_test.py @@ -69,36 +69,9 @@ def test_returns_false_when_yml_files_exist(project_config): class TestSkipWorkflow: - @staticmethod - def test_returns_true_for_not_maintained_workflow_in_existing_project( - project_config, - ): - workflow_directory = project_config.github_workflow_directory - workflow_directory.mkdir(parents=True) - (workflow_directory / "existing.yml").touch() - - result = WorkflowOrchestrator( - workflow_choice=NOT_MAINTAINED_WORKFLOW_NAMES[0], - config=project_config, - )._skip_workflow( - workflow_name=NOT_MAINTAINED_WORKFLOW_NAMES[0], - is_new_project=False, - ) - - assert result is True - - @staticmethod - def test_returns_false_for_maintained_workflow(project_config): - result = WorkflowOrchestrator( - workflow_choice="checks", - config=project_config, - )._skip_workflow(workflow_name="checks", is_new_project=False) - - assert result is False - @staticmethod @pytest.mark.parametrize("workflow_name", DOCUMENTATION_ONLY_WORKFLOW_NAMES) - def test_returns_ralse_for_documentation_only_workflow_when_docs_enabled( + def test_returns_false_for_documentation_only_workflow_when_docs_enabled( project_config, workflow_name ): @@ -107,7 +80,6 @@ def test_returns_ralse_for_documentation_only_workflow_when_docs_enabled( config=project_config, )._skip_workflow( workflow_name=workflow_name, - is_new_project=False, ) assert result is False @@ -130,7 +102,6 @@ def has_documentation(self) -> bool: config=config, )._skip_workflow( workflow_name=workflow_name, - is_new_project=False, ) assert result is True From 0a36b8655c5397453cbaccbd8e83070d5f960161 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 10 Jul 2026 10:35:38 +0200 Subject: [PATCH 09/24] Remove redundant check - This is already present in the CLI of the nox sessions workflow:check and workflow:generate - This is already required by the WorkflowOrchestrator which is a pydantic model which restricts WorkflowChoice and hard-code maps it to the right values. --- exasol/toolbox/util/workflows/workflow_orchestrator.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/exasol/toolbox/util/workflows/workflow_orchestrator.py b/exasol/toolbox/util/workflows/workflow_orchestrator.py index 679769bb3..0c0d38874 100644 --- a/exasol/toolbox/util/workflows/workflow_orchestrator.py +++ b/exasol/toolbox/util/workflows/workflow_orchestrator.py @@ -26,7 +26,6 @@ from exasol.toolbox.util.workflows.templates import ( DOCUMENTATION_ONLY_WORKFLOW_NAMES, WORKFLOW_TEMPLATE_OPTIONS, - validate_workflow_name, ) from exasol.toolbox.util.workflows.workflow import Workflow @@ -111,8 +110,6 @@ def _skip_workflow(self, workflow_name: str) -> bool: Return ``True`` if the workflow should be skipped because it is not maintained by the PTB or not applicable to the current project, otherwise return ``False``. """ - validate_workflow_name(workflow_name) - if ( workflow_name in DOCUMENTATION_ONLY_WORKFLOW_NAMES and not self.config.has_documentation From 1ac5e104a6c7ff8c64dbf754bafe7a5c10c2a273 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 10 Jul 2026 10:40:49 +0200 Subject: [PATCH 10/24] Ensure NOT_MAINTAINED_WORKFLOWS are seeded first --- exasol/toolbox/util/workflows/templates.py | 11 +++ .../util/workflows/workflow_orchestrator.py | 20 +++-- .../workflows/workflow_orchestrator_test.py | 86 +++++++------------ 3 files changed, 55 insertions(+), 62 deletions(-) diff --git a/exasol/toolbox/util/workflows/templates.py b/exasol/toolbox/util/workflows/templates.py index 860f35395..b854e09c7 100644 --- a/exasol/toolbox/util/workflows/templates.py +++ b/exasol/toolbox/util/workflows/templates.py @@ -27,6 +27,17 @@ def get_workflow_templates() -> Mapping[str, Path]: } +def get_not_maintained_workflow_templates() -> Mapping[str, Path]: + """ + A mapping of not-maintained workflow template names to paths. + """ + package_resources = resources.files(WORKFLOW_TEMPLATES_DIRECTORY) + return { + workflow_name: Path(str(package_resources / f"{workflow_name}.yml")) + for workflow_name in NOT_MAINTAINED_WORKFLOW_NAMES + } + + def validate_workflow_name(workflow_name: str) -> str: """ Validate that the given workflow exists and is allowed in the current context. diff --git a/exasol/toolbox/util/workflows/workflow_orchestrator.py b/exasol/toolbox/util/workflows/workflow_orchestrator.py index 0c0d38874..6e630b298 100644 --- a/exasol/toolbox/util/workflows/workflow_orchestrator.py +++ b/exasol/toolbox/util/workflows/workflow_orchestrator.py @@ -26,6 +26,7 @@ from exasol.toolbox.util.workflows.templates import ( DOCUMENTATION_ONLY_WORKFLOW_NAMES, WORKFLOW_TEMPLATE_OPTIONS, + get_not_maintained_workflow_templates, ) from exasol.toolbox.util.workflows.workflow import Workflow @@ -77,9 +78,9 @@ def _is_new_project(self) -> bool: """ return not any(self.config.github_workflow_directory.glob("*.yml")) - def _iter_workflows(self) -> Iterator[Workflow]: - logger.info(f"Selected workflow(s) to update: {list(self.templates.keys())}") - for workflow_name, template_path in self.templates.items(): + def _iter_workflows(self, templates: Mapping[str, Path]) -> Iterator[Workflow]: + logger.info(f"Selected workflow(s) to update: {list(templates.keys())}") + for workflow_name, template_path in templates.items(): patch_yaml = self._extract_workflow_patch(workflow_name=workflow_name) if self._skip_workflow(workflow_name): @@ -126,7 +127,16 @@ def generate_workflows(self) -> None: """ Render the selected workflows and write them to disk. """ - for workflow in self._iter_workflows(): + # First, generate not-maintained workflows for a new project. + # This ensures proper extraction and passing of secrets & permissions + # before parent workflows such as `merge-gate.yml` and + # `periodic-validation.yml` are rendered. + if self._is_new_project() and self.workflow_choice == ALL: + not_maintainted_templates = get_not_maintained_workflow_templates() + for workflow in self._iter_workflows(not_maintainted_templates): + workflow.write_to_file() + + for workflow in self._iter_workflows(self.templates): workflow.write_to_file() def find_differing_workflows(self) -> list[str]: @@ -136,7 +146,7 @@ def find_differing_workflows(self) -> list[str]: Returns the names of the workflows that differ from the file on disk. """ outdated_workflows: list[str] = [] - for workflow in self._iter_workflows(): + for workflow in self._iter_workflows(self.templates): comparison = workflow.compare_to_file() if comparison != "": workflow_name = workflow.output_path.stem diff --git a/test/unit/util/workflows/workflow_orchestrator_test.py b/test/unit/util/workflows/workflow_orchestrator_test.py index ec83f4072..6758312f5 100644 --- a/test/unit/util/workflows/workflow_orchestrator_test.py +++ b/test/unit/util/workflows/workflow_orchestrator_test.py @@ -114,10 +114,11 @@ def test_works_as_expected_without_patcher(project_config_without_patcher): project_config_without_patcher.github_workflow_directory.mkdir(parents=True) input_text = WORKFLOW_TEMPLATE_OPTIONS[workflow_name].read_text() - result = WorkflowOrchestrator( + orchestrator = WorkflowOrchestrator( workflow_choice=workflow_name, config=project_config_without_patcher, - )._iter_workflows() + ) + result = orchestrator._iter_workflows(orchestrator.templates) result = list(result) assert len(result) == 1 @@ -137,10 +138,11 @@ def test_works_as_expected_with_relevant_patcher(project_config, remove_job_yaml assert removed_job_name in remove_job_yaml assert removed_job_name in input_text - result = WorkflowOrchestrator( + orchestrator = WorkflowOrchestrator( workflow_choice=workflow_name, config=project_config, - )._iter_workflows() + ) + result = orchestrator._iter_workflows(orchestrator.templates) result = list(result) assert len(result) == 1 @@ -159,10 +161,11 @@ def test_works_as_expected_with_not_relevant_patcher( project_config.github_workflow_directory.mkdir(parents=True) input_text = WORKFLOW_TEMPLATE_OPTIONS[workflow_name].read_text() - result = WorkflowOrchestrator( + orchestrator = WorkflowOrchestrator( workflow_choice=workflow_name, config=project_config, - )._iter_workflows() + ) + result = orchestrator._iter_workflows(orchestrator.templates) result = list(result) assert len(result) == 1 @@ -172,55 +175,6 @@ def test_works_as_expected_with_not_relevant_patcher( == _remove_header(input_text)[:10] ) - @staticmethod - def test_not_maintained_workflows_added_to_new_project( - project_config_without_patcher, - ): - directory = project_config_without_patcher.github_workflow_directory - directory.mkdir(parents=True) - - result = WorkflowOrchestrator( - workflow_choice="all", - config=project_config_without_patcher, - )._iter_workflows() - - assert {f"{name}.yml" for name in NOT_MAINTAINED_WORKFLOW_NAMES}.issubset( - {workflow.output_path.name for workflow in result} - ) - - @staticmethod - @pytest.mark.parametrize("workflow_name", NOT_MAINTAINED_WORKFLOW_NAMES) - def test_not_maintained_workflows_not_modified_in_old_project( - project_config_without_patcher, workflow_name - ): - directory = project_config_without_patcher.github_workflow_directory - directory.mkdir(parents=True, exist_ok=True) - workflow = "slow-checks.yml" - (directory / workflow).touch() - - result = WorkflowOrchestrator( - workflow_choice=workflow_name, - config=project_config_without_patcher, - )._iter_workflows() - - assert list(result) == [] - - @staticmethod - @pytest.mark.parametrize("workflow_name", NOT_MAINTAINED_WORKFLOW_NAMES) - def test_not_maintained_workflows_not_added_to_old_project( - project_config_without_patcher, workflow_name - ): - directory = project_config_without_patcher.github_workflow_directory - directory.mkdir(parents=True, exist_ok=True) - (directory / "dummy.yml").touch() - - result = WorkflowOrchestrator( - workflow_choice=workflow_name, - config=project_config_without_patcher, - )._iter_workflows() - - assert list(result) == [] - @staticmethod def test_raises_invalidworkflowpatcherentryerror(project_config): patcher_yml = """ @@ -232,10 +186,11 @@ def test_raises_invalidworkflowpatcherentryerror(project_config): project_config.github_workflow_patcher_yaml.write_text(patcher_yml) with pytest.raises(InvalidWorkflowPatcherEntryError) as ex: - for _ in WorkflowOrchestrator( + orchestrator = WorkflowOrchestrator( workflow_choice="checks", config=project_config, - )._iter_workflows(): + ) + for _ in orchestrator._iter_workflows(orchestrator.templates): pass assert ( @@ -260,6 +215,23 @@ def test_writes_all_workflows_on_fresh_project(project_config_without_patcher): (directory / f"{name}.yml").exists() for name in WORKFLOW_TEMPLATE_OPTIONS ) + @staticmethod + def test_writes_not_maintained_workflows_on_fresh_project( + project_config_without_patcher, + ): + directory = project_config_without_patcher.github_workflow_directory + directory.mkdir(parents=True) + + WorkflowOrchestrator( + workflow_choice="all", + config=project_config_without_patcher, + ).generate_workflows() + + assert all( + (directory / f"{name}.yml").exists() + for name in NOT_MAINTAINED_WORKFLOW_NAMES + ) + @staticmethod def test_does_not_write_when_all_workflows_are_up_to_date( project_config_without_patcher, From 69e8d1492a8f290023a51fe3b06cad9f771d2f7a Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 10 Jul 2026 10:55:28 +0200 Subject: [PATCH 11/24] Tighten wording in documentation --- .../github_workflows/workflow_variables.rst | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/doc/user_guide/features/github_workflows/workflow_variables.rst b/doc/user_guide/features/github_workflows/workflow_variables.rst index fb6b8e8e5..838f77077 100644 --- a/doc/user_guide/features/github_workflows/workflow_variables.rst +++ b/doc/user_guide/features/github_workflows/workflow_variables.rst @@ -34,7 +34,7 @@ when they call reusable workflows. Secrets ------- -The PTB extracts secret names from reusable custom workflow files and exposes them +The PTB extracts secret names from custom workflow files and exposes them through :py:attr:`exasol.toolbox.config.BaseConfig.github_template_dict` under the ``custom_workflows`` entry. PTB-controlled workflow templates use those extracted names when they call reusable workflows and forward secrets via ``secrets:``. @@ -65,18 +65,13 @@ reference the custom workflow. Permissions ----------- -The PTB extracts the effective GitHub permissions required by reusable custom -workflow files. It scans every job in the workflow, reads each job's -``permissions`` block, and combines the results into a single ordered mapping. +The PTB extracts the permissions required by custom workflow files. It reads every job's +``permissions`` block and combines the results into a single ordered mapping, where +the most permissive level wins. -When multiple jobs request the same permission, the most permissive level wins -while the first-seen key order is preserved. For example, if one job requests -``contents: read`` and another later job requests ``contents: write``, the final -mapping keeps ``contents`` once, with ``write`` as the level. - -Please only configure the minimum required permissions and granting the least required +Please only configure the minimum required permissions by granting the least required access. In practice, ``contents: read`` is the most common baseline for workflows, and -other permissions should only be added when a particular step truly requires them. +other permissions should only be added when it is truly required. For example, a custom workflow can declare permissions like this: From 08f30e67115337bea3b3eba21fd97b7cb5f33718 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 14 Jul 2026 10:08:23 +0200 Subject: [PATCH 12/24] Apply text changes from reviewers --- doc/changes/changes_10.0.0.md | 17 ++++++----------- doc/changes/changes_10.2.0.md | 2 +- doc/changes/changes_10.2.1.md | 2 +- doc/changes/unreleased.md | 4 ++-- .../github_workflows/workflow_variables.rst | 2 +- 5 files changed, 11 insertions(+), 16 deletions(-) diff --git a/doc/changes/changes_10.0.0.md b/doc/changes/changes_10.0.0.md index d8ba3cd22..a1ba27cf3 100644 --- a/doc/changes/changes_10.0.0.md +++ b/doc/changes/changes_10.0.0.md @@ -42,21 +42,20 @@ This release fixes vulnerabilities by updating dependencies: | cryptography | GHSA-537c-gmf6-5ccf | 48.0.0 | 48.0.1 | | msgpack | GHSA-6v7p-g79w-8964 | 1.1.2 | 1.2.1 | -## Feature +* #867: Fixed zizmor linting results + +## Features * #874: Added the `security` label to dependency update PR creation * #699: Added `all-extras` support to the Python environment GitHub action * #875: Added `name` attribute to generated workflow jobs using `-extension.yml` workflows +* #878: Added Nox session `workflow:audit` which uses `zizmor` and added it in `checks.yml` +* #872: Added `custom_workflows` to `github_template_dict` for automatic custom workflow secret extraction -## Bug Fix +## Bugfixes * #744: Updated nox DB-version handling to use `BaseConfig.minimum_exasol_version` instead hardcoded `7.1.9` -## Feature - -* #878: Added Nox session `workflow:audit` which uses `zizmor` and added it in `checks.yml` -* #872: Added `custom_workflows` to `github_template_dict` for automatic custom workflow secret extraction - ## Refactoring * #744: Extracted shared minimum-version selection logic into `minimum_declared_version()` @@ -67,10 +66,6 @@ This release fixes vulnerabilities by updating dependencies: * #828: Removed the legacy migration page and merged the useful guidance into getting started * #789: Consolidated the metrics and Sonar documentation to reflect the current PTB reporting flow -## Security - -* #867: Fixed zizmor linting results - ## Dependency Updates ### `main` diff --git a/doc/changes/changes_10.2.0.md b/doc/changes/changes_10.2.0.md index e00721c99..83beca1d5 100644 --- a/doc/changes/changes_10.2.0.md +++ b/doc/changes/changes_10.2.0.md @@ -6,7 +6,7 @@ This minor release adds automated vulnerability updates through Nox session `vulnerabilities:update` and improves the `dependency-update.yml`. It also includes a few workflow-related bug fixes and documentation updates. -## Bug Fix +## Bugfixes * #909: Updated `cd.yml` workflow so that `cd-extension.yml` workflow depends on `build-and-publish`. This ensures that the custom release workflow only runs when the PyPi release was successful. * #910: Added `gh-pages.yml` to be ignored when `has_documentation=False` in the `PROJECT_CONFIG` diff --git a/doc/changes/changes_10.2.1.md b/doc/changes/changes_10.2.1.md index c29ae2b39..7846783d0 100644 --- a/doc/changes/changes_10.2.1.md +++ b/doc/changes/changes_10.2.1.md @@ -1,5 +1,5 @@ # 10.2.1 - 2026-07-08 -## Bug Fix +## Bugfixes * #920: Ensured extracted secrets are unique and alphabetically sorted from the custom workflows diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index e813a4483..4804c3efd 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -2,9 +2,9 @@ ## Summary -## Feature +## Features -* #922: Extended `custom_workflows` of `github_template_dict` for automatic custom workflow permissions extraction +* #922: Extended `custom_workflows` of `github_template_dict` for automatic custom workflow permissions extraction ## Refactoring diff --git a/doc/user_guide/features/github_workflows/workflow_variables.rst b/doc/user_guide/features/github_workflows/workflow_variables.rst index 838f77077..f0c626b0a 100644 --- a/doc/user_guide/features/github_workflows/workflow_variables.rst +++ b/doc/user_guide/features/github_workflows/workflow_variables.rst @@ -34,7 +34,7 @@ when they call reusable workflows. Secrets ------- -The PTB extracts secret names from custom workflow files and exposes them +The PTB extracts the names of secrets from custom workflow files and exposes them through :py:attr:`exasol.toolbox.config.BaseConfig.github_template_dict` under the ``custom_workflows`` entry. PTB-controlled workflow templates use those extracted names when they call reusable workflows and forward secrets via ``secrets:``. From 70a9f21220207316c3be9c84a8eb0bd4989b3c76 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 14 Jul 2026 10:54:47 +0200 Subject: [PATCH 13/24] Fix section headings --- .../features/github_workflows/workflow_variables.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/user_guide/features/github_workflows/workflow_variables.rst b/doc/user_guide/features/github_workflows/workflow_variables.rst index f0c626b0a..c307796ba 100644 --- a/doc/user_guide/features/github_workflows/workflow_variables.rst +++ b/doc/user_guide/features/github_workflows/workflow_variables.rst @@ -22,7 +22,7 @@ standardized baseline that can be overridden in individual projects. .. _custom_workflow_metadata: Custom Workflow Metadata -^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------ The PTB extracts metadata from reusable custom workflow files and exposes it through :py:attr:`exasol.toolbox.config.BaseConfig.github_template_dict` under the @@ -32,7 +32,7 @@ when they call reusable workflows. .. _custom_workflow_secrets: Secrets -------- +^^^^^^^ The PTB extracts the names of secrets from custom workflow files and exposes them through :py:attr:`exasol.toolbox.config.BaseConfig.github_template_dict` under the @@ -63,7 +63,7 @@ reference the custom workflow. .. _custom_workflow_permissions: Permissions ------------ +^^^^^^^^^^^ The PTB extracts the permissions required by custom workflow files. It reads every job's ``permissions`` block and combines the results into a single ordered mapping, where From c1028c5be02c0dc14df304a8fb267bb64cdc180f Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 14 Jul 2026 12:51:36 +0200 Subject: [PATCH 14/24] Introduce macro for workflow_permissions --- .../github/workflows/_workflow_macros.j2 | 9 ++ .../toolbox/templates/github/workflows/cd.yml | 8 +- .../toolbox/templates/github/workflows/ci.yml | 8 +- .../templates/github/workflows/fast-tests.yml | 8 +- .../templates/github/workflows/merge-gate.yml | 15 +--- .../github/workflows/periodic-validation.yml | 8 +- exasol/toolbox/util/workflows/render_yaml.py | 37 +++++--- .../util/workflows/render_yaml_macro_test.py | 58 ++++++++++++ test/unit/util/workflows/render_yaml_test.py | 88 ------------------- .../workflows/workflow_orchestrator_test.py | 10 ++- 10 files changed, 109 insertions(+), 140 deletions(-) create mode 100644 exasol/toolbox/templates/github/workflows/_workflow_macros.j2 create mode 100644 test/unit/util/workflows/render_yaml_macro_test.py diff --git a/exasol/toolbox/templates/github/workflows/_workflow_macros.j2 b/exasol/toolbox/templates/github/workflows/_workflow_macros.j2 new file mode 100644 index 000000000..f688805f0 --- /dev/null +++ b/exasol/toolbox/templates/github/workflows/_workflow_macros.j2 @@ -0,0 +1,9 @@ +(% macro workflow_permissions(workflow_name) -%) +(% set permissions = custom_workflows[workflow_name].permissions -%) +(% if permissions -%) +permissions: +(% for permission_name, permission_level in permissions.items() %) + (( permission_name )): (( permission_level )) +(% endfor -%) +(% endif -%) +(% endmacro %) diff --git a/exasol/toolbox/templates/github/workflows/cd.yml b/exasol/toolbox/templates/github/workflows/cd.yml index a856759cf..882e52126 100644 --- a/exasol/toolbox/templates/github/workflows/cd.yml +++ b/exasol/toolbox/templates/github/workflows/cd.yml @@ -1,4 +1,5 @@ (( workflow_header )) +(% from "_workflow_macros.j2" import workflow_permissions with context %) name: CD on: @@ -38,12 +39,7 @@ jobs: (( secret_name )): ${{ secrets.(( secret_name )) }} (% endfor %) (% endif %) - (% if custom_workflows["cd-extension"].permissions %) - permissions: - (% for permission_name, permission_level in custom_workflows["cd-extension"].permissions.items() %) - (( permission_name )): (( permission_level )) - (% endfor %) - (% endif %) + (( workflow_permissions("cd-extension") | trim )) (% endif %) (% if has_documentation %) diff --git a/exasol/toolbox/templates/github/workflows/ci.yml b/exasol/toolbox/templates/github/workflows/ci.yml index da524c3a7..6a12b950e 100644 --- a/exasol/toolbox/templates/github/workflows/ci.yml +++ b/exasol/toolbox/templates/github/workflows/ci.yml @@ -1,4 +1,5 @@ (( workflow_header )) +(% from "_workflow_macros.j2" import workflow_permissions with context %) name: CI on: @@ -15,12 +16,7 @@ jobs: (( secret_name )): ${{ secrets.(( secret_name )) }} (% endfor %) (% endif %) - (% if custom_workflows["merge-gate"].permissions %) - permissions: - (% for permission_name, permission_level in custom_workflows["merge-gate"].permissions.items() %) - (( permission_name )): (( permission_level )) - (% endfor %) - (% endif %) + (( workflow_permissions("merge-gate") | trim )) report: name: Report diff --git a/exasol/toolbox/templates/github/workflows/fast-tests.yml b/exasol/toolbox/templates/github/workflows/fast-tests.yml index c0151676a..d14dac414 100644 --- a/exasol/toolbox/templates/github/workflows/fast-tests.yml +++ b/exasol/toolbox/templates/github/workflows/fast-tests.yml @@ -1,4 +1,5 @@ (( workflow_header )) +(% from "_workflow_macros.j2" import workflow_permissions with context %) name: Fast-Tests on: @@ -52,10 +53,5 @@ jobs: (( secret_name )): ${{ secrets.(( secret_name )) }} (% endfor %) (% endif %) - (% if custom_workflows["fast-tests-extension"].permissions %) - permissions: - (% for permission_name, permission_level in custom_workflows["fast-tests-extension"].permissions.items() %) - (( permission_name )): (( permission_level )) - (% endfor %) - (% endif %) + (( workflow_permissions("fast-tests-extension") | trim )) (% endif %) diff --git a/exasol/toolbox/templates/github/workflows/merge-gate.yml b/exasol/toolbox/templates/github/workflows/merge-gate.yml index 8ee19db55..b17d64e9f 100644 --- a/exasol/toolbox/templates/github/workflows/merge-gate.yml +++ b/exasol/toolbox/templates/github/workflows/merge-gate.yml @@ -1,4 +1,5 @@ (( workflow_header )) +(% from "_workflow_macros.j2" import workflow_permissions with context %) name: Merge-Gate on: @@ -59,12 +60,7 @@ jobs: (( secret_name )): ${{ secrets.(( secret_name )) }} (% endfor %) (% endif %) - (% if custom_workflows["slow-checks"].permissions %) - permissions: - (% for permission_name, permission_level in custom_workflows["slow-checks"].permissions.items() %) - (( permission_name )): (( permission_level )) - (% endfor %) - (% endif %) + (( workflow_permissions("slow-checks") | trim )) (% if custom_workflows["merge-gate-extension"].exists %) merge-gate-extension: @@ -76,12 +72,7 @@ jobs: (( secret_name )): ${{ secrets.(( secret_name )) }} (% endfor %) (% endif %) - (% if custom_workflows["merge-gate-extension"].permissions %) - permissions: - (% for permission_name, permission_level in custom_workflows["merge-gate-extension"].permissions.items() %) - (( permission_name )): (( permission_level )) - (% endfor %) - (% endif %) + (( workflow_permissions("merge-gate-extension") | trim )) (% endif %) # This job ensures inputs have been executed successfully. diff --git a/exasol/toolbox/templates/github/workflows/periodic-validation.yml b/exasol/toolbox/templates/github/workflows/periodic-validation.yml index ae3adaa43..09d83b4dc 100644 --- a/exasol/toolbox/templates/github/workflows/periodic-validation.yml +++ b/exasol/toolbox/templates/github/workflows/periodic-validation.yml @@ -1,4 +1,5 @@ (( workflow_header )) +(% from "_workflow_macros.j2" import workflow_permissions with context %) name: Periodic-Validation on: @@ -51,12 +52,7 @@ jobs: (( secret_name )): ${{ secrets.(( secret_name )) }} (% endfor %) (% endif %) - (% if custom_workflows["slow-checks"].permissions %) - permissions: - (% for permission_name, permission_level in custom_workflows["slow-checks"].permissions.items() %) - (( permission_name )): (( permission_level )) - (% endfor %) - (% endif %) + (( workflow_permissions("slow-checks") | trim )) report: name: Report diff --git a/exasol/toolbox/util/workflows/render_yaml.py b/exasol/toolbox/util/workflows/render_yaml.py index 3e02db891..b89ab8202 100644 --- a/exasol/toolbox/util/workflows/render_yaml.py +++ b/exasol/toolbox/util/workflows/render_yaml.py @@ -6,6 +6,7 @@ from jinja2 import ( Environment, + FileSystemLoader, StrictUndefined, TemplateError, ) @@ -21,20 +22,28 @@ YamlOutputError, YamlParsingError, ) +from exasol.toolbox.util.workflows.templates import WORKFLOW_TEMPLATE_OPTIONS -jinja_env = Environment( - variable_start_string="((", - variable_end_string="))", - autoescape=True, - # This requires that all Jinja variables must be defined in the provided - # dictionary. If not, then a `jinja2.exceptions.UndefinedError` exception - # will be raised. - undefined=StrictUndefined, - block_start_string="(%", - block_end_string="%)", - trim_blocks=True, # Removes the newline after a block - lstrip_blocks=True, # Removes tabs/spaces before a block -) + +def build_jinja_env(template_path: Path) -> Environment: + """ + Create a Jinja environment for a workflow template file. + """ + workflow_template_directory = WORKFLOW_TEMPLATE_OPTIONS["ci"].parent + return Environment( + loader=FileSystemLoader([template_path.parent, workflow_template_directory]), + variable_start_string="((", + variable_end_string="))", + autoescape=True, + # This requires that all Jinja variables must be defined in the provided + # dictionary. If not, then a `jinja2.exceptions.UndefinedError` exception + # will be raised. + undefined=StrictUndefined, + block_start_string="(%", + block_end_string="%)", + trim_blocks=True, # Removes the newline after a block + lstrip_blocks=True, # Removes tabs/spaces before a block + ) def get_standard_yaml() -> YAML: @@ -82,7 +91,7 @@ def _render_with_jinja(self, input_str: str) -> str: jinja_dict_source="PROJECT_CONFIG.github_template_dict", jinja_dict_values=self.github_template_dict, ) - jinja_template = jinja_env.from_string(input_str) + jinja_template = build_jinja_env(self.file_path).from_string(input_str) return jinja_template.render(self.github_template_dict) def get_yaml_dict(self) -> CommentedMap: diff --git a/test/unit/util/workflows/render_yaml_macro_test.py b/test/unit/util/workflows/render_yaml_macro_test.py new file mode 100644 index 000000000..932fc153f --- /dev/null +++ b/test/unit/util/workflows/render_yaml_macro_test.py @@ -0,0 +1,58 @@ +from inspect import cleandoc +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from exasol.toolbox.util.workflows.render_yaml import YamlRenderer + + +@pytest.fixture +def macro_template(tmp_path: Path) -> Path: + template_path = tmp_path / "dummy.yml" + template_path.write_text(cleandoc(""" + (% from "_workflow_macros.j2" import workflow_permissions with context %) + jobs: + macro-job: + uses: ./.github/workflows/dummy.yml + (( workflow_permissions("dummy-extension") )) + """)) + return template_path + + +class TestWorkflowPermissions: + @staticmethod + def test_when_permissions_empty(macro_template): + yaml_renderer = YamlRenderer( + github_template_dict={ + "custom_workflows": { + "dummy-extension": SimpleNamespace(permissions={}), + } + }, + file_path=macro_template, + ) + + yaml_dict = yaml_renderer.get_yaml_dict() + assert "permissions" not in yaml_dict["jobs"]["macro-job"] + + @staticmethod + def test_permissions_are_set(macro_template): + yaml_renderer = YamlRenderer( + github_template_dict={ + "custom_workflows": { + "dummy-extension": SimpleNamespace( + permissions={ + "contents": "write", + "packages": "read", + } + ), + } + }, + file_path=macro_template, + ) + + yaml_dict = yaml_renderer.get_yaml_dict() + assert yaml_dict["jobs"]["macro-job"]["permissions"] == { + "contents": "write", + "packages": "read", + } diff --git a/test/unit/util/workflows/render_yaml_test.py b/test/unit/util/workflows/render_yaml_test.py index 61611c05e..7f0606f6f 100644 --- a/test/unit/util/workflows/render_yaml_test.py +++ b/test/unit/util/workflows/render_yaml_test.py @@ -521,94 +521,6 @@ def test_includes_extension_with_multiple_secrets(test_yml, project_config): yaml_dict = yaml_renderer.get_yaml_dict() assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) - @staticmethod - def test_includes_extension_with_custom_permissions(test_yml, project_config): - input_yaml = """ - jobs: - run-unit-tests: - name: Unit Tests (Python-${{ matrix.python-versions }}) - runs-on: "(( os_version ))" - permissions: - contents: read - strategy: - fail-fast: false - matrix: - python-versions: (( python_versions | tojson )) - - steps: - - name: Check out Repository - id: check-out-repository - uses: actions/checkout@v6 - - (% if custom_workflows["fast-tests-extension"].exists %) - fast-tests-extension: - name: Extension - uses: ./.github/workflows/fast-tests-extension.yml - (% if custom_workflows["fast-tests-extension"].permissions %) - permissions: - (% for permission_name, permission_level in custom_workflows["fast-tests-extension"].permissions.items() %) - (( permission_name )): (( permission_level )) - (% endfor %) - (% endif %) - (% endif %) - - """ - expected_yaml = """ - jobs: - run-unit-tests: - name: Unit Tests (Python-${{ matrix.python-versions }}) - runs-on: "ubuntu-24.04" - permissions: - contents: read - strategy: - fail-fast: false - matrix: - python-versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] - - steps: - - name: Check out Repository - id: check-out-repository - uses: actions/checkout@v6 - - fast-tests-extension: - name: Extension - uses: ./.github/workflows/fast-tests-extension.yml - permissions: - contents: write - packages: read - id-token: write - - """ - workflow_directory = project_config.github_workflow_directory - workflow_directory.mkdir(parents=True, exist_ok=True) - (workflow_directory / "fast-tests-extension.yml").write_text(cleandoc(""" - name: Fast-Tests-Extension - - on: - workflow_call: - - jobs: - build-artifacts: - permissions: - contents: read - packages: read - publish-artifacts: - permissions: - contents: write - id-token: write - """)) - - content = cleandoc(input_yaml) - test_yml.write_text(content) - - yaml_renderer = YamlRenderer( - github_template_dict=project_config.github_template_dict, - file_path=test_yml, - ) - - yaml_dict = yaml_renderer.get_yaml_dict() - assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) - @staticmethod def test_jinja_variable_unknown(test_yml, yaml_renderer): input_yaml = """ diff --git a/test/unit/util/workflows/workflow_orchestrator_test.py b/test/unit/util/workflows/workflow_orchestrator_test.py index 6758312f5..e88a4a14a 100644 --- a/test/unit/util/workflows/workflow_orchestrator_test.py +++ b/test/unit/util/workflows/workflow_orchestrator_test.py @@ -20,9 +20,15 @@ def _remove_header(template_text: str) -> str: """ - Remove the Jinja header placeholder line from a workflow template. + Remove leading Jinja-only lines from a workflow template. """ - return template_text.split("\n", 1)[1].strip() + lines = template_text.splitlines() + while lines and ( + lines[0].strip() == "(( workflow_header ))" + or lines[0].strip().startswith('(% from "_workflow_macros.j2" import ') + ): + lines.pop(0) + return "\n".join(lines).strip() class TestTemplates: From cfc71fe5815673355bc52a44091082866f4a1215 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 14 Jul 2026 14:01:17 +0200 Subject: [PATCH 15/24] Introduce macro for workflow_passed_secrets --- .../github/workflows/_workflow_macros.j2 | 18 ++++++- .../toolbox/templates/github/workflows/cd.yml | 11 ++--- .../toolbox/templates/github/workflows/ci.yml | 12 ++--- .../templates/github/workflows/fast-tests.yml | 11 ++--- .../templates/github/workflows/merge-gate.yml | 22 ++------- .../github/workflows/periodic-validation.yml | 12 ++--- .../util/workflows/render_yaml_macro_test.py | 49 +++++++++++++++++-- 7 files changed, 79 insertions(+), 56 deletions(-) diff --git a/exasol/toolbox/templates/github/workflows/_workflow_macros.j2 b/exasol/toolbox/templates/github/workflows/_workflow_macros.j2 index f688805f0..77084476c 100644 --- a/exasol/toolbox/templates/github/workflows/_workflow_macros.j2 +++ b/exasol/toolbox/templates/github/workflows/_workflow_macros.j2 @@ -1,9 +1,25 @@ (% macro workflow_permissions(workflow_name) -%) (% set permissions = custom_workflows[workflow_name].permissions -%) (% if permissions -%) +(% set rendered_permissions -%) permissions: (% for permission_name, permission_level in permissions.items() %) - (( permission_name )): (( permission_level )) + (( permission_name )): (( permission_level )) (% endfor -%) +(% endset -%) +(( rendered_permissions | trim | indent(width=4, first=true) )) (% endif -%) (% endmacro %) + +(% macro workflow_passed_secrets(workflow_name) -%) +(% set secrets = custom_workflows[workflow_name].secrets -%) +(% if secrets -%) +(% set rendered_secrets -%) +secrets: +(% for secret_name in secrets %) + (( secret_name )): ${{ secrets.(( secret_name )) }} +(% endfor -%) +(% endset -%) +(( rendered_secrets | trim | indent(width=4, first=true) )) +(% endif %) +(% endmacro %) diff --git a/exasol/toolbox/templates/github/workflows/cd.yml b/exasol/toolbox/templates/github/workflows/cd.yml index 882e52126..927592dd1 100644 --- a/exasol/toolbox/templates/github/workflows/cd.yml +++ b/exasol/toolbox/templates/github/workflows/cd.yml @@ -1,5 +1,5 @@ (( workflow_header )) -(% from "_workflow_macros.j2" import workflow_permissions with context %) +(% from "_workflow_macros.j2" import workflow_passed_secrets, workflow_permissions with context %) name: CD on: @@ -33,13 +33,8 @@ jobs: needs: - check-release-tag - build-and-publish - (% if custom_workflows["cd-extension"].secrets %) - secrets: - (% for secret_name in custom_workflows["cd-extension"].secrets %) - (( secret_name )): ${{ secrets.(( secret_name )) }} - (% endfor %) - (% endif %) - (( workflow_permissions("cd-extension") | trim )) +(( workflow_passed_secrets("cd-extension") -)) +(( workflow_permissions("cd-extension") )) (% endif %) (% if has_documentation %) diff --git a/exasol/toolbox/templates/github/workflows/ci.yml b/exasol/toolbox/templates/github/workflows/ci.yml index 6a12b950e..d7cd53bb5 100644 --- a/exasol/toolbox/templates/github/workflows/ci.yml +++ b/exasol/toolbox/templates/github/workflows/ci.yml @@ -1,5 +1,5 @@ (( workflow_header )) -(% from "_workflow_macros.j2" import workflow_permissions with context %) +(% from "_workflow_macros.j2" import workflow_passed_secrets, workflow_permissions with context %) name: CI on: @@ -10,14 +10,8 @@ jobs: merge-gate: name: Merge Gate uses: ./.github/workflows/merge-gate.yml - (% if custom_workflows["merge-gate"].secrets %) - secrets: - (% for secret_name in custom_workflows["merge-gate"].secrets %) - (( secret_name )): ${{ secrets.(( secret_name )) }} - (% endfor %) - (% endif %) - (( workflow_permissions("merge-gate") | trim )) - +(( workflow_passed_secrets("merge-gate") -)) +(( workflow_permissions("merge-gate") )) report: name: Report needs: diff --git a/exasol/toolbox/templates/github/workflows/fast-tests.yml b/exasol/toolbox/templates/github/workflows/fast-tests.yml index d14dac414..990c6aece 100644 --- a/exasol/toolbox/templates/github/workflows/fast-tests.yml +++ b/exasol/toolbox/templates/github/workflows/fast-tests.yml @@ -1,5 +1,5 @@ (( workflow_header )) -(% from "_workflow_macros.j2" import workflow_permissions with context %) +(% from "_workflow_macros.j2" import workflow_passed_secrets, workflow_permissions with context %) name: Fast-Tests on: @@ -47,11 +47,6 @@ jobs: fast-tests-extension: name: Extension uses: ./.github/workflows/fast-tests-extension.yml - (% if custom_workflows["fast-tests-extension"].secrets %) - secrets: - (% for secret_name in custom_workflows["fast-tests-extension"].secrets %) - (( secret_name )): ${{ secrets.(( secret_name )) }} - (% endfor %) - (% endif %) - (( workflow_permissions("fast-tests-extension") | trim )) +(( workflow_passed_secrets("fast-tests-extension") -)) +(( workflow_permissions("fast-tests-extension") )) (% endif %) diff --git a/exasol/toolbox/templates/github/workflows/merge-gate.yml b/exasol/toolbox/templates/github/workflows/merge-gate.yml index b17d64e9f..c549d3f90 100644 --- a/exasol/toolbox/templates/github/workflows/merge-gate.yml +++ b/exasol/toolbox/templates/github/workflows/merge-gate.yml @@ -1,5 +1,5 @@ (( workflow_header )) -(% from "_workflow_macros.j2" import workflow_permissions with context %) +(% from "_workflow_macros.j2" import workflow_passed_secrets, workflow_permissions with context %) name: Merge-Gate on: @@ -54,27 +54,15 @@ jobs: needs: - approve-run-slow-tests uses: ./.github/workflows/slow-checks.yml - (% if custom_workflows["slow-checks"].secrets %) - secrets: - (% for secret_name in custom_workflows["slow-checks"].secrets %) - (( secret_name )): ${{ secrets.(( secret_name )) }} - (% endfor %) - (% endif %) - (( workflow_permissions("slow-checks") | trim )) - +(( workflow_passed_secrets("slow-checks") -)) +(( workflow_permissions("slow-checks") )) (% if custom_workflows["merge-gate-extension"].exists %) merge-gate-extension: name: Extension uses: ./.github/workflows/merge-gate-extension.yml - (% if custom_workflows["merge-gate-extension"].secrets %) - secrets: - (% for secret_name in custom_workflows["merge-gate-extension"].secrets %) - (( secret_name )): ${{ secrets.(( secret_name )) }} - (% endfor %) - (% endif %) - (( workflow_permissions("merge-gate-extension") | trim )) +(( workflow_passed_secrets("merge-gate-extension") -)) +(( workflow_permissions("merge-gate-extension") )) (% endif %) - # This job ensures inputs have been executed successfully. allow-merge: name: Allow Merge diff --git a/exasol/toolbox/templates/github/workflows/periodic-validation.yml b/exasol/toolbox/templates/github/workflows/periodic-validation.yml index 09d83b4dc..a78acf752 100644 --- a/exasol/toolbox/templates/github/workflows/periodic-validation.yml +++ b/exasol/toolbox/templates/github/workflows/periodic-validation.yml @@ -1,5 +1,5 @@ (( workflow_header )) -(% from "_workflow_macros.j2" import workflow_permissions with context %) +(% from "_workflow_macros.j2" import workflow_passed_secrets, workflow_permissions with context %) name: Periodic-Validation on: @@ -46,14 +46,8 @@ jobs: uses: ./.github/workflows/slow-checks.yml needs: - restrict-to-default-branch - (% if custom_workflows["slow-checks"].secrets %) - secrets: - (% for secret_name in custom_workflows["slow-checks"].secrets %) - (( secret_name )): ${{ secrets.(( secret_name )) }} - (% endfor %) - (% endif %) - (( workflow_permissions("slow-checks") | trim )) - +(( workflow_passed_secrets("slow-checks") -)) +(( workflow_permissions("slow-checks") )) report: name: Report needs: diff --git a/test/unit/util/workflows/render_yaml_macro_test.py b/test/unit/util/workflows/render_yaml_macro_test.py index 932fc153f..6865b630a 100644 --- a/test/unit/util/workflows/render_yaml_macro_test.py +++ b/test/unit/util/workflows/render_yaml_macro_test.py @@ -8,18 +8,21 @@ @pytest.fixture -def macro_template(tmp_path: Path) -> Path: +def macro_template(tmp_path: Path, macro_input: str) -> Path: template_path = tmp_path / "dummy.yml" template_path.write_text(cleandoc(""" - (% from "_workflow_macros.j2" import workflow_permissions with context %) + (% from "_workflow_macros.j2" import workflow_passed_secrets, workflow_permissions with context %) jobs: macro-job: uses: ./.github/workflows/dummy.yml - (( workflow_permissions("dummy-extension") )) - """)) + {macro_input} + """).format(macro_input=macro_input)) return template_path +@pytest.mark.parametrize( + "macro_input", ['(( workflow_permissions("dummy-extension") ))'] +) class TestWorkflowPermissions: @staticmethod def test_when_permissions_empty(macro_template): @@ -56,3 +59,41 @@ def test_permissions_are_set(macro_template): "contents": "write", "packages": "read", } + + +@pytest.mark.parametrize( + "macro_input", ['(( workflow_passed_secrets("dummy-extension") -))'] +) +class TestWorkflowPassedSecrets: + @staticmethod + def test_when_secrets_empty(macro_template): + yaml_renderer = YamlRenderer( + github_template_dict={ + "custom_workflows": { + "dummy-extension": SimpleNamespace(secrets=()), + } + }, + file_path=macro_template, + ) + + yaml_dict = yaml_renderer.get_yaml_dict() + assert "secrets" not in yaml_dict["jobs"]["macro-job"] + + @staticmethod + def test_when_secrets_are_set(macro_template): + yaml_renderer = YamlRenderer( + github_template_dict={ + "custom_workflows": { + "dummy-extension": SimpleNamespace( + secrets=("SECRET_A", "SECRET_B"), + ), + } + }, + file_path=macro_template, + ) + + yaml_dict = yaml_renderer.get_yaml_dict() + assert yaml_dict["jobs"]["macro-job"]["secrets"] == { + "SECRET_A": "${{ secrets.SECRET_A }}", + "SECRET_B": "${{ secrets.SECRET_B }}", + } From 29d22ac2dcd8a13ca851f2f2464fb75bb8249142 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 14 Jul 2026 14:37:53 +0200 Subject: [PATCH 16/24] Reduce enum usage --- .../toolbox/util/workflows/custom_workflow.py | 35 ++++++------------- 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/exasol/toolbox/util/workflows/custom_workflow.py b/exasol/toolbox/util/workflows/custom_workflow.py index 941142e0e..62b01661a 100644 --- a/exasol/toolbox/util/workflows/custom_workflow.py +++ b/exasol/toolbox/util/workflows/custom_workflow.py @@ -1,7 +1,10 @@ from __future__ import annotations -from enum import IntEnum from pathlib import Path +from typing import ( + Literal, + TypeAlias, +) from pydantic import ( BaseModel, @@ -13,34 +16,18 @@ from exasol.toolbox.util.workflows.render_yaml import parse_yaml_text - -class PermissionRank(IntEnum): - """GitHub permission levels ranked from least to most permissive.""" - - NONE = 0 - READ = 1 - WRITE = 2 - - @classmethod - def _missing_(cls, value: object) -> PermissionRank: - """Convert a GitHub permission string to its rank.""" - if isinstance(value, str): - for rank in cls: - if rank.name.lower() == value: - return rank - raise ValueError(f"Unknown GitHub permission level: {value!r}") +PermissionLevel: TypeAlias = Literal["none", "read", "write"] +PERMISSION_RANK: dict[PermissionLevel, int] = {"none": 0, "read": 1, "write": 2} +Permissions: TypeAlias = dict[str, PermissionLevel] -def merge_permissions(permission_maps: list[dict[str, str]]) -> dict[str, str]: +def merge_permissions(permission_maps: list[Permissions]) -> Permissions: """Merge permission maps to keep the greater permission.""" - merged_permissions: dict[str, str] = {} + merged_permissions: Permissions = {} for permission_map in permission_maps: for permission_name, requested_level in permission_map.items(): - current_level = merged_permissions.get(permission_name, None) - if current_level is None: - merged_permissions[permission_name] = requested_level - continue - if PermissionRank(requested_level) > PermissionRank(current_level): # type: ignore[arg-type] + current_level = merged_permissions.get(permission_name, "none") + if PERMISSION_RANK[requested_level] > PERMISSION_RANK[current_level]: merged_permissions[permission_name] = requested_level return merged_permissions From 98dbec050207d6cf9dfd882bdc30e7a4e908686b Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 14 Jul 2026 14:43:36 +0200 Subject: [PATCH 17/24] Resolve sonar finding --- exasol/toolbox/config.py | 37 +++++++++++++------ exasol/toolbox/util/workflows/render_yaml.py | 8 +++- test/unit/config_test.py | 2 +- .../workflows/workflow_orchestrator_test.py | 11 +++--- 4 files changed, 38 insertions(+), 20 deletions(-) diff --git a/exasol/toolbox/config.py b/exasol/toolbox/config.py index 38bcd5169..a67661317 100644 --- a/exasol/toolbox/config.py +++ b/exasol/toolbox/config.py @@ -38,6 +38,21 @@ WORKFLOW_HEADER_PATTERN = rf"\A{re.escape(WORKFLOW_HEADER_PREFIX)}[^\n]+\.\n" +class GithubTemplateContext(BaseModel): + """Immutable template values exposed to Jinja workflow rendering.""" + + model_config = ConfigDict(frozen=True) + + custom_workflows: dict[str, Any] + dependency_manager_version: str + has_documentation: bool + minimum_python_version: str + os_version: str + python_versions: tuple[str, ...] + sonar_token_name: str + workflow_header: str + + def get_methods_with_hook_implementation( plugin_class: type[Any], ) -> tuple[tuple[str, Callable], ...]: @@ -323,7 +338,7 @@ def github_workflow_directory(self) -> Path: @computed_field # type: ignore[misc] @property - def github_template_dict(self) -> dict[str, Any]: + def github_template_dict(self) -> GithubTemplateContext: """ Dictionary of variables to dynamically render Jinja2 templates into valid YAML configurations. @@ -333,16 +348,16 @@ def github_template_dict(self) -> dict[str, Any]: sonar_token_name=self.sonar_token_name, ) - return { - "custom_workflows": custom_workflow_extractor.build_custom_workflow_dict(), - "dependency_manager_version": self.dependency_manager.version, - "has_documentation": self.has_documentation, - "minimum_python_version": self.minimum_python_version, - "os_version": self.os_version, - "python_versions": self.python_versions, - "sonar_token_name": self.sonar_token_name, - "workflow_header": f"{WORKFLOW_HEADER_PREFIX}{__version__}.", - } + return GithubTemplateContext( + custom_workflows=custom_workflow_extractor.build_custom_workflow_dict(), + dependency_manager_version=self.dependency_manager.version, + has_documentation=self.has_documentation, + minimum_python_version=self.minimum_python_version, + os_version=self.os_version, + python_versions=self.python_versions, + sonar_token_name=self.sonar_token_name, + workflow_header=f"{WORKFLOW_HEADER_PREFIX}{__version__}.", + ) @computed_field # type: ignore[misc] @property diff --git a/exasol/toolbox/util/workflows/render_yaml.py b/exasol/toolbox/util/workflows/render_yaml.py index b89ab8202..b9f161429 100644 --- a/exasol/toolbox/util/workflows/render_yaml.py +++ b/exasol/toolbox/util/workflows/render_yaml.py @@ -89,10 +89,14 @@ def _render_with_jinja(self, input_str: str) -> str: logger.debug( "Render template with Jinja", jinja_dict_source="PROJECT_CONFIG.github_template_dict", - jinja_dict_values=self.github_template_dict, ) jinja_template = build_jinja_env(self.file_path).from_string(input_str) - return jinja_template.render(self.github_template_dict) + render_context = ( + self.github_template_dict.model_dump() + if hasattr(self.github_template_dict, "model_dump") + else self.github_template_dict + ) + return jinja_template.render(**render_context) def get_yaml_dict(self) -> CommentedMap: """ diff --git a/test/unit/config_test.py b/test/unit/config_test.py index 5fa06445b..cf86c057b 100644 --- a/test/unit/config_test.py +++ b/test/unit/config_test.py @@ -205,7 +205,7 @@ def test_sonar_token_name_can_be_overridden(tmp_path): config = AlternateSonarConfig(project_name="test", root_path=tmp_path) assert config.sonar_token_name == "SONAR_ANOTHER_TOKEN" - assert config.github_template_dict["sonar_token_name"] == "SONAR_ANOTHER_TOKEN" + assert config.github_template_dict.sonar_token_name == "SONAR_ANOTHER_TOKEN" @pytest.mark.parametrize("minimum_python_version", ["3.10", "3.10.5"]) diff --git a/test/unit/util/workflows/workflow_orchestrator_test.py b/test/unit/util/workflows/workflow_orchestrator_test.py index e88a4a14a..877c4e8a7 100644 --- a/test/unit/util/workflows/workflow_orchestrator_test.py +++ b/test/unit/util/workflows/workflow_orchestrator_test.py @@ -190,14 +190,13 @@ def test_raises_invalidworkflowpatcherentryerror(project_config): - unknown-job """ project_config.github_workflow_patcher_yaml.write_text(patcher_yml) + orchestrator = WorkflowOrchestrator( + workflow_choice="checks", + config=project_config, + ) with pytest.raises(InvalidWorkflowPatcherEntryError) as ex: - orchestrator = WorkflowOrchestrator( - workflow_choice="checks", - config=project_config, - ) - for _ in orchestrator._iter_workflows(orchestrator.templates): - pass + list(orchestrator._iter_workflows(orchestrator.templates)) assert ( f"In file '{project_config.github_workflow_patcher_yaml}', " From 3663d2b1d39e0da75f9dadfc0bd7a8ded46d55bb Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 14 Jul 2026 15:17:04 +0200 Subject: [PATCH 18/24] Fix type hints --- exasol/toolbox/config.py | 24 +++----------- .../toolbox/util/workflows/custom_workflow.py | 2 +- .../workflows/custom_workflow_extractor.py | 11 +++---- exasol/toolbox/util/workflows/render_yaml.py | 31 ++++++++++++++----- exasol/toolbox/util/workflows/workflow.py | 6 ++-- 5 files changed, 37 insertions(+), 37 deletions(-) diff --git a/exasol/toolbox/config.py b/exasol/toolbox/config.py index a67661317..9e4b010ae 100644 --- a/exasol/toolbox/config.py +++ b/exasol/toolbox/config.py @@ -26,33 +26,15 @@ PLUGIN_ATTR_NAME, ) from exasol.toolbox.util.version import Version -from exasol.toolbox.util.workflows.custom_workflow_extractor import ( - CustomWorkflowExtractor, -) +from exasol.toolbox.util.workflows.render_yaml import GithubTemplateContext WORKFLOW_HEADER_PREFIX = ( "# Generated and maintained by the exasol-toolbox.\n" "# Last generated with exasol-toolbox version " ) - WORKFLOW_HEADER_PATTERN = rf"\A{re.escape(WORKFLOW_HEADER_PREFIX)}[^\n]+\.\n" -class GithubTemplateContext(BaseModel): - """Immutable template values exposed to Jinja workflow rendering.""" - - model_config = ConfigDict(frozen=True) - - custom_workflows: dict[str, Any] - dependency_manager_version: str - has_documentation: bool - minimum_python_version: str - os_version: str - python_versions: tuple[str, ...] - sonar_token_name: str - workflow_header: str - - def get_methods_with_hook_implementation( plugin_class: type[Any], ) -> tuple[tuple[str, Callable], ...]: @@ -343,6 +325,10 @@ def github_template_dict(self) -> GithubTemplateContext: Dictionary of variables to dynamically render Jinja2 templates into valid YAML configurations. """ + from exasol.toolbox.util.workflows.custom_workflow_extractor import ( + CustomWorkflowExtractor, + ) + custom_workflow_extractor = CustomWorkflowExtractor( github_workflow_directory=self.github_workflow_directory, sonar_token_name=self.sonar_token_name, diff --git a/exasol/toolbox/util/workflows/custom_workflow.py b/exasol/toolbox/util/workflows/custom_workflow.py index 62b01661a..214d8a0f8 100644 --- a/exasol/toolbox/util/workflows/custom_workflow.py +++ b/exasol/toolbox/util/workflows/custom_workflow.py @@ -73,7 +73,7 @@ def extract_secrets(self) -> tuple[str, ...]: return tuple(secrets.keys()) return () - def extract_permissions(self) -> dict[str, str]: + def extract_permissions(self) -> Permissions: """Return the effective job permissions required by the workflow. The extractor scans all jobs and merges their ``permissions`` blocks into a diff --git a/exasol/toolbox/util/workflows/custom_workflow_extractor.py b/exasol/toolbox/util/workflows/custom_workflow_extractor.py index 8637fd7e3..8c5ce32e6 100644 --- a/exasol/toolbox/util/workflows/custom_workflow_extractor.py +++ b/exasol/toolbox/util/workflows/custom_workflow_extractor.py @@ -10,10 +10,11 @@ from exasol.toolbox.util.workflows.custom_workflow import ( CustomWorkflow, + Permissions, merge_permissions, ) -MINIMUM_GITHUB_PERMISSIONS: dict[str, str] = {"contents": "read"} +MINIMUM_GITHUB_PERMISSIONS: Permissions = {"contents": "read"} class CustomWorkflowEntry(BaseModel): @@ -21,7 +22,7 @@ class CustomWorkflowEntry(BaseModel): exists: bool secrets: tuple[str, ...] - permissions: dict[str, str] + permissions: Permissions @field_validator("secrets", mode="before") @classmethod @@ -31,9 +32,7 @@ def _normalize_secrets(cls, secrets: tuple[str, ...]) -> list[str]: @field_validator("permissions", mode="before") @classmethod - def _normalize_permissions( - cls, permissions: list[dict[str, str]] - ) -> dict[str, str]: + def _normalize_permissions(cls, permissions: list[Permissions]) -> Permissions: """Merge permission maps while preserving first-seen order.""" return merge_permissions(permissions) @@ -57,7 +56,7 @@ def _build_custom_workflow_entry( file_path = self.github_workflow_directory / f"{workflow}.yml" secrets: tuple[str, ...] = () - permissions: list[dict[str, str]] = [] + permissions: list[Permissions] = [] if file_path.is_file(): custom_workflow = CustomWorkflow.load_from_file(file_path=file_path) secrets = custom_workflow.extract_secrets() diff --git a/exasol/toolbox/util/workflows/render_yaml.py b/exasol/toolbox/util/workflows/render_yaml.py index b9f161429..aba7e24cb 100644 --- a/exasol/toolbox/util/workflows/render_yaml.py +++ b/exasol/toolbox/util/workflows/render_yaml.py @@ -2,7 +2,6 @@ from dataclasses import dataclass from inspect import cleandoc from pathlib import Path -from typing import Any from jinja2 import ( Environment, @@ -10,6 +9,10 @@ StrictUndefined, TemplateError, ) +from pydantic import ( + BaseModel, + ConfigDict, +) from ruamel.yaml import ( YAML, CommentedMap, @@ -25,6 +28,21 @@ from exasol.toolbox.util.workflows.templates import WORKFLOW_TEMPLATE_OPTIONS +class GithubTemplateContext(BaseModel): + """Immutable template values exposed to Jinja workflow rendering.""" + + model_config = ConfigDict(frozen=True) + + custom_workflows: dict[str, object] + dependency_manager_version: str + has_documentation: bool + minimum_python_version: str + os_version: str + python_versions: tuple[str, ...] + sonar_token_name: str + workflow_header: str + + def build_jinja_env(template_path: Path) -> Environment: """ Create a Jinja environment for a workflow template file. @@ -79,7 +97,7 @@ class YamlRenderer: YAML files. """ - github_template_dict: dict[str, Any] + github_template_dict: GithubTemplateContext file_path: Path def _render_with_jinja(self, input_str: str) -> str: @@ -91,11 +109,10 @@ def _render_with_jinja(self, input_str: str) -> str: jinja_dict_source="PROJECT_CONFIG.github_template_dict", ) jinja_template = build_jinja_env(self.file_path).from_string(input_str) - render_context = ( - self.github_template_dict.model_dump() - if hasattr(self.github_template_dict, "model_dump") - else self.github_template_dict - ) + if isinstance(self.github_template_dict, dict): + render_context = self.github_template_dict + else: + render_context = self.github_template_dict.model_dump() return jinja_template.render(**render_context) def get_yaml_dict(self) -> CommentedMap: diff --git a/exasol/toolbox/util/workflows/workflow.py b/exasol/toolbox/util/workflows/workflow.py index 8b6d73199..3b9143969 100644 --- a/exasol/toolbox/util/workflows/workflow.py +++ b/exasol/toolbox/util/workflows/workflow.py @@ -1,9 +1,6 @@ import difflib import re from pathlib import Path -from typing import ( - Any, -) from pydantic import ( BaseModel, @@ -23,6 +20,7 @@ WorkflowCommentedMap, ) from exasol.toolbox.util.workflows.process_template import WorkflowRenderer +from exasol.toolbox.util.workflows.render_yaml import GithubTemplateContext class Workflow(BaseModel): @@ -57,7 +55,7 @@ def load_from_template( cls, template_path: Path, output_directory: Path, - github_template_dict: dict[str, Any], + github_template_dict: GithubTemplateContext, patch_yaml: WorkflowCommentedMap | None = None, ): with bound_contextvars(template_file_name=template_path.name): From 5087874a8806613884f368402c49de428a727dad Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 14 Jul 2026 15:34:43 +0200 Subject: [PATCH 19/24] Resolve sonar finding - attempt 2 --- test/unit/util/workflows/workflow_orchestrator_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/unit/util/workflows/workflow_orchestrator_test.py b/test/unit/util/workflows/workflow_orchestrator_test.py index 877c4e8a7..e7b5ceff2 100644 --- a/test/unit/util/workflows/workflow_orchestrator_test.py +++ b/test/unit/util/workflows/workflow_orchestrator_test.py @@ -194,9 +194,10 @@ def test_raises_invalidworkflowpatcherentryerror(project_config): workflow_choice="checks", config=project_config, ) + workflow_iter = orchestrator._iter_workflows(orchestrator.templates) with pytest.raises(InvalidWorkflowPatcherEntryError) as ex: - list(orchestrator._iter_workflows(orchestrator.templates)) + next(workflow_iter) assert ( f"In file '{project_config.github_workflow_patcher_yaml}', " From d9b094c405f5c8a7473cf58d785bf103bc974ff3 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 14 Jul 2026 15:42:04 +0200 Subject: [PATCH 20/24] Set autoescape=False as unneeded --- exasol/toolbox/util/workflows/render_yaml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol/toolbox/util/workflows/render_yaml.py b/exasol/toolbox/util/workflows/render_yaml.py index aba7e24cb..63d75dbbf 100644 --- a/exasol/toolbox/util/workflows/render_yaml.py +++ b/exasol/toolbox/util/workflows/render_yaml.py @@ -52,7 +52,7 @@ def build_jinja_env(template_path: Path) -> Environment: loader=FileSystemLoader([template_path.parent, workflow_template_directory]), variable_start_string="((", variable_end_string="))", - autoescape=True, + autoescape=False, # This requires that all Jinja variables must be defined in the provided # dictionary. If not, then a `jinja2.exceptions.UndefinedError` exception # will be raised. From 85d28739422db836b54f3d8e3be60bcb410d5819 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 14 Jul 2026 15:48:55 +0200 Subject: [PATCH 21/24] Simplify tests as already covered --- test/unit/util/workflows/render_yaml_test.py | 100 +------------------ 1 file changed, 1 insertion(+), 99 deletions(-) diff --git a/test/unit/util/workflows/render_yaml_test.py b/test/unit/util/workflows/render_yaml_test.py index 7f0606f6f..b3c4d6a94 100644 --- a/test/unit/util/workflows/render_yaml_test.py +++ b/test/unit/util/workflows/render_yaml_test.py @@ -377,16 +377,7 @@ def test_includes_if_block_when_extension_is_present(test_yml, project_config): fast-tests-extension: name: Extension uses: ./.github/workflows/fast-tests-extension.yml - (% if custom_workflows["fast-tests-extension"].secrets %) - secrets: - (% for secret_name in custom_workflows["fast-tests-extension"].secrets %) - (( secret_name )): ${{ secrets.(( secret_name )) }} - (% endfor %) - (% endif %) - permissions: - contents: read - (% endif %) - + (% endif %) """ expected_yaml = """ jobs: @@ -408,11 +399,6 @@ def test_includes_if_block_when_extension_is_present(test_yml, project_config): fast-tests-extension: name: Extension uses: ./.github/workflows/fast-tests-extension.yml - secrets: - FAST_TEST_SECRET: ${{ secrets.FAST_TEST_SECRET }} - permissions: - contents: read - """ workflow_directory = project_config.github_workflow_directory workflow_directory.mkdir(parents=True) @@ -437,90 +423,6 @@ def test_includes_if_block_when_extension_is_present(test_yml, project_config): assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) - @staticmethod - def test_includes_extension_with_multiple_secrets(test_yml, project_config): - input_yaml = """ - jobs: - run-unit-tests: - name: Unit Tests (Python-${{ matrix.python-versions }}) - runs-on: "(( os_version ))" - permissions: - contents: read - strategy: - fail-fast: false - matrix: - python-versions: (( python_versions | tojson )) - - steps: - - name: Check out Repository - id: check-out-repository - uses: actions/checkout@v6 - - (% if custom_workflows["merge-gate"].exists %) - merge-gate-extension: - uses: ./.github/workflows/merge-gate-extension.yml - (% if custom_workflows["merge-gate-extension"].secrets %) - secrets: - (% for secret_name in custom_workflows["merge-gate-extension"].secrets %) - (( secret_name )): ${{ secrets.(( secret_name )) }} - (% endfor %) - (% endif %) - permissions: - contents: read - (% endif %) - - """ - expected_yaml = """ - jobs: - run-unit-tests: - name: Unit Tests (Python-${{ matrix.python-versions }}) - runs-on: "ubuntu-24.04" - permissions: - contents: read - strategy: - fail-fast: false - matrix: - python-versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] - - steps: - - name: Check out Repository - id: check-out-repository - uses: actions/checkout@v6 - - merge-gate-extension: - uses: ./.github/workflows/merge-gate-extension.yml - secrets: - ANOTHER_SECRET: ${{ secrets.ANOTHER_SECRET }} - MERGE_GATE_SECRET: ${{ secrets.MERGE_GATE_SECRET }} - permissions: - contents: read - - """ - workflow_directory = project_config.github_workflow_directory - workflow_directory.mkdir(parents=True) - (workflow_directory / "merge-gate-extension.yml").write_text(cleandoc(""" - name: Merge-Gate-Extension - - on: - workflow_call: - secrets: - MERGE_GATE_SECRET: - required: true - ANOTHER_SECRET: - required: true - """)) - - content = cleandoc(input_yaml) - test_yml.write_text(content) - - yaml_renderer = YamlRenderer( - github_template_dict=project_config.github_template_dict, - file_path=test_yml, - ) - - yaml_dict = yaml_renderer.get_yaml_dict() - assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) - @staticmethod def test_jinja_variable_unknown(test_yml, yaml_renderer): input_yaml = """ From f9680a81874ce93b50ec88ff16561d22c52e31ee Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 14 Jul 2026 15:52:25 +0200 Subject: [PATCH 22/24] Resolve sonar finding - attempt 3 --- exasol/toolbox/util/workflows/render_yaml.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/exasol/toolbox/util/workflows/render_yaml.py b/exasol/toolbox/util/workflows/render_yaml.py index 63d75dbbf..bdd95e106 100644 --- a/exasol/toolbox/util/workflows/render_yaml.py +++ b/exasol/toolbox/util/workflows/render_yaml.py @@ -8,6 +8,7 @@ FileSystemLoader, StrictUndefined, TemplateError, + select_autoescape, ) from pydantic import ( BaseModel, @@ -52,7 +53,14 @@ def build_jinja_env(template_path: Path) -> Environment: loader=FileSystemLoader([template_path.parent, workflow_template_directory]), variable_start_string="((", variable_end_string="))", - autoescape=False, + # Workflow templates are YAML-only. We still route through select_autoescape + # so the Jinja environment has an explicit escaping policy, but no file + # extensions are treated as HTML/XML templates here. + autoescape=select_autoescape( + enabled_extensions=(), + default_for_string=False, + default=False, + ), # This requires that all Jinja variables must be defined in the provided # dictionary. If not, then a `jinja2.exceptions.UndefinedError` exception # will be raised. From 923c88e7686a3d17c720a3d4b70101cb382e6e3b Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 14 Jul 2026 15:56:29 +0200 Subject: [PATCH 23/24] Resolve sonar finding - attempt 4 --- exasol/toolbox/util/workflows/render_yaml.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/exasol/toolbox/util/workflows/render_yaml.py b/exasol/toolbox/util/workflows/render_yaml.py index bdd95e106..19b7e9c83 100644 --- a/exasol/toolbox/util/workflows/render_yaml.py +++ b/exasol/toolbox/util/workflows/render_yaml.py @@ -108,7 +108,7 @@ class YamlRenderer: github_template_dict: GithubTemplateContext file_path: Path - def _render_with_jinja(self, input_str: str) -> str: + def _render_with_jinja(self) -> str: """ Render the template with Jinja. """ @@ -116,7 +116,9 @@ def _render_with_jinja(self, input_str: str) -> str: "Render template with Jinja", jinja_dict_source="PROJECT_CONFIG.github_template_dict", ) - jinja_template = build_jinja_env(self.file_path).from_string(input_str) + jinja_template = build_jinja_env(self.file_path).get_template( + self.file_path.name + ) if isinstance(self.github_template_dict, dict): render_context = self.github_template_dict else: @@ -128,9 +130,8 @@ def get_yaml_dict(self) -> CommentedMap: Load a file as a CommentedMap (dictionary form of a YAML), after rendering it with Jinja. """ - raw_content = self.file_path.read_text() try: - workflow_string = self._render_with_jinja(raw_content) + workflow_string = self._render_with_jinja() except TemplateError as ex: raise TemplateRenderingError(file_path=self.file_path) from ex return parse_yaml_text( From 4e01cdf0018635a68de47e5caff450429528d1ce Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 14 Jul 2026 16:00:41 +0200 Subject: [PATCH 24/24] Resolve new sonar finding --- test/unit/util/workflows/render_yaml_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/util/workflows/render_yaml_test.py b/test/unit/util/workflows/render_yaml_test.py index b3c4d6a94..a521496a5 100644 --- a/test/unit/util/workflows/render_yaml_test.py +++ b/test/unit/util/workflows/render_yaml_test.py @@ -256,13 +256,13 @@ def test_parsing_fails_when_yaml_malformed(tmp_path): - name: SCM Checkout uses: actions/checkout@v5 """ + workflow_string = cleandoc(bad_template) with pytest.raises( YamlParsingError, match="Check for invalid YAML syntax." ) as ex: parse_yaml_text( - origin_path=tmp_path / "dummy.yml", - workflow_string=cleandoc(bad_template), + origin_path=tmp_path / "dummy.yml", workflow_string=workflow_string ) assert isinstance(ex.value.__cause__, ParserError)