From 452c0840e1b933f3163ac066b2f16a7ac423ed26 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 20 Oct 2026 11:14:28 +0100 Subject: [PATCH 1/3] feat(cluster): derive the container runtime from the Docker context A filter that tells Colima, Docker Desktop and any other runtime apart from the Docker context the role drives, and which Colima profile a context belongs to, with the runtime and profile overridable for hosts whose context does not say. It refuses a profile that names another VM than the context, and any profile name that is not a valid Lima instance name. --- plugins/filter/container_runtime.py | 74 ++++++++++++++++++++++ tests/unit/test_container_runtime.py | 91 ++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 plugins/filter/container_runtime.py create mode 100644 tests/unit/test_container_runtime.py diff --git a/plugins/filter/container_runtime.py b/plugins/filter/container_runtime.py new file mode 100644 index 0000000..79e6ddb --- /dev/null +++ b/plugins/filter/container_runtime.py @@ -0,0 +1,74 @@ +"""Filter the github_runner_cluster role uses to tell which container runtime a macOS host's Docker context belongs to, and so whether the role can start that runtime itself.""" + +from __future__ import annotations + +import re +from typing import Any + +from ansible.errors import AnsibleFilterError + +RUNTIME_CHOICES = ("auto", "colima", "docker_desktop", "other") +# Colima names the Docker context of its default profile "colima" and any other profile's "colima-". +COLIMA_CONTEXT = "colima" +COLIMA_DEFAULT_PROFILE = "default" +# Docker Desktop's own context for its Linux engine. +DOCKER_DESKTOP_CONTEXTS = ("desktop-linux",) +# Lima's rule for an instance name, which is what a Colima profile becomes. It also keeps the value free of every character that would need escaping in a launchd property list or on a command line. +PROFILE_PATTERN = re.compile(r"^[A-Za-z0-9]+([._-][A-Za-z0-9]+)*$") + + +def _text(name: str, value: Any) -> str: + if value is None: + return "" + if not isinstance(value, str): + raise AnsibleFilterError(f"{name} must be a string, not {type(value).__name__}: {value!r}.") + return value.strip() + + +def _colima_profile_of(context: str) -> str | None: + """Return the Colima profile a Docker context belongs to, or None when it is not a Colima context.""" + if context == COLIMA_CONTEXT: + return COLIMA_DEFAULT_PROFILE + prefix = COLIMA_CONTEXT + "-" + if context.startswith(prefix) and len(context) > len(prefix): + return context[len(prefix) :] + return None + + +def cluster_container_runtime(context: Any, requested: Any = "auto", colima_profile: Any = "") -> dict[str, str]: + """Work out which container runtime a macOS host's Docker daemon belongs to. + + The role calls it only for a macOS host, the only kind it starts a runtime on. ``auto`` takes the runtime from the Docker context the role drives: a Colima context (``colima`` or ``colima-``) is Colima and ``desktop-linux`` is Docker Desktop, and anything else (including no context because ``DOCKER_HOST`` is set) is ``other``. + + :param context: the Docker context the role drives, or an empty value for none. :param requested: ``auto``, ``colima``, ``docker_desktop`` or ``other``. :param colima_profile: the Colima profile to start, or an empty value to take it from the context (``default`` when the context names none). + :returns: ``{"kind": "colima" | "docker_desktop" | "other", "colima_profile": }``. + :raises AnsibleFilterError: if a value is not a string, ``requested`` is not one of the choices, the profile is not a valid Colima profile name, or it names another profile than the Colima context the role drives. + """ + docker_context = _text("The Docker context", context) + choice = _text("The container runtime", requested) or "auto" + profile = _text("The Colima profile", colima_profile) + if choice not in RUNTIME_CHOICES: + raise AnsibleFilterError(f"The container runtime must be one of {', '.join(RUNTIME_CHOICES)}, not {choice!r}.") + if profile and not PROFILE_PATTERN.match(profile): + raise AnsibleFilterError(f"The Colima profile {profile!r} is not a valid profile name: use letters and digits, separated by single dots, hyphens or underscores.") + context_profile = _colima_profile_of(docker_context) + if choice == "auto": + if context_profile is not None: + choice = "colima" + elif docker_context in DOCKER_DESKTOP_CONTEXTS: + choice = "docker_desktop" + else: + choice = "other" + if choice != "colima": + return {"kind": choice, "colima_profile": ""} + if profile and context_profile is not None and profile != context_profile: + raise AnsibleFilterError(f"The Colima profile {profile!r} is not the one the Docker context {docker_context!r} belongs to ({context_profile!r}); the role would start one VM and drive another.") + return {"kind": "colima", "colima_profile": profile or context_profile or COLIMA_DEFAULT_PROFILE} + + +class FilterModule: + """Registers the container runtime filter with Ansible.""" + + def filters(self) -> dict[str, Any]: + """Return the filters this plugin provides.""" + return {"cluster_container_runtime": cluster_container_runtime} diff --git a/tests/unit/test_container_runtime.py b/tests/unit/test_container_runtime.py new file mode 100644 index 0000000..8eb7690 --- /dev/null +++ b/tests/unit/test_container_runtime.py @@ -0,0 +1,91 @@ +"""Tests for the filter that tells which container runtime a macOS host's Docker context belongs to.""" + +from __future__ import annotations + +import importlib.util +import unittest +from pathlib import Path + +PLUGIN = Path(__file__).resolve().parents[2] / "plugins" / "filter" / "container_runtime.py" +_spec = importlib.util.spec_from_file_location("container_runtime_filters", PLUGIN) +assert _spec is not None and _spec.loader is not None +container_runtime = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(container_runtime) + +runtime = container_runtime.cluster_container_runtime + + +class AutoDetectionTest(unittest.TestCase): + def test_the_default_colima_context_is_the_default_profile(self) -> None: + self.assertEqual(runtime("colima"), {"kind": "colima", "colima_profile": "default"}) + + def test_another_profile_s_context_names_the_profile(self) -> None: + self.assertEqual(runtime("colima-example"), {"kind": "colima", "colima_profile": "example"}) + self.assertEqual(runtime("colima-build-2", "auto"), {"kind": "colima", "colima_profile": "build-2"}) + + def test_docker_desktop_s_context_is_docker_desktop(self) -> None: + self.assertEqual(runtime("desktop-linux"), {"kind": "docker_desktop", "colima_profile": ""}) + + def test_any_other_context_or_none_is_another_runtime(self) -> None: + for context in ("default", "orbstack", "colimas", "colima-", "", None, " "): + with self.subTest(context=context): + self.assertEqual(runtime(context), {"kind": "other", "colima_profile": ""}) + + def test_surrounding_whitespace_is_not_part_of_the_name(self) -> None: + self.assertEqual(runtime(" colima-example\n"), {"kind": "colima", "colima_profile": "example"}) + + +class RequestedRuntimeTest(unittest.TestCase): + def test_colima_can_be_named_for_a_context_that_does_not_say(self) -> None: + self.assertEqual(runtime("", "colima"), {"kind": "colima", "colima_profile": "default"}) + self.assertEqual(runtime("orbstack", "colima", "example"), {"kind": "colima", "colima_profile": "example"}) + + def test_a_named_runtime_wins_over_the_context(self) -> None: + self.assertEqual(runtime("colima", "docker_desktop"), {"kind": "docker_desktop", "colima_profile": ""}) + self.assertEqual(runtime("colima", "other"), {"kind": "other", "colima_profile": ""}) + self.assertEqual(runtime("desktop-linux", "colima"), {"kind": "colima", "colima_profile": "default"}) + + def test_only_colima_has_a_profile(self) -> None: + self.assertEqual(runtime("desktop-linux", "docker_desktop", "example"), {"kind": "docker_desktop", "colima_profile": ""}) + + def test_a_profile_that_agrees_with_the_context_is_accepted(self) -> None: + self.assertEqual(runtime("colima-example", "auto", "example"), {"kind": "colima", "colima_profile": "example"}) + self.assertEqual(runtime("colima", "colima", "default"), {"kind": "colima", "colima_profile": "default"}) + + def test_a_profile_that_names_another_vm_than_the_context_is_an_error(self) -> None: + for context, profile in (("colima-example", "other"), ("colima", "example"), ("colima-example", "default")): + with self.subTest(context=context, profile=profile), self.assertRaises(container_runtime.AnsibleFilterError): + runtime(context, "auto", profile) + + +class InvalidInputTest(unittest.TestCase): + def test_an_unknown_runtime_is_an_error(self) -> None: + for value in ("docker-desktop", "podman", "COLIMA"): + with self.subTest(value=value), self.assertRaises(container_runtime.AnsibleFilterError): + runtime("colima", value) + + def test_an_empty_runtime_is_auto(self) -> None: + self.assertEqual(runtime("colima", ""), runtime("colima", "auto")) + self.assertEqual(runtime("colima", None), runtime("colima", "auto")) + + def test_a_profile_that_is_not_a_lima_instance_name_is_an_error(self) -> None: + for profile in ("a b", "-x", "x-", "a--b", "a/b", "a&b", "a None: + for profile in ("a", "A1", "build-2", "build_2", "build.2", "a-b.c_d"): + with self.subTest(profile=profile): + self.assertEqual(runtime("", "colima", profile), {"kind": "colima", "colima_profile": profile}) + + def test_a_value_that_is_not_a_string_is_an_error(self) -> None: + for args in ((["colima"],), ("colima", ["auto"]), ("colima", "colima", 3)): + with self.subTest(args=args), self.assertRaises(container_runtime.AnsibleFilterError): + runtime(*args) + + def test_the_filter_is_registered(self) -> None: + self.assertIs(container_runtime.FilterModule().filters()["cluster_container_runtime"], runtime) + + +if __name__ == "__main__": + unittest.main() From efe23ef9c5abd0d24ca44371637d1055fc937b78 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 20 Oct 2026 11:14:44 +0100 Subject: [PATCH 2/3] feat(cluster): start Colima at login on macOS hosts and check the runtime A reboot leaves a Mac's container runtime stopped, so the node's k3s container never comes back and a cluster that loses two such hosts loses quorum. On a macOS host whose Docker context is Colima, the role now starts the profile if it is stopped, installs a per-user launchd agent that starts it at every login with a PATH naming both Homebrew prefixes, and loads the agent into the connecting user's login session. Docker Desktop cannot be started by the role, so there the start-at-login setting is documented. The runtime tasks run before anything reads the daemon, and end checks after the Compose project is started assert that the runtime answers and the node's k3s container is running, so a converge of a rebooted host repairs it or fails with a message that says what to do. Linux hosts are not touched, and --check and --diff show what would change without starting anything. github_runner_cluster_runtime_autostart turns the whole thing off. The node's container name is now derived once as github_runner_cluster_node_container and shared with the running-version check. --- README.md | 2 +- roles/github_runner_cluster/README.md | 27 ++ roles/github_runner_cluster/defaults/main.yml | 15 + .../tasks/end_checks.yml | 34 ++ roles/github_runner_cluster/tasks/main.yml | 13 + .../tasks/running_version.yml | 5 +- roles/github_runner_cluster/tasks/runtime.yml | 120 +++++ .../tasks/runtime_probe.yml | 15 + .../templates/colima-launchd.plist.j2 | 33 ++ tests/runtime_autostart/render_plist.yml | 20 + tests/runtime_autostart/run.yml | 20 + tests/unit/test_runtime_autostart.py | 437 ++++++++++++++++++ 12 files changed, 736 insertions(+), 5 deletions(-) create mode 100644 roles/github_runner_cluster/tasks/end_checks.yml create mode 100644 roles/github_runner_cluster/tasks/runtime.yml create mode 100644 roles/github_runner_cluster/tasks/runtime_probe.yml create mode 100644 roles/github_runner_cluster/templates/colima-launchd.plist.j2 create mode 100644 tests/runtime_autostart/render_plist.yml create mode 100644 tests/runtime_autostart/run.yml create mode 100644 tests/unit/test_runtime_autostart.py diff --git a/README.md b/README.md index 9937b52..ad6c1f3 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ There are two ways to deploy. A fleet's hosts are managed from a control machine with Ansible, from the fleet's own inventory (see [Ansible](#ansible) below). `bootstrap.sh`, described here, brings up a single machine from a checkout of this repository without a control node or inventory. It runs the same Ansible role against the machine itself, so both paths deploy identically. -Prerequisites on the host: Docker (or Colima) with the Compose plugin, any release but 2.37.1 to 2.38.x (the cluster role refuses those; see Docker Compose versions in `roles/github_runner_cluster/README.md`), `python3`, `jq`, `openssl`, and `gh`. On macOS, the playbook installs `helm`, `kubectl` and Docker through Homebrew itself. Elsewhere, install `helm` and `kubectl` from the host's package manager first. +Prerequisites on the host: Docker (or Colima) with the Compose plugin, any release but 2.37.1 to 2.38.x (the cluster role refuses those; see Docker Compose versions in `roles/github_runner_cluster/README.md`), `python3`, `jq`, `openssl`, and `gh`. On macOS, the playbook installs `helm`, `kubectl` and Docker through Homebrew itself, and the cluster role starts a Colima host's profile at every login and fails unless the runtime is up (Docker Desktop needs its own start-at-login setting; see Container runtime at login in `roles/github_runner_cluster/README.md`). Elsewhere, install `helm` and `kubectl` from the host's package manager first. 1. **Create a GitHub App** on the organisation with **Self-hosted runners: read & write** permission, and install it on the organisation. `playbooks/github_app_setup.yml` does this through GitHub's manifest flow (see `roles/github_runner_arc/README.md`). 2. **Configure the environment.** Copy `.env.bootstrap.example` to `.env.bootstrap` and fill in the following (the file's own comments cover the rest, such as joining an existing cluster or running as an agent): diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index f5ac7f4..8f2fe37 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -56,6 +56,30 @@ By default the role drives whichever Docker daemon the host's current Docker con Set `github_runner_cluster_docker_context` in the host's variables to pin the context by name (`colima` in that example) on any host with more than one engine. Every `community.docker` module call in the role then passes it as `cli_context`, and every `docker` command the role runs (Compose, `exec`, `inspect`, `volume`, image pulls, and the commands in `playbooks/recover_in_cluster_mesh.yml` and `playbooks/promote_headscale_standby.yml`) runs with `DOCKER_CONTEXT` set to it, so the modules and the CLI always reach the same daemon. The role checks the pin before it touches the host and fails if the context does not exist there, or if `DOCKER_HOST` is also set, since the two would name the daemon in conflicting ways. It never creates a context and never changes the host's current context; `docker context show` on the host reads the same before and after a run. The Headscale tasks that act on another host (the `headscale_hosted` server and standby, and the promotion playbook) use that host's own value. Run `docker context show` on each host to see what it currently points at before choosing a value. +## Container runtime at login (macOS) + +On a Mac the container runtime is a Linux VM (Colima, or Docker Desktop) that stays stopped after a reboot until something starts it. The node's k3s container has `restart: unless-stopped`, so it comes back by itself once the runtime does, but a host whose runtime never starts never rejoins the cluster. With three servers, two such hosts leave the cluster without quorum and no runner pods anywhere. Only macOS hosts are affected by any of this: the role does nothing about the runtime on a Linux host, where Docker is a system service. + +Which runtime a Mac runs is taken from the Docker context the role drives (see [Docker contexts](#docker-contexts)): `colima`, or `colima-` for another profile, is Colima, and `desktop-linux` is Docker Desktop. `github_runner_cluster_runtime` says so explicitly for a host whose context does not tell, for example one that uses `DOCKER_HOST`, and `github_runner_cluster_colima_profile` names the profile to start when the context does not. + +**Colima.** With `github_runner_cluster_runtime_autostart` on (the default, which acts on Colima hosts only), the role: + +- starts the profile if it is stopped, with `colima start --profile --activate=false`, so the profile does not make its context the host's current one; +- installs a launchd agent, `com.exadev.github-runner.colima.`, as `~/Library/LaunchAgents/