feat(cluster): start the container runtime at login on macOS hosts

README.md

@@ -6,7 +6,7 @@
 
 There are two ways to deploy. A fleet's hosts are managed from a control machine with Ansible, from the fleet's own inventory (see [Ansible](#ansible) below). `bootstrap.sh`, described here, brings up a single machine from a checkout of this repository without a control node or inventory. It runs the same Ansible role against the machine itself, so both paths deploy identically.
 
-Prerequisites on the host: Docker (or Colima) with the Compose plugin, any release but 2.37.1 to 2.38.x (the cluster role refuses those; see Docker Compose versions in `roles/github_runner_cluster/README.md`), `python3`, `jq`, `openssl`, and `gh`. On macOS, the playbook installs `helm`, `kubectl` and Docker through Homebrew itself. Elsewhere, install `helm` and `kubectl` from the host's package manager first.
+Prerequisites on the host: Docker (or Colima) with the Compose plugin, any release but 2.37.1 to 2.38.x (the cluster role refuses those; see Docker Compose versions in `roles/github_runner_cluster/README.md`), `python3`, `jq`, `openssl`, and `gh`. On macOS, the playbook installs `helm`, `kubectl` and Docker through Homebrew itself, and the cluster role starts a Colima host's profile at every login and fails unless the runtime is up (Docker Desktop needs its own start-at-login setting; see Container runtime at login in `roles/github_runner_cluster/README.md`). Elsewhere, install `helm` and `kubectl` from the host's package manager first.
 
 1. **Create a GitHub App** on the organisation with **Self-hosted runners: read & write** permission, and install it on the organisation. `playbooks/github_app_setup.yml` does this through GitHub's manifest flow (see `roles/github_runner_arc/README.md`).
 2. **Configure the environment.** Copy `.env.bootstrap.example` to `.env.bootstrap` and fill in the following (the file's own comments cover the rest, such as joining an existing cluster or running as an agent):

plugins/filter/container_runtime.py

@@ -0,0 +1,74 @@
+"""Filter the github_runner_cluster role uses to tell which container runtime a macOS host's Docker context belongs to, and so whether the role can start that runtime itself."""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from ansible.errors import AnsibleFilterError
+
+RUNTIME_CHOICES = ("auto", "colima", "docker_desktop", "other")
+# Colima names the Docker context of its default profile "colima" and any other profile's "colima-<profile>".
+COLIMA_CONTEXT = "colima"
+COLIMA_DEFAULT_PROFILE = "default"
+# Docker Desktop's own context for its Linux engine.
+DOCKER_DESKTOP_CONTEXTS = ("desktop-linux",)
+# Lima's rule for an instance name, which is what a Colima profile becomes. It also keeps the value free of every character that would need escaping in a launchd property list or on a command line.
+PROFILE_PATTERN = re.compile(r"^[A-Za-z0-9]+([._-][A-Za-z0-9]+)*$")
+
+
+def _text(name: str, value: Any) -> str:
+    if value is None:
+        return ""
+    if not isinstance(value, str):
+        raise AnsibleFilterError(f"{name} must be a string, not {type(value).__name__}: {value!r}.")
+    return value.strip()
+
+
+def _colima_profile_of(context: str) -> str | None:
+    """Return the Colima profile a Docker context belongs to, or None when it is not a Colima context."""
+    if context == COLIMA_CONTEXT:
+        return COLIMA_DEFAULT_PROFILE
+    prefix = COLIMA_CONTEXT + "-"
+    if context.startswith(prefix) and len(context) > len(prefix):
+        return context[len(prefix) :]
+    return None
+
+
+def cluster_container_runtime(context: Any, requested: Any = "auto", colima_profile: Any = "") -> dict[str, str]:
+    """Work out which container runtime a macOS host's Docker daemon belongs to.
+
+    The role calls it only for a macOS host, the only kind it starts a runtime on. ``auto`` takes the runtime from the Docker context the role drives: a Colima context (``colima`` or ``colima-<profile>``) is Colima and ``desktop-linux`` is Docker Desktop, and anything else (including no context because ``DOCKER_HOST`` is set) is ``other``.
+
+    :param context: the Docker context the role drives, or an empty value for none. :param requested: ``auto``, ``colima``, ``docker_desktop`` or ``other``. :param colima_profile: the Colima profile to start, or an empty value to take it from the context (``default`` when the context names none).
+    :returns: ``{"kind": "colima" | "docker_desktop" | "other", "colima_profile": <profile, empty unless kind is colima>}``.
+    :raises AnsibleFilterError: if a value is not a string, ``requested`` is not one of the choices, the profile is not a valid Colima profile name, or it names another profile than the Colima context the role drives.
+    """
+    docker_context = _text("The Docker context", context)
+    choice = _text("The container runtime", requested) or "auto"
+    profile = _text("The Colima profile", colima_profile)
+    if choice not in RUNTIME_CHOICES:
+        raise AnsibleFilterError(f"The container runtime must be one of {', '.join(RUNTIME_CHOICES)}, not {choice!r}.")
+    if profile and not PROFILE_PATTERN.match(profile):
+        raise AnsibleFilterError(f"The Colima profile {profile!r} is not a valid profile name: use letters and digits, separated by single dots, hyphens or underscores.")
+    context_profile = _colima_profile_of(docker_context)
+    if choice == "auto":
+        if context_profile is not None:
+            choice = "colima"
+        elif docker_context in DOCKER_DESKTOP_CONTEXTS:
+            choice = "docker_desktop"
+        else:
+            choice = "other"
+    if choice != "colima":
+        return {"kind": choice, "colima_profile": ""}
+    if profile and context_profile is not None and profile != context_profile:
+        raise AnsibleFilterError(f"The Colima profile {profile!r} is not the one the Docker context {docker_context!r} belongs to ({context_profile!r}); the role would start one VM and drive another.")
+    return {"kind": "colima", "colima_profile": profile or context_profile or COLIMA_DEFAULT_PROFILE}
+
+
+class FilterModule:
+    """Registers the container runtime filter with Ansible."""
+
+    def filters(self) -> dict[str, Any]:
+        """Return the filters this plugin provides."""
+        return {"cluster_container_runtime": cluster_container_runtime}

roles/github_runner_cluster/README.md

@@ -56,6 +56,30 @@ By default the role drives whichever Docker daemon the host's current Docker con
 
 Set `github_runner_cluster_docker_context` in the host's variables to pin the context by name (`colima` in that example) on any host with more than one engine. Every `community.docker` module call in the role then passes it as `cli_context`, and every `docker` command the role runs (Compose, `exec`, `inspect`, `volume`, image pulls, and the commands in `playbooks/recover_in_cluster_mesh.yml` and `playbooks/promote_headscale_standby.yml`) runs with `DOCKER_CONTEXT` set to it, so the modules and the CLI always reach the same daemon. The role checks the pin before it touches the host and fails if the context does not exist there, or if `DOCKER_HOST` is also set, since the two would name the daemon in conflicting ways. It never creates a context and never changes the host's current context; `docker context show` on the host reads the same before and after a run. The Headscale tasks that act on another host (the `headscale_hosted` server and standby, and the promotion playbook) use that host's own value. Run `docker context show` on each host to see what it currently points at before choosing a value.
 
+## Container runtime at login (macOS)
+
+On a Mac the container runtime is a Linux VM (Colima, or Docker Desktop) that stays stopped after a reboot until something starts it. The node's k3s container has `restart: unless-stopped`, so it comes back by itself once the runtime does, but a host whose runtime never starts never rejoins the cluster. With three servers, two such hosts leave the cluster without quorum and no runner pods anywhere. Only macOS hosts are affected by any of this: the role does nothing about the runtime on a Linux host, where Docker is a system service.
+
+Which runtime a Mac runs is taken from the Docker context the role drives (see [Docker contexts](#docker-contexts)): `colima`, or `colima-<profile>` for another profile, is Colima, and `desktop-linux` is Docker Desktop. `github_runner_cluster_runtime` says so explicitly for a host whose context does not tell, for example one that uses `DOCKER_HOST`, and `github_runner_cluster_colima_profile` names the profile to start when the context does not.
+
+**Colima.** With `github_runner_cluster_runtime_autostart` on (the default, which acts on Colima hosts only), the role:
+
+- starts the profile if it is stopped, with `colima start --profile <profile> --activate=false`, so the profile does not make its context the host's current one;
+- installs a launchd agent, `com.exadev.github-runner.colima.<profile>`, as `~/Library/LaunchAgents/<label>.plist` of the user Ansible connects to, which runs the same command at every login with a `PATH` that names both Homebrew prefixes (`github_runner_cluster_runtime_path`), since launchd gives an agent only the system `PATH` and Colima starts `limactl` and `docker` from the one it has; and
+- loads the agent into that user's login session. A user with no login session on the host (nobody has logged in at the console since the last boot) cannot have an agent loaded, so the role installs the file, says so, and leaves launchd to load it at the next login.
+
+The role reads what it needs to tell a Mac from another host (the kernel name) and to name the agent (the connecting user's id and home directory) with minimal `setup` subsets of its own when the caller has not gathered them, so a play with `gather_facts: false` works. The role never uses `become` for this: a LaunchAgent belongs to the user whose login session loads it, and Colima's VM runs as that user. Changing a definition unloads the agent before loading the new one; a rerun with nothing changed touches neither the agent nor the VM, and `--check --diff` shows the property list that would be written. With the variable off, the role installs nothing and leaves an agent from an earlier run as it is.
+
+A Mac still has to log in for the agent to run: a launch agent starts with the user's login session, not with the machine. A host that should recover from a reboot unattended needs automatic login for that user (System Settings > Users & Groups).
+
+**Docker Desktop.** The role cannot start Docker Desktop. Turn on Settings > General > Start Docker Desktop when you sign in to your computer on each such host (https://docs.docker.com/desktop/settings-and-maintenance/settings/), and give it the same automatic login. If it is not running when the role runs, the run fails with that message.
+
+**End checks.** On every macOS host the role checks, before anything reads the daemon and again after it has started the node's Compose project, that the runtime answers (`docker info` through the host's Docker context), and after starting the project that the node's k3s container (`github-runner-k3s`, or `github-runner-k3s-agent` on an agent) is running. A converge of a host that rebooted therefore repairs a stopped Colima or fails with a message that says what to do, and never reports success over a node that is not up. A `--check` run does not fail on a stopped runtime or container it has not started.
+
+A Colima VM that starts after a reboot is a restarted VM, so the `br_netfilter` note under Non-obvious behaviour in the repository README applies to it: pods can lose cluster service IPs until that module is loaded in the VM.
+
+This is the role's whole scope: it never moves a host between Colima and Docker Desktop.
+
 ## Mesh providers
 
 `github_runner_cluster_mesh` picks the provider. Each one lives in `tasks/mesh/<provider>.yml` and publishes the same facts, so nothing else in the role depends on which is in use:
@@ -138,6 +162,7 @@ Each site is its own cluster: its own inventory group, bootstrap server and mesh
 - `github_runner_cluster_node_taints`: Kubernetes taints (`key=value:Effect`, no spaces) the host's node registers with, empty by default. `NoSchedule` keeps pods without a matching toleration off it and `NoExecute` also evicts the ones already there. k3s applies them on every registration, so they come back if the node object is deleted, which a taint set with `kubectl` does not.
 - Per-host overrides: `github_runner_cluster_node_role` (`server` or `agent`), `github_runner_cluster_bootstrap` (true picks the bootstrap host, false excludes a host), `github_runner_cluster_server_url` (join this server instead, for a cluster whose servers are outside the inventory), `github_runner_cluster_tls_sans`, `github_runner_cluster_node_address`. The run fails if the result has an even number of servers, no servers, or more than one bootstrap host.
 - `github_runner_cluster_docker_context` (default empty): pins the Docker context this host's node runs in (see [Docker contexts](#docker-contexts)).
+- `github_runner_cluster_runtime_autostart` (default true), `github_runner_cluster_runtime` (default `auto`; `colima`, `docker_desktop` or `other`), `github_runner_cluster_colima_profile` (default empty, taken from the Docker context), `github_runner_cluster_runtime_path` (default both Homebrew prefixes and the system directories): on a macOS host, start Colima at login through a launchd agent and check the runtime (see [Container runtime at login](#container-runtime-at-login-macos)).
 - `github_runner_cluster_compose_files`: extra Compose files for the host's k3s project, merged after `docker-compose.yml`, as paths relative to `github_runner_cluster_dir` or absolute. Several nodes on one Docker host each need their own `github_runner_cluster_dir` and `github_runner_cluster_compose_project`.
 - `github_runner_cluster_wait_for_nodes`: wait for every node to be Ready afterwards, clearing stale node-password registrations if a node fails to rejoin (installs the kubernetes Python client through `github_runner_k8s_client`).
 
@@ -180,3 +205,5 @@ or read from 1Password by adding `exadev.github_runner.github_runner_secrets_one
 `tests/litestream/run.sh` runs the `headscale_hosted` provider with Litestream and a warm standby against an S3-compatible store on this machine's Docker, registers a real Tailscale client, promotes the standby and checks that users, nodes and pre-auth keys survive, that the client reconnects and that the promoted server keeps replicating. It also checks that `headscale_in_cluster` refuses to run without Litestream. `.github/workflows/litestream.yml` runs it on pull requests that touch the Headscale providers.
 
 `tests/mesh/run.sh` brings up three k3s nodes in Docker on one machine and joins them through Headscale with this role, for the `headscale_hosted` and `headscale_in_cluster` providers, and checks the `headscale_existing` policy check against a real server. Each cluster scenario runs the role a second time and fails if any node container restarted. The `compose_faulty` scenario checks the role refuses a Compose release that has that fault. With `GRTEST_PIN_CONTEXT=1` the cluster scenarios pin `github_runner_cluster_docker_context` while the current Docker context the playbooks see points at a socket that does not exist, so any docker call that ignored the pin fails, and they check the current context is unchanged afterwards; `tests/litestream/run.sh` takes the same setting for the Headscale hosts it delegates to and the promotion playbook. The `context_checks` scenario checks the role refuses a pinned context that does not exist, and one set alongside `DOCKER_HOST`, before writing anything. `.github/workflows/mesh-integration.yml` runs them on pull requests that touch the role, the hosted scenario on 2.39.0, the first fixed release, as well as on a current one.
+
+`tests/unit/test_runtime_autostart.py` covers the [container runtime tasks](#container-runtime-at-login-macos) without a Mac: it renders the launchd agent through `tests/runtime_autostart/render_plist.yml`, and runs `tasks/runtime.yml` and `tasks/end_checks.yml` through the role against stand-ins for `docker`, `colima`, `launchctl` and `plutil` on `PATH`. It checks the variable defaults, that Linux hosts are left alone, that a second run changes nothing, that `--check` writes and starts nothing, and that each way the runtime or the node's container can be down fails with its message. `tests/unit/test_container_runtime.py` covers the filter that works out the runtime from the Docker context. Nothing there touches a real Colima, launchd or Docker Desktop, so loading the agent into a real login session at a real reboot is the part only a run on a Mac can show.

roles/github_runner_cluster/defaults/main.yml

@@ -124,6 +124,21 @@ github_runner_cluster_compose_file_list: >-
 github_runner_cluster_docker_context: ""
 # The node's k3s container, as docker-compose.yml names it. The headscale_in_cluster provider runs commands inside it; override it only alongside an extra Compose file that renames the container.
 github_runner_cluster_container_name: github-runner-k3s
+# The node's k3s container as this host runs it: the server's, or the agent's, which docker-compose.yml names differently. Derived, never set directly.
+github_runner_cluster_node_container: "{{ github_runner_cluster_container_name if github_runner_cluster_effective_node_role == 'server' else 'github-runner-k3s-agent' }}"
+
+# macOS hosts only (Linux hosts are never touched): whether the role makes the container runtime come back after a reboot. On a Colima host it installs a per-user launchd agent that starts the Colima profile at login, loads it, and starts the profile at once if it is stopped. true (the default) does that for Colima hosts and nothing for the others: the role cannot start Docker Desktop, whose start-at-login setting is documented in the README. false installs nothing and leaves an agent from an earlier run as it is. Either way the end checks fail unless the runtime is up and the node's container is running.
+github_runner_cluster_runtime_autostart: true
+# Which container runtime a macOS host runs: auto takes it from the Docker context the role drives (colima or colima-<profile> is Colima, desktop-linux is Docker Desktop, anything else is other), colima, docker_desktop and other say so for a host whose context does not tell (DOCKER_HOST set, a context with another name).
+github_runner_cluster_runtime: auto
+# The Colima profile to start. Empty takes it from the Docker context (colima is the default profile, colima-<profile> another) and falls back to the default profile when the context names none.
+github_runner_cluster_colima_profile: ""
+# The PATH the agent runs colima with, and the role runs colima and docker with on a macOS host: both Homebrew prefixes (Apple Silicon and Intel), since launchd gives an agent only the system PATH and colima starts limactl and docker from the one it is given.
+github_runner_cluster_runtime_path: "/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin"
+# Derived, never set directly: the agent's launchd label, one per Colima profile.
+github_runner_cluster_runtime_launchd_label: "com.exadev.github-runner.colima.{{ github_runner_cluster_effective_runtime.colima_profile }}"
+# Derived, never set directly: where the agent's property list lives, in the connecting user's own LaunchAgents directory.
+github_runner_cluster_runtime_agent_plist: "{{ ansible_facts['env']['HOME'] }}/Library/LaunchAgents/{{ github_runner_cluster_runtime_launchd_label }}.plist"
 
 # The directory on the target host the role manages as this host's Compose project: it copies the collection's docker-compose.yml and k3s/ build context there, templates .env there, and k3s writes the kubeconfig to its kubeconfig/ subdirectory. Needs no checkout of anything on the host. The default reads HOME from the target's gathered facts, so it is the connecting user's home on that host, not the control node's.
 github_runner_cluster_dir: "{{ ansible_facts['env']['HOME'] }}/{{ github_runner_cluster_dir_name }}"