diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c9e835..0277d24 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,35 +65,42 @@ jobs: run: python3 scripts/check-dashboard.py compose: - name: compose files parse + name: compose and scripts runs-on: ubuntu-latest - # The overlays only patch services, so they are not valid on their own - - # they are checked the way they are actually used. GITHUB_TOKEN is a - # required variable; any value satisfies interpolation. + # GITHUB_TOKEN is a required variable in the compose file; any value + # satisfies interpolation. env: GITHUB_TOKEN: dummy-value-for-interpolation-only steps: - uses: actions/checkout@v4 + - name: Compose files parse + run: | + docker compose -f docker-compose.yml config --quiet + docker compose -f docker-compose.yml -f docker-compose.admin.yml config --quiet + + # The collector runs on the host now, so these are the entry points a + # user actually touches. A syntax error in one of them used to be caught + # by nothing at all. + - name: Shell scripts parse + run: for f in scripts/*.sh; do bash -n "$f"; done + # repos.yml is gitignored, so CI builds one the way a new user would and - # proves gen-repos.py emits an override that actually merges. Nothing - # else in CI reads it, so a malformed one would otherwise only surface - # on somebody's laptop. - - name: Generate the fleet override + # proves gen-repos.py emits the two lines the collector reads - including + # for a checkout that does NOT sit at //, which is the + # case the paths line exists for. Nothing else in CI reads repos.yml, so + # a malformed one would otherwise only surface on somebody's laptop. + - name: The fleet resolves to an environment run: | pip install --quiet pyyaml printf 'repos:\n' > repos.yml - for full in Jebel-Quant/rhiza cvxgrp/cvxsimulator; do - git init -q -b main "ci-checkouts/$full" - git -C "ci-checkouts/$full" remote add origin "https://github.com/$full.git" - printf ' - path: ci-checkouts/%s\n' "$full" >> repos.yml - done - # A repo with no checkout: GitHub panels only, no bind mount. + git init -q -b main nested/somewhere/rhiza + git -C nested/somewhere/rhiza remote add origin https://github.com/Jebel-Quant/rhiza.git + printf ' - path: nested/somewhere/rhiza\n' >> repos.yml + # A repo with no checkout: GitHub panels only, no path. printf ' - repo: Jebel-Quant/actions\n' >> repos.yml - python3 scripts/gen-repos.py - cat docker-compose.repos.yml - - name: Base + fleet override - run: docker compose -f docker-compose.yml -f docker-compose.repos.yml config --quiet - - name: Base + admin overlay - run: docker compose -f docker-compose.yml -f docker-compose.repos.yml -f docker-compose.admin.yml config --quiet + python3 scripts/gen-repos.py | tee env.out + grep -qx 'JQ_REPOS=Jebel-Quant/rhiza,Jebel-Quant/actions' env.out + grep -q 'JQ_REPO_PATHS=Jebel-Quant/rhiza=.*/nested/somewhere/rhiza$' env.out + test "$(grep -c 'Jebel-Quant/actions=' env.out)" = 0 diff --git a/.gitignore b/.gitignore index d6ea74d..eba6d5f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,14 @@ .env -# Your fleet and the mounts generated from it: both describe the folder layout -# of one machine, which has no business in a public repo. Start from -# repos.example.yml. +# Your fleet: it describes the folder layout of one machine, which has no +# business in a public repo. Start from repos.example.yml. Nothing is generated +# from it any more - scripts/collector.sh reads it at every launch. repos.yml -docker-compose.repos.yml -# Left behind by running the collector natively instead of in its container. +# Where the launchd agent's output goes. +.collector-logs/ + +# Left behind by running the collector, which is how it runs now. .venv/ .ruff_cache/ .pytest_cache/ diff --git a/README.md b/README.md index ee468df..ff3373e 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,10 @@ the board because you listed it in `repos.yml`, and for no other reason. ## What you need -Docker, and the [`gh` CLI](https://cli.github.com) signed in (`up.sh` mints the -token from it — otherwise put a `GITHUB_TOKEN` in `.env` yourself). +Docker (for Prometheus and Grafana), [`uv`](https://docs.astral.sh/uv/) (for the +collector, which runs on your machine rather than in a container), and the +[`gh` CLI](https://cli.github.com) signed in — `up.sh` mints the token from it, +otherwise put a `GITHUB_TOKEN` in `.env` yourself. ## Recipe @@ -34,7 +36,9 @@ repos: - repo: Jebel-Quant/actions # monitored, but not cloned here ``` -`owner/name` comes from each checkout's `origin`, so the path is all you write. +`owner/name` comes from each checkout's `origin`, so the path is all you write, +and the path is used as written — a checkout does not have to live at +`//`. Then: ```bash @@ -43,8 +47,9 @@ open http://localhost:3000/d/jq-fleet ``` The board fills in within a minute — the local panels first, the GitHub panels -after the first API refresh. Both `repos.yml` and the generated -`docker-compose.repos.yml` are gitignored: they describe one machine's folders. +after the first API refresh. `repos.yml` is gitignored: it describes one machine's folders. Nothing is +generated from it — the collector reads it at every launch, so there is no +second file to fall out of step. ## Then what @@ -53,6 +58,7 @@ after the first API refresh. Both `repos.yml` and the generated | Add or drop a repo | edit `repos.yml`, `./scripts/up.sh` again — [details](docs/configuration.md) | | Erase a dropped repo's history | [`./scripts/purge-repo.sh owner/name`](docs/configuration.md#dropping-a-repo) (irreversible) | | Stop | `./scripts/down.sh` (add `--volumes` to discard the history too) | +| See the collector's log | `tail -f .collector-logs/collector.log` — it runs on your machine, not in Docker ([why](docs/operations.md#the-collector-runs-on-your-machine)) | | Edit the board | change `grafana/dashboards/fleet.json`; it reloads in 30s — [read the traps first](docs/dashboard.md#traps-worth-not-re-introducing) | | Get notified | add a contact point under *Alerting → Contact points* — [why it is not provisioned](docs/dashboard.md#alerting) | diff --git a/collector/Dockerfile b/collector/Dockerfile deleted file mode 100644 index 2cebba0..0000000 --- a/collector/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM python:3.12-slim - -# git is the collector's other data source, alongside the GitHub API. -RUN apt-get update \ - && apt-get install -y --no-install-recommends git ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -# The clones are bind-mounted from the host and are owned by a uid this -# container does not have, which git refuses to touch by default. The mount is -# read-only, so widening the trust here cannot lead to a write. -RUN git config --system --add safe.directory '*' - -WORKDIR /app -COPY pyproject.toml README.md ./ -COPY jq_collector ./jq_collector -RUN pip install --no-cache-dir . - -EXPOSE 9109 -CMD ["python", "-m", "jq_collector"] diff --git a/collector/jq_collector/config.py b/collector/jq_collector/config.py index 30a1c32..ed74d62 100644 --- a/collector/jq_collector/config.py +++ b/collector/jq_collector/config.py @@ -20,6 +20,25 @@ def _csv(name: str, default: tuple[str, ...] = ()) -> tuple[str, ...]: return tuple(part.strip() for part in raw.split(",") if part.strip()) +def _pairs(name: str) -> dict[str, str]: + """``owner/name=path`` pairs, comma separated, into a mapping. + + Raises rather than skipping a malformed entry. A dropped pair would take + one repo's working-copy panels off the board and say nothing about why, + which is the failure mode this whole module is shaped to avoid; refusing to + start is louder and cheaper to diagnose. A path containing a comma cannot + be expressed here - ``scripts/gen-repos.py`` refuses to emit one. + """ + mapping: dict[str, str] = {} + for item in _csv(name): + key, separator, value = item.partition("=") + key, value = key.strip(), value.strip() + if not separator or not key or not value: + raise ValueError(f"{name}: expected owner/name=path, got {item!r}") + mapping[key] = os.path.expanduser(value) + return mapping + + @dataclass(frozen=True) class Config: """Everything the collector needs to know about its environment. @@ -43,12 +62,21 @@ class Config: token: str = os.environ.get("GITHUB_TOKEN", "") api: str = os.environ.get("GITHUB_API", "https://api.github.com") - # Where the checkouts are mounted (read-only) inside the container, one per - # repo at //. Set empty to skip local scanning - # entirely, which is the right setting anywhere there are no working copies - # to report on - the local panels then simply have nothing to say. + # Where the checkouts are, for repos JQ_REPO_PATHS does not name: one per + # repo at //. That is exactly where the generated + # compose override bind-mounts them, so in the container this is all that + # is needed. Set empty to skip local scanning entirely, which is the right + # setting anywhere there are no working copies to report on - the local + # panels then simply have nothing to say. repo_root: str = os.environ.get("JQ_REPO_ROOT", "/repos") + # Explicit path per repo, as owner/name=path. Needed whenever a checkout + # does not sit at // - which is normal outside the + # container, where paths are whatever they are on disk rather than whatever + # a bind mount normalised them to. `scripts/gen-repos.py --env` emits this + # from repos.yml, so the layout is stated once and in one place. + repo_paths: dict[str, str] = field(default_factory=lambda: _pairs("JQ_REPO_PATHS")) + # owner/name of the repo whose releases define "up to date". template_repo: str = os.environ.get("JQ_TEMPLATE_REPO", "Jebel-Quant/rhiza") template_pointer: str = os.environ.get("JQ_TEMPLATE_POINTER", ".rhiza/template.yml") diff --git a/collector/jq_collector/localgit.py b/collector/jq_collector/localgit.py index 606e036..43cd99d 100644 --- a/collector/jq_collector/localgit.py +++ b/collector/jq_collector/localgit.py @@ -17,10 +17,14 @@ drift is the one thing that changes while the clone stands still, because it is the upstream that moved. -Nothing here searches for checkouts either. Each monitored repo is expected at -``//``, which is exactly where the generated compose -override mounts it. The fleet is decided by ``repos.yml``, not by whatever -happened to be lying around under a scanned directory. +Nothing here searches for checkouts either. A repo is read at the path +``JQ_REPO_PATHS`` gives for it, or failing that at ``//`` +- which is exactly where the generated compose override mounts it, so inside the +container the default is always right. Outside it, paths are whatever they are on +disk: ``repos.yml`` may well say ``~/repos/tschm/rhiza_projects/cs`` for +``tschm/cs``, and no amount of joining owner to name will produce that. Either +way the fleet is decided by ``repos.yml``, not by whatever happened to be lying +around under a scanned directory. """ from __future__ import annotations @@ -354,20 +358,29 @@ def scan( """ prior = previous or {} found: dict[str, LocalRepo] = {} - if not cfg.repo_root: - # Deliberate: with no repo root there are no working copies to report - # on, and an empty result is the honest answer rather than an error - # every minute. - return found - if not os.path.isdir(cfg.repo_root): - log.error("repo root %s is not a directory", cfg.repo_root) + + # A configured root that does not exist is worth one error, not one per + # repo per minute. Explicit paths are unaffected by it, so this only + # withdraws the fallback rather than abandoning the whole scan. + root = cfg.repo_root + if root and not os.path.isdir(root): + log.error("repo root %s is not a directory", root) + root = "" + if not root and not cfg.repo_paths: + # Deliberate: nothing points at a working copy, so there are none to + # report on, and an empty result is the honest answer rather than an + # error every minute. return found for key in cfg.repos: if "/" not in key or key in skip: continue owner, repo_name = key.split("/", 1) - path = os.path.join(cfg.repo_root, owner, repo_name) + path = cfg.repo_paths.get(key) + if path is None: + if not root: + continue + path = os.path.join(root, owner, repo_name) # `.git` is a directory in a plain checkout and a file in a worktree. if not os.path.exists(os.path.join(path, ".git")): # Not mounted, or mounted somewhere else. Normal for a repo you diff --git a/collector/tests/test_fleet.py b/collector/tests/test_fleet.py index 32cbde6..0bb64fc 100644 --- a/collector/tests/test_fleet.py +++ b/collector/tests/test_fleet.py @@ -14,6 +14,7 @@ import dataclasses import importlib.util +import os import pathlib import subprocess import sys @@ -46,10 +47,15 @@ def make_checkout( return path -def config(repos: tuple[str, ...], repo_root: pathlib.Path) -> Config: +def config( + repos: tuple[str, ...], + repo_root: pathlib.Path | str, + repo_paths: dict[str, str] | None = None, +) -> Config: cfg = Config() object.__setattr__(cfg, "repos", repos) object.__setattr__(cfg, "repo_root", str(repo_root)) + object.__setattr__(cfg, "repo_paths", repo_paths or {}) return cfg @@ -105,6 +111,96 @@ def test_the_checkout_is_read_at_the_canonical_path(tmp_path): assert found["cvxgrp/cvxsimulator"].branch == "main" +# -- where a checkout actually is -------------------------------------------- + + +def test_a_checkout_outside_the_canonical_layout_is_read(tmp_path): + """// is a default, not a rule. + + The bind mounts normalise every checkout onto that shape inside the + container, so the default is always right there. On the host it is not: + ~/repos/tschm/rhiza_projects/cs is a checkout of tschm/cs, and no amount of + joining owner to name will produce that path. Four repos in the real fleet + are laid out this way, and they silently vanished from the working-copy + panels while staying on the GitHub ones. + """ + path = make_checkout(tmp_path / "nested" / "elsewhere", "tschm", "cs") + cfg = config(("tschm/cs",), tmp_path, {"tschm/cs": str(path)}) + + found = localgit.scan(cfg, {}) + + assert set(found) == {"tschm/cs"} + assert found["tschm/cs"].path == str(path) + + +def test_an_explicit_path_wins_over_the_canonical_one(tmp_path): + canonical = make_checkout(tmp_path, "tschm", "cs") + elsewhere = make_checkout(tmp_path / "other", "tschm", "cs") + cfg = config(("tschm/cs",), tmp_path, {"tschm/cs": str(elsewhere)}) + + found = localgit.scan(cfg, {}) + + assert found["tschm/cs"].path == str(elsewhere) + assert found["tschm/cs"].path != str(canonical) + + +def test_repos_without_an_explicit_path_still_fall_back_to_the_root(tmp_path): + """The container relies on this: mounts, no JQ_REPO_PATHS at all.""" + make_checkout(tmp_path, "Jebel-Quant", "rhiza") + nested = make_checkout(tmp_path / "nested", "tschm", "cs") + cfg = config(("Jebel-Quant/rhiza", "tschm/cs"), tmp_path, {"tschm/cs": str(nested)}) + + found = localgit.scan(cfg, {}) + + assert set(found) == {"Jebel-Quant/rhiza", "tschm/cs"} + + +def test_an_explicit_path_to_the_wrong_repo_is_still_refused(tmp_path, caplog): + """The origin check matters more now, not less - the path is arbitrary.""" + path = make_checkout( + tmp_path / "nested", "tschm", "cs", origin="git@github.com:cvxgrp/cvxpy.git" + ) + cfg = config(("tschm/cs",), tmp_path, {"tschm/cs": str(path)}) + + assert localgit.scan(cfg, {}) == {} + assert "check repos.yml" in caplog.text + + +def test_explicit_paths_work_with_no_repo_root_at_all(tmp_path): + """Running outside the container there is no /repos to fall back to.""" + path = make_checkout(tmp_path / "nested", "tschm", "cs") + cfg = config(("tschm/cs",), "", {"tschm/cs": str(path)}) + + assert set(localgit.scan(cfg, {})) == {"tschm/cs"} + + +def test_nothing_local_configured_is_a_quiet_no_op(tmp_path): + assert localgit.scan(config(("tschm/cs",), "", {}), {}) == {} + + +def test_a_missing_root_does_not_abandon_the_explicit_paths(tmp_path, caplog): + """One error for the bad root, and the repos that do not need it still work.""" + path = make_checkout(tmp_path / "nested", "tschm", "cs") + cfg = config(("tschm/cs",), tmp_path / "does-not-exist", {"tschm/cs": str(path)}) + + assert set(localgit.scan(cfg, {})) == {"tschm/cs"} + assert "is not a directory" in caplog.text + + +def test_malformed_repo_paths_refuse_to_start(monkeypatch): + """A dropped pair would take one repo off the board and say nothing.""" + monkeypatch.setenv("JQ_REPO_PATHS", "tschm/cs") + with pytest.raises(ValueError, match="owner/name=path"): + Config() + + +def test_repo_paths_are_parsed_and_expanded(monkeypatch): + monkeypatch.setenv("JQ_REPO_PATHS", "tschm/cs=~/repos/tschm/rhiza_projects/cs, a/b=/tmp/b") + paths = Config().repo_paths + assert paths["a/b"] == "/tmp/b" + assert paths["tschm/cs"] == os.path.expanduser("~/repos/tschm/rhiza_projects/cs") + + # -- measurements and the activity cache ------------------------------------- @@ -340,25 +436,44 @@ def test_a_directory_that_is_not_a_checkout_fails(gen_repos, tmp_path): gen_repos.resolve({"path": str(tmp_path / "empty")}, 1) -def test_env_mode_prints_the_list_and_writes_nothing(gen_repos, tmp_path, monkeypatch, capsys): - """A server names its fleet in .env, and that line must come from repos.yml. +def test_it_prints_the_fleet_and_the_paths_and_writes_nothing( + gen_repos, tmp_path, monkeypatch, capsys +): + """Both lines must come from repos.yml, not from anyone's memory. - Retyping it is how the two drift: a repo added here never reaches the board - and nothing reports the difference. --env must also leave the compose - override alone, so it is safe to run on a machine that has no stack. + Retyping either is how they drift: a repo added here never reaches the + board and nothing reports the difference. The paths line is what lets the + collector find a checkout that does not sit at //, and a + repo with no checkout must not appear in it at all. Nothing is written to + disk - scripts/collector.sh runs this at every launch and exports the + result, so there is no generated file in between to go stale. """ path = make_checkout(tmp_path, "Jebel-Quant", "rhiza") source = tmp_path / "repos.yml" source.write_text(f"repos:\n - path: {path}\n - repo: cvxgrp/cvxsimulator\n") - target = tmp_path / "docker-compose.repos.yml" monkeypatch.setattr(gen_repos, "SOURCE", source) - monkeypatch.setattr(gen_repos, "TARGET", target) - monkeypatch.setattr(sys, "argv", ["gen-repos.py", "--env"]) + monkeypatch.setattr(sys, "argv", ["gen-repos.py"]) gen_repos.main() - assert capsys.readouterr().out.strip() == "JQ_REPOS=Jebel-Quant/rhiza,cvxgrp/cvxsimulator" - assert not target.exists() + assert capsys.readouterr().out.strip().splitlines() == [ + "JQ_REPOS=Jebel-Quant/rhiza,cvxgrp/cvxsimulator", + f"JQ_REPO_PATHS=Jebel-Quant/rhiza={path}", + ] + assert list(tmp_path.glob("docker-compose*")) == [] + + +def test_it_refuses_a_path_it_cannot_express(gen_repos, tmp_path, monkeypatch, capsys): + """A comma is the separator, so a path holding one would read as two repos.""" + path = make_checkout(tmp_path, "Jebel-Quant", "rhi,za") + source = tmp_path / "repos.yml" + source.write_text(f"repos:\n - path: {path}\n") + monkeypatch.setattr(gen_repos, "SOURCE", source) + monkeypatch.setattr(sys, "argv", ["gen-repos.py"]) + + with pytest.raises(SystemExit): + gen_repos.main() + assert "contains a comma" in capsys.readouterr().err def test_an_unknown_flag_is_refused(gen_repos, monkeypatch): diff --git a/docker-compose.yml b/docker-compose.yml index 71b232f..ae1e8e1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,37 +1,15 @@ -# The laptop stack. Not usable on its own - the fleet lives in repos.yml, -# so bring it up with scripts/up.sh, or with: +# Prometheus and Grafana. The collector is NOT here: it runs on the host, via +# scripts/collector.sh, because it reads your working copies and a container can +# only see them through bind mounts - which cannot name a checkout that sits +# somewhere other than //, and which are slow enough on +# macOS to have needed a caching layer to work around. # -# python3 scripts/gen-repos.py -# docker compose -f docker-compose.yml -f docker-compose.repos.yml up -d +# ./scripts/up.sh starts both halves +# docker compose up -d starts only this half name: jq-monitoring services: - collector: - build: ./collector - container_name: jq-collector - restart: unless-stopped - environment: - GITHUB_TOKEN: ${GITHUB_TOKEN:?set GITHUB_TOKEN in monitoring/.env - scripts/up.sh does it for you} - # JQ_REPOS and the per-repo bind mounts both come from - # docker-compose.repos.yml, generated from repos.yml. This file has no - # opinion about which repos are yours. - JQ_REPO_ROOT: /repos - JQ_TEMPLATE_REPO: ${JQ_TEMPLATE_REPO:-Jebel-Quant/rhiza} - JQ_GITHUB_INTERVAL: ${JQ_GITHUB_INTERVAL:-600} - JQ_LOCAL_INTERVAL: ${JQ_LOCAL_INTERVAL:-60} - JQ_MEASURE_MAX_AGE: ${JQ_MEASURE_MAX_AGE:-86400} - JQ_IGNORE: ${JQ_IGNORE:-} - JQ_PUBLIC_ONLY: ${JQ_PUBLIC_ONLY:-false} - JQ_LOG_LEVEL: ${JQ_LOG_LEVEL:-INFO} - # No volumes here on purpose. Every checkout is mounted individually by - # docker-compose.repos.yml, read-only: the collector reports on your working - # copies and must never be able to touch them, and mounting only what is - # listed means an unlisted repo is not merely filtered out - it is not even - # visible to the container. Every git call is --no-optional-locks too. - ports: - - "127.0.0.1:9109:9109" - prometheus: image: prom/prometheus:v2.55.1 container_name: jq-prometheus @@ -46,8 +24,10 @@ services: - prometheus-data:/prometheus ports: - "127.0.0.1:9090:9090" - depends_on: - - collector + # Docker Desktop resolves host.docker.internal on its own; plain Docker on + # Linux does not, and without this the collector target is simply down. + extra_hosts: + - "host.docker.internal:host-gateway" grafana: image: grafana/grafana:11.3.1 diff --git a/docs/configuration.md b/docs/configuration.md index 7526b54..d0e1ef4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -87,14 +87,46 @@ better a refusal than a board that is quietly one repo short. | `JQ_IGNORE` | — | Repos to drop without editing `repos.yml`, as bare names or `owner/name`. Applies to both halves. | | `JQ_INCLUDE_ARCHIVED` | `false` | Archived repos are dropped from both halves. | | `JQ_PUBLIC_ONLY` | `false` | Drop private repos entirely — not just their details, their existence. | +| `JQ_REPO_PATHS` | — | Where each checkout really is, as `owner/name=path` pairs. `scripts/collector.sh` sets it from `repos.yml` — see [Checkout paths](#checkout-paths). | | `JQ_GITHUB_INTERVAL` | `300` | Seconds between GitHub refreshes. | | `JQ_MEASURE_MAX_AGE` | `86400` | Seconds an unchanged line/commit count may stand before it is retaken. See [Size and cadence](dashboard.md#size-and-cadence). | | `PROM_RETENTION` | `180d` | How much history to keep. | -`scripts/gen-repos.py` turns `repos.yml` into the `JQ_REPOS` list the -collector actually reads, so both halves see the same fleet. Setting `JQ_REPOS` -by hand works too, and skips `repos.yml` entirely — but then nothing mounts the -checkouts, so only the GitHub panels have anything to say. +`scripts/gen-repos.py` turns `repos.yml` into the `JQ_REPOS` list the collector +actually reads, so both halves see the same fleet. Setting `JQ_REPOS` by hand +works too and skips `repos.yml` entirely — but then no checkout paths are set +either, so only the GitHub panels have anything to say. + +## Checkout paths + +The collector reads a repo at the path `JQ_REPO_PATHS` gives for it, and failing +that at `//`. + +The collector runs on your machine, so paths are whatever they are on disk and +the fallback is often wrong. An entry like + +```yaml + - path: ~/repos/tschm/rhiza_projects/cs # this is tschm/cs +``` + +cannot be recovered by joining owner to name, so that repo drops off the +working-copy panels while staying on the GitHub ones — present on the board, and +quietly missing half its columns. Four repos in the fleet this was built against +are laid out that way. + +You do not normally set this yourself: `scripts/collector.sh` runs +`scripts/gen-repos.py` at every launch and exports both lines, so `repos.yml` +stays the only place the fleet and the layout are written down. To see what it +resolves to: + +```bash +uv run --with pyyaml python scripts/gen-repos.py +``` + +A path containing a comma cannot be expressed — comma is the separator — and +`gen-repos.py` refuses to emit one rather than produce a line that would be +misread as two repos. A malformed pair stops the collector at startup instead of +silently shrinking the board. ## API budget diff --git a/docs/dashboard.md b/docs/dashboard.md index 68df28c..2b190a7 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -107,6 +107,14 @@ name to `owner/name` when the cvxgrp repos were added, so history from before that point lives under the old names. Windows spanning the change show both generations; they age out. This is inherent to Prometheus, not a misconfiguration. +**The `instance` label is pinned on purpose.** There is one collector, so +`instance` says only where the process happened to be running — and left alone +it is the scrape address. When the collector moved out of its container the +address changed, every series forked in two, and for one stale window each repo +appeared on the board twice. `prometheus.yml` relabels it to a constant so that +cannot recur; series from before the move keep the old value and age out with +retention. + **Never `group_left` onto an `_info` metric without a `topk` guard.** The `_info` metrics carry a label that changes value — `ref`, `branch`, `conclusion`, `checks`. When one changes, Prometheus returns the old *and* the diff --git a/docs/operations.md b/docs/operations.md index d033001..02e4334 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -6,6 +6,42 @@ keywords: grafana login, no data, prometheus staleness, macos sleep, caffeinate # Running it day to day +## The collector runs on your machine + +Two halves, not three containers. Prometheus and Grafana are in Docker; +**the collector is an ordinary process on your Mac**, started by `up.sh` as a +launchd agent and stopped by `down.sh`. + +It is not containerised because it reads your working copies, and a container +can only reach those through bind mounts. Those mounts have to place every +checkout at `//`, which silently loses any repo that lives +somewhere else, and reading thousands of small files back through them on macOS +is slow enough that the line counts needed a cache to stay affordable. On the +host both problems disappear. + +| | | +|---|---| +| Its log | `.collector-logs/collector.log` | +| Restart it | `launchctl kickstart -k gui/$UID/com.jebel-quant.jq-collector` | +| Run it in the foreground instead | `./scripts/collector.sh` (Ctrl-C stops it) | +| Is Prometheus reaching it? | — the `jq-collector` job | + +Prometheus scrapes it at `host.docker.internal:9109`, which is how a container +reaches the machine it runs on. + +**The trade.** Inside the container the collector could only see the checkouts +mounted into it, so an unlisted repo was not merely filtered out — it was +invisible. It now runs as you and could read anything you can. It still never +writes: every git call is read-only and passes `--no-optional-locks`. But that +is now a property of the code rather than something the sandbox enforces. + +**If it will not start**, the usual cause is `PATH`. A launchd agent inherits +`/usr/bin:/bin:/usr/sbin:/sbin` and nothing else, so a `uv` under +`/opt/homebrew` or `~/.local` is invisible to it. `up.sh` pins `uv`'s directory +into the plist when it installs the agent, so re-running `./scripts/up.sh` +after moving or reinstalling `uv` is the fix. + + ## The "Sign in" button Grafana's own local login, against a SQLite file in the `grafana-data` volume on diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index 24ac892..9e47616 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -9,8 +9,22 @@ global: scrape_configs: - job_name: jq-collector + # The collector runs on the host, not in this stack - it reads your working + # copies at their real paths, which a bind mount cannot express and which + # is far faster than one anyway. host.docker.internal is how a container + # reaches the machine it runs on; docker-compose.yml maps it on Linux, + # where it is not built in. static_configs: - - targets: ["collector:9109"] + - targets: ["host.docker.internal:9109"] + relabel_configs: + # There is exactly one collector, so `instance` carries no information - + # only where the process happened to be running. Left alone it is the + # scrape address, and moving the collector (container -> host, or back) + # silently forks every series in two: the board then joins a repo against + # itself and shows each one twice until the old series go stale. Pin it, + # and the history stays continuous across the move. + - target_label: instance + replacement: jq-collector - job_name: prometheus static_configs: diff --git a/scripts/collector.sh b/scripts/collector.sh new file mode 100755 index 0000000..cd3b91a --- /dev/null +++ b/scripts/collector.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Run the collector on this machine, in the foreground. +# +# Not in a container, on purpose. It reads your working copies, and a container +# can only reach those through bind mounts - which cannot name a checkout that +# sits somewhere other than //, and which are slow enough on +# macOS that the line counts needed a caching layer to stay affordable. +# +# Prometheus, which is still in Docker, scrapes this at host.docker.internal:9109. +# +# Run it under launchd (./scripts/up.sh sets that up on macOS), in a terminal, +# or under tmux - it is an ordinary foreground process either way, and Ctrl-C +# stops it. +# +# The trade this makes: inside the container the collector could only see the +# checkouts that were mounted into it, so an unlisted repo was not merely +# filtered out, it was invisible. Here it runs as you and could read anything +# you can. It still never writes - every git call is read-only and passes +# --no-optional-locks - but that is now a property of the code rather than +# something the sandbox enforces. +set -euo pipefail +cd "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +die() { printf '\033[31merror\033[0m %s\n' "$1" >&2; exit 1; } + +[[ -f .env ]] || die ".env not found - run ./scripts/up.sh first" +[[ -f repos.yml ]] || die "repos.yml not found - run ./scripts/up.sh first" +# Says "on PATH", not "installed", because under launchd it is almost always +# the former: an agent inherits /usr/bin:/bin:/usr/sbin:/sbin and nothing else, +# so a perfectly good uv under /opt/homebrew or ~/.local is invisible. up.sh +# pins uv's directory into the plist for exactly this reason. +command -v uv >/dev/null 2>&1 || die "uv is not on PATH ($PATH) - see https://docs.astral.sh/uv/" + +set -a; . ./.env; set +a + +# repos.yml is the fleet and the layout, and it stays the only place either is +# written down: both lines are computed at every launch rather than generated +# into a file that can fall out of step with it. +# +# `export "$line"` and not `export $line` - a path may contain spaces, and +# unquoted word splitting would turn one repo into two broken ones. +while IFS= read -r line; do + [[ -n "$line" ]] && export "$line" +done < <(uv run --quiet --with pyyaml python scripts/gen-repos.py) + +# No container means no bind mounts and no /repos to fall back on. Every path +# is real and absolute, and JQ_REPO_PATHS carries all of them. +export JQ_REPO_ROOT="" + +cd collector +exec uv run --quiet python -m jq_collector diff --git a/scripts/down.sh b/scripts/down.sh index 7042271..cba596f 100755 --- a/scripts/down.sh +++ b/scripts/down.sh @@ -1,9 +1,15 @@ #!/usr/bin/env bash -# Stop the stack. Add --volumes to also throw away the Prometheus history. +# Stop both halves. Add --volumes to also throw away the Prometheus history. set -euo pipefail cd "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -# The generated override may be missing on a fresh clone; the project name in -# docker-compose.yml is enough to find the running containers either way. -files=(-f docker-compose.yml) -[[ -f docker-compose.repos.yml ]] && files+=(-f docker-compose.repos.yml) -docker compose "${files[@]}" down "$@" + +label="com.jebel-quant.jq-collector" +if [[ "$(uname -s)" == "Darwin" ]]; then + # KeepAlive means the agent restarts itself if merely killed, so unload it. + # Fails when it was never loaded, which is fine. + launchctl bootout "gui/$UID/$label" 2>/dev/null && echo "collector: launchd agent unloaded" || true +else + pkill -f "python -m jq_collector" 2>/dev/null && echo "collector: stopped" || true +fi + +docker compose down "$@" diff --git a/scripts/gen-repos.py b/scripts/gen-repos.py index 1928e56..0b1965e 100755 --- a/scripts/gen-repos.py +++ b/scripts/gen-repos.py @@ -1,20 +1,25 @@ #!/usr/bin/env python3 -"""Turn repos.yml into the compose override that mounts the fleet. +"""Turn repos.yml into the two environment lines the collector reads. -The collector never searches for checkouts. It expects each repo at -/repos//, so this script resolves every listed path, asks its -origin remote who it is, and writes one read-only bind mount per repo plus the -matching JQ_REPOS list. Both halves of the collector then read the same list, -and the board's contents are decided by repos.yml alone. +The collector never searches for checkouts. This script resolves every listed +path, asks its origin remote who it is, and prints: -Run it after editing repos.yml; scripts/up.sh does that for you. + JQ_REPOS the fleet, as owner/name - read by both halves + JQ_REPO_PATHS where each checkout actually is - read by the local half -With ``--env`` it writes nothing and prints the ``JQ_REPOS=`` line instead. A -server has no checkouts, so it names its fleet in .env by hand - and a list -retyped by hand is a list that drifts from the one here. Generate it on a -machine that has the checkouts and paste the one line over: + python3 scripts/gen-repos.py - python3 scripts/gen-repos.py --env +Nothing is written. scripts/collector.sh runs this at every launch and exports +the result, so repos.yml stays the only place the fleet and the layout are +written down - there is no generated file in between to go stale. + +The paths line exists because a repos.yml entry like + + - path: ~/repos/tschm/rhiza_projects/cs # a checkout of tschm/cs + +cannot be recovered by joining owner to name. This used to be papered over by +bind-mounting every checkout onto /repos//, back when the collector +ran in a container; it does not, so the real path has to be carried. """ from __future__ import annotations @@ -26,7 +31,6 @@ ROOT = Path(__file__).resolve().parent.parent SOURCE = ROOT / "repos.yml" -TARGET = ROOT / "docker-compose.repos.yml" def fail(message: str) -> None: @@ -112,10 +116,9 @@ def resolve(entry: object, index: int) -> tuple[str, Path | None]: def main() -> None: - env_only = "--env" in sys.argv[1:] - unknown = [a for a in sys.argv[1:] if a != "--env"] + unknown = sys.argv[1:] if unknown: - fail(f"unknown argument {unknown[0]} - the only flag is --env") + fail(f"unknown argument {unknown[0]} - this script takes none") resolved: dict[str, Path | None] = {} for index, entry in enumerate(load_entries(), start=1): @@ -124,32 +127,20 @@ def main() -> None: fail(f"{full_name} is listed twice") resolved[full_name] = path - # Nothing but the line itself on stdout, so it can be piped or appended. - if env_only: - print(f'JQ_REPOS={",".join(resolved)}') - return - - lines = [ - "# Generated by scripts/gen-repos.py from repos.yml - do not edit.", - "#", - "# One read-only bind mount per monitored checkout, at the canonical", - "# /repos// the collector looks for, plus the matching", - "# JQ_REPOS list so the GitHub half sees exactly the same fleet.", - "", - "services:", - " collector:", - " environment:", - f' JQ_REPOS: "{",".join(resolved)}"', - ] - mounts = {name: path for name, path in resolved.items() if path is not None} - if mounts: - lines.append(" volumes:") - lines += [f' - "{path}:/repos/{name}:ro"' for name, path in mounts.items()] - TARGET.write_text("\n".join(lines) + "\n", encoding="utf-8") - - without = len(resolved) - len(mounts) - tail = f", {without} without a local checkout" if without else "" - print(f"gen-repos: wrote {TARGET.name} - {len(resolved)} repos{tail}") + # Nothing but the lines themselves on stdout, so they can be exported, + # piped, or appended to a .env. + print(f'JQ_REPOS={",".join(resolved)}') + + paths = {name: path for name, path in resolved.items() if path is not None} + # A comma is the separator, so a path containing one cannot be expressed. + # Refuse rather than print a line the collector would reject or, worse, + # silently misread as two repos. + for name, path in paths.items(): + if "," in str(path): + fail(f"{name}: path contains a comma, which JQ_REPO_PATHS cannot express: {path}") + if paths: + joined = ",".join(f"{name}={path}" for name, path in paths.items()) + print(f"JQ_REPO_PATHS={joined}") if __name__ == "__main__": diff --git a/scripts/up.sh b/scripts/up.sh index ac0579f..e16e40f 100755 --- a/scripts/up.sh +++ b/scripts/up.sh @@ -1,5 +1,10 @@ #!/usr/bin/env bash -# Bring the stack up, minting a GitHub token from the gh CLI if there isn't one. +# Bring both halves up: Prometheus and Grafana in Docker, the collector on this +# machine. Mints a GitHub token from the gh CLI if there isn't one. +# +# The collector is not containerised - see scripts/collector.sh for why. On +# macOS it is installed as a launchd agent so it comes back after a reboot; +# elsewhere this script says how to run it and leaves supervision to you. set -euo pipefail here="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" @@ -31,16 +36,66 @@ if ! grep -qE '^GITHUB_TOKEN=.+' .env; then echo "wrote a token from 'gh auth token' into monitoring/.env" fi -# repos.yml is the fleet. Regenerate the mounts every time, so editing the list -# and running ./scripts/up.sh is the whole workflow. -if command -v uv >/dev/null 2>&1; then - uv run --quiet --with pyyaml scripts/gen-repos.py +command -v uv >/dev/null 2>&1 || { + echo "uv is not installed - the collector needs it; see https://docs.astral.sh/uv/" >&2 + exit 1 +} + +# Fail on a broken repos.yml here, while there is still a human watching, rather +# than inside a launchd agent whose output nobody is reading. The collector +# recomputes this at every launch; this run is only a check. +uv run --quiet --with pyyaml python scripts/gen-repos.py >/dev/null + +docker compose up -d + +# -- the collector ----------------------------------------------------------- +label="com.jebel-quant.jq-collector" +if [[ "$(uname -s)" == "Darwin" ]]; then + plist="$HOME/Library/LaunchAgents/$label.plist" + # A launchd agent inherits almost no PATH - /usr/bin:/bin:/usr/sbin:/sbin and + # nothing else - so uv, which lives under /opt/homebrew or ~/.local, is simply + # not found. Pin the directory it is actually in rather than guessing at + # either location. + uv_dir="$(dirname "$(command -v uv)")" + logs="$here/.collector-logs" + mkdir -p "$HOME/Library/LaunchAgents" "$logs" + # Regenerated every time: the paths are absolute, so a moved checkout of this + # repo would otherwise leave a plist pointing at where it used to be. + cat > "$plist" < + + + + Label$label + ProgramArguments + $here/scripts/collector.sh + WorkingDirectory$here + EnvironmentVariables + + PATH$uv_dir:/usr/bin:/bin:/usr/sbin:/sbin + + RunAtLoad + KeepAlive + + ThrottleInterval10 + StandardOutPath$logs/collector.log + StandardErrorPath$logs/collector.log + + +PLIST + # bootout first so an edited plist is actually re-read; it fails when nothing + # is loaded, which is the normal first run. + launchctl bootout "gui/$UID/$label" 2>/dev/null || true + launchctl bootstrap "gui/$UID" "$plist" + echo "collector: loaded as a launchd agent - logs in .collector-logs/collector.log" else - python3 scripts/gen-repos.py + echo "collector: not started. Run it yourself, e.g." + echo " ./scripts/collector.sh # foreground" + echo " or install it as a systemd --user service running that script." fi -docker compose -f docker-compose.yml -f docker-compose.repos.yml up -d --build - echo echo "Grafana http://localhost:3000/d/jq-fleet (anonymous read-only; admin/admin to edit)" echo "Prometheus http://localhost:9090"