From 3b791dd22ffbf859812ead679a2d8c415c425cd1 Mon Sep 17 00:00:00 2001 From: Thomas Schmelzer Date: Sun, 30 Aug 2026 18:12:23 +0400 Subject: [PATCH 1/2] fix(collector): read a checkout where it actually is localgit.scan derived every path as //. Inside the container that is always right, because the generated bind mounts put each checkout at exactly that path - repos.yml can say anything and the mount normalises it. Outside the container it is often wrong, and silently so. Four repos in the fleet this was built against are laid out like this: - path: ~/repos/tschm/rhiza_projects/cs # a checkout of tschm/cs No amount of joining owner to name produces that path, so those repos dropped off the working-copy panels while staying on the GitHub ones: present on the board, quietly missing half their columns, and nothing said why. Running the collector on the host reported 20 of 24 checkouts. JQ_REPO_PATHS now carries the real path per repo, as owner/name=path pairs, with // kept as the fallback. `gen-repos.py --env` emits it from repos.yml alongside JQ_REPOS, so the layout is stated once, in the file that already owns it, rather than retyped. * The container is untouched: it sets no JQ_REPO_PATHS and uses the fallback exactly as before. Verified - still 24 of 24, 1743 metric lines. * On the host, 24 of 24 now, with JQ_REPO_ROOT empty and every path coming from JQ_REPO_PATHS alone. The origin check matters more now rather than less, since the path is arbitrary - a path pointing at the wrong repo is still refused rather than filed under the listed repo's name, and there is a test for it. Two ways this could have gone quiet, both closed: * A malformed pair raises at startup instead of being skipped. Dropping one would take a repo's local panels off the board and say nothing - the exact failure this module is shaped to avoid. * A path containing a comma cannot be expressed, comma being the separator. gen-repos.py refuses to emit one rather than print a line that would be misread as two repos. A bad JQ_REPO_ROOT no longer abandons the whole scan either: it logs once and withdraws the fallback, leaving repos with explicit paths working. --- collector/jq_collector/config.py | 36 +++++++- collector/jq_collector/localgit.py | 37 +++++--- collector/tests/test_fleet.py | 133 +++++++++++++++++++++++++++-- docs/configuration.md | 34 ++++++++ scripts/gen-repos.py | 36 +++++--- 5 files changed, 243 insertions(+), 33 deletions(-) diff --git a/collector/jq_collector/config.py b/collector/jq_collector/config.py index 30a1c32..ed74d62 100644 --- a/collector/jq_collector/config.py +++ b/collector/jq_collector/config.py @@ -20,6 +20,25 @@ def _csv(name: str, default: tuple[str, ...] = ()) -> tuple[str, ...]: return tuple(part.strip() for part in raw.split(",") if part.strip()) +def _pairs(name: str) -> dict[str, str]: + """``owner/name=path`` pairs, comma separated, into a mapping. + + Raises rather than skipping a malformed entry. A dropped pair would take + one repo's working-copy panels off the board and say nothing about why, + which is the failure mode this whole module is shaped to avoid; refusing to + start is louder and cheaper to diagnose. A path containing a comma cannot + be expressed here - ``scripts/gen-repos.py`` refuses to emit one. + """ + mapping: dict[str, str] = {} + for item in _csv(name): + key, separator, value = item.partition("=") + key, value = key.strip(), value.strip() + if not separator or not key or not value: + raise ValueError(f"{name}: expected owner/name=path, got {item!r}") + mapping[key] = os.path.expanduser(value) + return mapping + + @dataclass(frozen=True) class Config: """Everything the collector needs to know about its environment. @@ -43,12 +62,21 @@ class Config: token: str = os.environ.get("GITHUB_TOKEN", "") api: str = os.environ.get("GITHUB_API", "https://api.github.com") - # Where the checkouts are mounted (read-only) inside the container, one per - # repo at //. Set empty to skip local scanning - # entirely, which is the right setting anywhere there are no working copies - # to report on - the local panels then simply have nothing to say. + # Where the checkouts are, for repos JQ_REPO_PATHS does not name: one per + # repo at //. That is exactly where the generated + # compose override bind-mounts them, so in the container this is all that + # is needed. Set empty to skip local scanning entirely, which is the right + # setting anywhere there are no working copies to report on - the local + # panels then simply have nothing to say. repo_root: str = os.environ.get("JQ_REPO_ROOT", "/repos") + # Explicit path per repo, as owner/name=path. Needed whenever a checkout + # does not sit at // - which is normal outside the + # container, where paths are whatever they are on disk rather than whatever + # a bind mount normalised them to. `scripts/gen-repos.py --env` emits this + # from repos.yml, so the layout is stated once and in one place. + repo_paths: dict[str, str] = field(default_factory=lambda: _pairs("JQ_REPO_PATHS")) + # owner/name of the repo whose releases define "up to date". template_repo: str = os.environ.get("JQ_TEMPLATE_REPO", "Jebel-Quant/rhiza") template_pointer: str = os.environ.get("JQ_TEMPLATE_POINTER", ".rhiza/template.yml") diff --git a/collector/jq_collector/localgit.py b/collector/jq_collector/localgit.py index 606e036..43cd99d 100644 --- a/collector/jq_collector/localgit.py +++ b/collector/jq_collector/localgit.py @@ -17,10 +17,14 @@ drift is the one thing that changes while the clone stands still, because it is the upstream that moved. -Nothing here searches for checkouts either. Each monitored repo is expected at -``//``, which is exactly where the generated compose -override mounts it. The fleet is decided by ``repos.yml``, not by whatever -happened to be lying around under a scanned directory. +Nothing here searches for checkouts either. A repo is read at the path +``JQ_REPO_PATHS`` gives for it, or failing that at ``//`` +- which is exactly where the generated compose override mounts it, so inside the +container the default is always right. Outside it, paths are whatever they are on +disk: ``repos.yml`` may well say ``~/repos/tschm/rhiza_projects/cs`` for +``tschm/cs``, and no amount of joining owner to name will produce that. Either +way the fleet is decided by ``repos.yml``, not by whatever happened to be lying +around under a scanned directory. """ from __future__ import annotations @@ -354,20 +358,29 @@ def scan( """ prior = previous or {} found: dict[str, LocalRepo] = {} - if not cfg.repo_root: - # Deliberate: with no repo root there are no working copies to report - # on, and an empty result is the honest answer rather than an error - # every minute. - return found - if not os.path.isdir(cfg.repo_root): - log.error("repo root %s is not a directory", cfg.repo_root) + + # A configured root that does not exist is worth one error, not one per + # repo per minute. Explicit paths are unaffected by it, so this only + # withdraws the fallback rather than abandoning the whole scan. + root = cfg.repo_root + if root and not os.path.isdir(root): + log.error("repo root %s is not a directory", root) + root = "" + if not root and not cfg.repo_paths: + # Deliberate: nothing points at a working copy, so there are none to + # report on, and an empty result is the honest answer rather than an + # error every minute. return found for key in cfg.repos: if "/" not in key or key in skip: continue owner, repo_name = key.split("/", 1) - path = os.path.join(cfg.repo_root, owner, repo_name) + path = cfg.repo_paths.get(key) + if path is None: + if not root: + continue + path = os.path.join(root, owner, repo_name) # `.git` is a directory in a plain checkout and a file in a worktree. if not os.path.exists(os.path.join(path, ".git")): # Not mounted, or mounted somewhere else. Normal for a repo you diff --git a/collector/tests/test_fleet.py b/collector/tests/test_fleet.py index 32cbde6..1475123 100644 --- a/collector/tests/test_fleet.py +++ b/collector/tests/test_fleet.py @@ -14,6 +14,7 @@ import dataclasses import importlib.util +import os import pathlib import subprocess import sys @@ -46,10 +47,15 @@ def make_checkout( return path -def config(repos: tuple[str, ...], repo_root: pathlib.Path) -> Config: +def config( + repos: tuple[str, ...], + repo_root: pathlib.Path | str, + repo_paths: dict[str, str] | None = None, +) -> Config: cfg = Config() object.__setattr__(cfg, "repos", repos) object.__setattr__(cfg, "repo_root", str(repo_root)) + object.__setattr__(cfg, "repo_paths", repo_paths or {}) return cfg @@ -105,6 +111,96 @@ def test_the_checkout_is_read_at_the_canonical_path(tmp_path): assert found["cvxgrp/cvxsimulator"].branch == "main" +# -- where a checkout actually is -------------------------------------------- + + +def test_a_checkout_outside_the_canonical_layout_is_read(tmp_path): + """// is a default, not a rule. + + The bind mounts normalise every checkout onto that shape inside the + container, so the default is always right there. On the host it is not: + ~/repos/tschm/rhiza_projects/cs is a checkout of tschm/cs, and no amount of + joining owner to name will produce that path. Four repos in the real fleet + are laid out this way, and they silently vanished from the working-copy + panels while staying on the GitHub ones. + """ + path = make_checkout(tmp_path / "nested" / "elsewhere", "tschm", "cs") + cfg = config(("tschm/cs",), tmp_path, {"tschm/cs": str(path)}) + + found = localgit.scan(cfg, {}) + + assert set(found) == {"tschm/cs"} + assert found["tschm/cs"].path == str(path) + + +def test_an_explicit_path_wins_over_the_canonical_one(tmp_path): + canonical = make_checkout(tmp_path, "tschm", "cs") + elsewhere = make_checkout(tmp_path / "other", "tschm", "cs") + cfg = config(("tschm/cs",), tmp_path, {"tschm/cs": str(elsewhere)}) + + found = localgit.scan(cfg, {}) + + assert found["tschm/cs"].path == str(elsewhere) + assert found["tschm/cs"].path != str(canonical) + + +def test_repos_without_an_explicit_path_still_fall_back_to_the_root(tmp_path): + """The container relies on this: mounts, no JQ_REPO_PATHS at all.""" + make_checkout(tmp_path, "Jebel-Quant", "rhiza") + nested = make_checkout(tmp_path / "nested", "tschm", "cs") + cfg = config(("Jebel-Quant/rhiza", "tschm/cs"), tmp_path, {"tschm/cs": str(nested)}) + + found = localgit.scan(cfg, {}) + + assert set(found) == {"Jebel-Quant/rhiza", "tschm/cs"} + + +def test_an_explicit_path_to_the_wrong_repo_is_still_refused(tmp_path, caplog): + """The origin check matters more now, not less - the path is arbitrary.""" + path = make_checkout( + tmp_path / "nested", "tschm", "cs", origin="git@github.com:cvxgrp/cvxpy.git" + ) + cfg = config(("tschm/cs",), tmp_path, {"tschm/cs": str(path)}) + + assert localgit.scan(cfg, {}) == {} + assert "check repos.yml" in caplog.text + + +def test_explicit_paths_work_with_no_repo_root_at_all(tmp_path): + """Running outside the container there is no /repos to fall back to.""" + path = make_checkout(tmp_path / "nested", "tschm", "cs") + cfg = config(("tschm/cs",), "", {"tschm/cs": str(path)}) + + assert set(localgit.scan(cfg, {})) == {"tschm/cs"} + + +def test_nothing_local_configured_is_a_quiet_no_op(tmp_path): + assert localgit.scan(config(("tschm/cs",), "", {}), {}) == {} + + +def test_a_missing_root_does_not_abandon_the_explicit_paths(tmp_path, caplog): + """One error for the bad root, and the repos that do not need it still work.""" + path = make_checkout(tmp_path / "nested", "tschm", "cs") + cfg = config(("tschm/cs",), tmp_path / "does-not-exist", {"tschm/cs": str(path)}) + + assert set(localgit.scan(cfg, {})) == {"tschm/cs"} + assert "is not a directory" in caplog.text + + +def test_malformed_repo_paths_refuse_to_start(monkeypatch): + """A dropped pair would take one repo off the board and say nothing.""" + monkeypatch.setenv("JQ_REPO_PATHS", "tschm/cs") + with pytest.raises(ValueError, match="owner/name=path"): + Config() + + +def test_repo_paths_are_parsed_and_expanded(monkeypatch): + monkeypatch.setenv("JQ_REPO_PATHS", "tschm/cs=~/repos/tschm/rhiza_projects/cs, a/b=/tmp/b") + paths = Config().repo_paths + assert paths["a/b"] == "/tmp/b" + assert paths["tschm/cs"] == os.path.expanduser("~/repos/tschm/rhiza_projects/cs") + + # -- measurements and the activity cache ------------------------------------- @@ -340,12 +436,17 @@ def test_a_directory_that_is_not_a_checkout_fails(gen_repos, tmp_path): gen_repos.resolve({"path": str(tmp_path / "empty")}, 1) -def test_env_mode_prints_the_list_and_writes_nothing(gen_repos, tmp_path, monkeypatch, capsys): - """A server names its fleet in .env, and that line must come from repos.yml. +def test_env_mode_prints_the_fleet_and_the_paths_and_writes_nothing( + gen_repos, tmp_path, monkeypatch, capsys +): + """Both lines must come from repos.yml, not from anyone's memory. - Retyping it is how the two drift: a repo added here never reaches the board - and nothing reports the difference. --env must also leave the compose - override alone, so it is safe to run on a machine that has no stack. + Retyping either is how they drift: a repo added here never reaches the + board and nothing reports the difference. The paths line is what lets a + collector running outside the container find a checkout that does not sit + at // - and a repo with no checkout must not appear in + it at all. --env must also leave the compose override alone, so it is safe + to run on a machine that has no stack. """ path = make_checkout(tmp_path, "Jebel-Quant", "rhiza") source = tmp_path / "repos.yml" @@ -357,10 +458,28 @@ def test_env_mode_prints_the_list_and_writes_nothing(gen_repos, tmp_path, monkey gen_repos.main() - assert capsys.readouterr().out.strip() == "JQ_REPOS=Jebel-Quant/rhiza,cvxgrp/cvxsimulator" + lines = capsys.readouterr().out.strip().splitlines() + assert lines == [ + "JQ_REPOS=Jebel-Quant/rhiza,cvxgrp/cvxsimulator", + f"JQ_REPO_PATHS=Jebel-Quant/rhiza={path}", + ] assert not target.exists() +def test_env_mode_refuses_a_path_it_cannot_express(gen_repos, tmp_path, monkeypatch, capsys): + """A comma is the separator, so a path holding one would read as two repos.""" + path = make_checkout(tmp_path, "Jebel-Quant", "rhi,za") + source = tmp_path / "repos.yml" + source.write_text(f"repos:\n - path: {path}\n") + monkeypatch.setattr(gen_repos, "SOURCE", source) + monkeypatch.setattr(gen_repos, "TARGET", tmp_path / "out.yml") + monkeypatch.setattr(sys, "argv", ["gen-repos.py", "--env"]) + + with pytest.raises(SystemExit): + gen_repos.main() + assert "contains a comma" in capsys.readouterr().err + + def test_an_unknown_flag_is_refused(gen_repos, monkeypatch): monkeypatch.setattr(sys, "argv", ["gen-repos.py", "--all"]) with pytest.raises(SystemExit): diff --git a/docs/configuration.md b/docs/configuration.md index 7526b54..8526031 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -87,6 +87,7 @@ better a refusal than a board that is quietly one repo short. | `JQ_IGNORE` | — | Repos to drop without editing `repos.yml`, as bare names or `owner/name`. Applies to both halves. | | `JQ_INCLUDE_ARCHIVED` | `false` | Archived repos are dropped from both halves. | | `JQ_PUBLIC_ONLY` | `false` | Drop private repos entirely — not just their details, their existence. | +| `JQ_REPO_PATHS` | — | Where each checkout really is, as `owner/name=path` pairs. Only needed outside the container — see [Checkout paths](#checkout-paths). | | `JQ_GITHUB_INTERVAL` | `300` | Seconds between GitHub refreshes. | | `JQ_MEASURE_MAX_AGE` | `86400` | Seconds an unchanged line/commit count may stand before it is retaken. See [Size and cadence](dashboard.md#size-and-cadence). | | `PROM_RETENTION` | `180d` | How much history to keep. | @@ -96,6 +97,39 @@ collector actually reads, so both halves see the same fleet. Setting `JQ_REPOS` by hand works too, and skips `repos.yml` entirely — but then nothing mounts the checkouts, so only the GitHub panels have anything to say. +## Checkout paths + +The collector reads a repo at the path `JQ_REPO_PATHS` gives for it, and failing +that at `//`. + +**In the container only the fallback is used, and it is always right**, because +the generated bind mounts put every checkout at exactly that path. `repos.yml` +can say anything and the mount normalises it. + +**Outside the container it is often wrong**, because paths are whatever they are +on disk. An entry like + +```yaml + - path: ~/repos/tschm/rhiza_projects/cs # this is tschm/cs +``` + +cannot be recovered by joining owner to name, so that repo drops off the +working-copy panels while staying on the GitHub ones — present on the board, and +quietly missing half its columns. Four repos in the fleet this was built against +are laid out that way. + +So when you run the collector on the host, take both lines from `repos.yml` +rather than writing them out: + +```bash +uv run --with pyyaml python scripts/gen-repos.py --env >> .env +``` + +A path containing a comma cannot be expressed — comma is the separator — and +`gen-repos.py` refuses to emit one rather than produce a line that would be +misread as two repos. A malformed pair stops the collector at startup instead of +silently shrinking the board. + ## API budget A refresh costs roughly `3 × repos + open PRs` REST calls. Measured on this diff --git a/scripts/gen-repos.py b/scripts/gen-repos.py index 1928e56..dea4e0c 100755 --- a/scripts/gen-repos.py +++ b/scripts/gen-repos.py @@ -1,20 +1,25 @@ #!/usr/bin/env python3 """Turn repos.yml into the compose override that mounts the fleet. -The collector never searches for checkouts. It expects each repo at -/repos//, so this script resolves every listed path, asks its -origin remote who it is, and writes one read-only bind mount per repo plus the -matching JQ_REPOS list. Both halves of the collector then read the same list, -and the board's contents are decided by repos.yml alone. +The collector never searches for checkouts. This script resolves every listed +path, asks its origin remote who it is, and writes one read-only bind mount per +repo at /repos// plus the matching JQ_REPOS list. Both halves of +the collector then read the same list, and the board's contents are decided by +repos.yml alone. Run it after editing repos.yml; scripts/up.sh does that for you. -With ``--env`` it writes nothing and prints the ``JQ_REPOS=`` line instead. A -server has no checkouts, so it names its fleet in .env by hand - and a list -retyped by hand is a list that drifts from the one here. Generate it on a -machine that has the checkouts and paste the one line over: +With ``--env`` it writes nothing and prints the environment instead - JQ_REPOS, +and JQ_REPO_PATHS naming where each checkout really is: python3 scripts/gen-repos.py --env + +That second line is what makes a collector running outside the container agree +with repos.yml. In the container the bind mounts normalise every checkout onto +/repos//, so the path is implied; on the host it is not, because a +repos.yml entry like ~/repos/tschm/rhiza_projects/cs for tschm/cs cannot be +recovered by joining owner to name. Without the paths those repos silently +vanish from the working-copy panels while staying on the GitHub ones. """ from __future__ import annotations @@ -124,9 +129,20 @@ def main() -> None: fail(f"{full_name} is listed twice") resolved[full_name] = path - # Nothing but the line itself on stdout, so it can be piped or appended. + # Nothing but the lines themselves on stdout, so they can be piped or + # appended to a .env. if env_only: print(f'JQ_REPOS={",".join(resolved)}') + paths = {name: path for name, path in resolved.items() if path is not None} + # A comma is the separator, so a path containing one cannot be + # expressed. Refuse rather than emit a line the collector would reject + # or, worse, silently misread as two repos. + for name, path in paths.items(): + if "," in str(path): + fail(f"{name}: path contains a comma, which JQ_REPO_PATHS cannot express: {path}") + if paths: + joined = ",".join(f"{name}={path}" for name, path in paths.items()) + print(f"JQ_REPO_PATHS={joined}") return lines = [ From be3011de72a0a708f1d2c8a261fe4655f4d274c6 Mon Sep 17 00:00:00 2001 From: Thomas Schmelzer Date: Sun, 30 Aug 2026 18:32:58 +0400 Subject: [PATCH 2/2] feat: run the collector on the host, not in a container Prometheus and Grafana stay in Docker - they are pinned images with repo-tracked provisioning, which is what Docker is good at here. The collector is not; it is a Python package that reads your working copies, and the container was costing more than it gave: * bind mounts have to place every checkout at //, so a repo living anywhere else silently lost its local panels. Four in this fleet did. * reading thousands of small files back through a macOS bind mount is slow enough that the line counts needed a cache to stay affordable. * eight of the Dockerfile's nineteen lines existed only to undo problems the container created - installing git, and trusting a bind mount owned by a uid the container does not have. What changes * collector/Dockerfile is gone, and so is the collector service. * prometheus.yml scrapes host.docker.internal:9109. extra_hosts maps it on Linux, where it is not built in. * scripts/collector.sh runs it in the foreground - under launchd, in a terminal, or under tmux. up.sh installs and loads a launchd agent on macOS and says what to do elsewhere; down.sh unloads it. * gen-repos.py no longer writes a compose override, because there is no container to mount anything into. It prints JQ_REPOS and JQ_REPO_PATHS, and collector.sh runs it at every launch - so repos.yml is the only place the fleet and the layout are written down, with no generated file in between to go stale. Two things this turned up, both fixed here * A launchd agent inherits /usr/bin:/bin:/usr/sbin:/sbin and nothing else, so uv under /opt/homebrew was invisible and the agent failed on every launch. KeepAlive then respawned it in a tight loop - fourteen failures before it was noticed. up.sh now pins uv's directory into the plist, and ThrottleInterval turns any future misconfiguration into a slow retry with a readable log rather than thousands of lines a minute. * Moving the collector changed its `instance` label from collector:9109 to the new scrape address, which forked every series in two and showed each repo on the board twice until the old ones went stale. There is exactly one collector, so the label carries no information; prometheus.yml now relabels it to a constant and the history stays continuous across any future move. The trade, stated in scripts/collector.sh and docs/operations.md: inside the container the collector could only see the checkouts mounted into it, so an unlisted repo was invisible rather than merely filtered out. It now runs as you and could read anything you can. It still never writes - every git call is read-only and passes --no-optional-locks - but that is now a property of the code rather than something the sandbox enforces. Verified end to end on the real fleet: launchd agent running, 24 of 24 checkouts reporting, Prometheus target up at host.docker.internal:9109, one instance label, no duplicate series, Grafana serving. --- .github/workflows/ci.yml | 47 +++++++++++-------- .gitignore | 12 +++-- README.md | 16 +++++-- collector/Dockerfile | 19 -------- collector/tests/test_fleet.py | 26 +++++------ docker-compose.yml | 42 +++++------------ docs/configuration.md | 26 +++++------ docs/dashboard.md | 8 ++++ docs/operations.md | 36 +++++++++++++++ prometheus/prometheus.yml | 16 ++++++- scripts/collector.sh | 51 ++++++++++++++++++++ scripts/down.sh | 18 +++++--- scripts/gen-repos.py | 87 +++++++++++++---------------------- scripts/up.sh | 71 ++++++++++++++++++++++++---- 14 files changed, 295 insertions(+), 180 deletions(-) delete mode 100644 collector/Dockerfile create mode 100755 scripts/collector.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c9e835..0277d24 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,35 +65,42 @@ jobs: run: python3 scripts/check-dashboard.py compose: - name: compose files parse + name: compose and scripts runs-on: ubuntu-latest - # The overlays only patch services, so they are not valid on their own - - # they are checked the way they are actually used. GITHUB_TOKEN is a - # required variable; any value satisfies interpolation. + # GITHUB_TOKEN is a required variable in the compose file; any value + # satisfies interpolation. env: GITHUB_TOKEN: dummy-value-for-interpolation-only steps: - uses: actions/checkout@v4 + - name: Compose files parse + run: | + docker compose -f docker-compose.yml config --quiet + docker compose -f docker-compose.yml -f docker-compose.admin.yml config --quiet + + # The collector runs on the host now, so these are the entry points a + # user actually touches. A syntax error in one of them used to be caught + # by nothing at all. + - name: Shell scripts parse + run: for f in scripts/*.sh; do bash -n "$f"; done + # repos.yml is gitignored, so CI builds one the way a new user would and - # proves gen-repos.py emits an override that actually merges. Nothing - # else in CI reads it, so a malformed one would otherwise only surface - # on somebody's laptop. - - name: Generate the fleet override + # proves gen-repos.py emits the two lines the collector reads - including + # for a checkout that does NOT sit at //, which is the + # case the paths line exists for. Nothing else in CI reads repos.yml, so + # a malformed one would otherwise only surface on somebody's laptop. + - name: The fleet resolves to an environment run: | pip install --quiet pyyaml printf 'repos:\n' > repos.yml - for full in Jebel-Quant/rhiza cvxgrp/cvxsimulator; do - git init -q -b main "ci-checkouts/$full" - git -C "ci-checkouts/$full" remote add origin "https://github.com/$full.git" - printf ' - path: ci-checkouts/%s\n' "$full" >> repos.yml - done - # A repo with no checkout: GitHub panels only, no bind mount. + git init -q -b main nested/somewhere/rhiza + git -C nested/somewhere/rhiza remote add origin https://github.com/Jebel-Quant/rhiza.git + printf ' - path: nested/somewhere/rhiza\n' >> repos.yml + # A repo with no checkout: GitHub panels only, no path. printf ' - repo: Jebel-Quant/actions\n' >> repos.yml - python3 scripts/gen-repos.py - cat docker-compose.repos.yml - - name: Base + fleet override - run: docker compose -f docker-compose.yml -f docker-compose.repos.yml config --quiet - - name: Base + admin overlay - run: docker compose -f docker-compose.yml -f docker-compose.repos.yml -f docker-compose.admin.yml config --quiet + python3 scripts/gen-repos.py | tee env.out + grep -qx 'JQ_REPOS=Jebel-Quant/rhiza,Jebel-Quant/actions' env.out + grep -q 'JQ_REPO_PATHS=Jebel-Quant/rhiza=.*/nested/somewhere/rhiza$' env.out + test "$(grep -c 'Jebel-Quant/actions=' env.out)" = 0 diff --git a/.gitignore b/.gitignore index d6ea74d..eba6d5f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,14 @@ .env -# Your fleet and the mounts generated from it: both describe the folder layout -# of one machine, which has no business in a public repo. Start from -# repos.example.yml. +# Your fleet: it describes the folder layout of one machine, which has no +# business in a public repo. Start from repos.example.yml. Nothing is generated +# from it any more - scripts/collector.sh reads it at every launch. repos.yml -docker-compose.repos.yml -# Left behind by running the collector natively instead of in its container. +# Where the launchd agent's output goes. +.collector-logs/ + +# Left behind by running the collector, which is how it runs now. .venv/ .ruff_cache/ .pytest_cache/ diff --git a/README.md b/README.md index ee468df..ff3373e 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,10 @@ the board because you listed it in `repos.yml`, and for no other reason. ## What you need -Docker, and the [`gh` CLI](https://cli.github.com) signed in (`up.sh` mints the -token from it — otherwise put a `GITHUB_TOKEN` in `.env` yourself). +Docker (for Prometheus and Grafana), [`uv`](https://docs.astral.sh/uv/) (for the +collector, which runs on your machine rather than in a container), and the +[`gh` CLI](https://cli.github.com) signed in — `up.sh` mints the token from it, +otherwise put a `GITHUB_TOKEN` in `.env` yourself. ## Recipe @@ -34,7 +36,9 @@ repos: - repo: Jebel-Quant/actions # monitored, but not cloned here ``` -`owner/name` comes from each checkout's `origin`, so the path is all you write. +`owner/name` comes from each checkout's `origin`, so the path is all you write, +and the path is used as written — a checkout does not have to live at +`//`. Then: ```bash @@ -43,8 +47,9 @@ open http://localhost:3000/d/jq-fleet ``` The board fills in within a minute — the local panels first, the GitHub panels -after the first API refresh. Both `repos.yml` and the generated -`docker-compose.repos.yml` are gitignored: they describe one machine's folders. +after the first API refresh. `repos.yml` is gitignored: it describes one machine's folders. Nothing is +generated from it — the collector reads it at every launch, so there is no +second file to fall out of step. ## Then what @@ -53,6 +58,7 @@ after the first API refresh. Both `repos.yml` and the generated | Add or drop a repo | edit `repos.yml`, `./scripts/up.sh` again — [details](docs/configuration.md) | | Erase a dropped repo's history | [`./scripts/purge-repo.sh owner/name`](docs/configuration.md#dropping-a-repo) (irreversible) | | Stop | `./scripts/down.sh` (add `--volumes` to discard the history too) | +| See the collector's log | `tail -f .collector-logs/collector.log` — it runs on your machine, not in Docker ([why](docs/operations.md#the-collector-runs-on-your-machine)) | | Edit the board | change `grafana/dashboards/fleet.json`; it reloads in 30s — [read the traps first](docs/dashboard.md#traps-worth-not-re-introducing) | | Get notified | add a contact point under *Alerting → Contact points* — [why it is not provisioned](docs/dashboard.md#alerting) | diff --git a/collector/Dockerfile b/collector/Dockerfile deleted file mode 100644 index 2cebba0..0000000 --- a/collector/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM python:3.12-slim - -# git is the collector's other data source, alongside the GitHub API. -RUN apt-get update \ - && apt-get install -y --no-install-recommends git ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -# The clones are bind-mounted from the host and are owned by a uid this -# container does not have, which git refuses to touch by default. The mount is -# read-only, so widening the trust here cannot lead to a write. -RUN git config --system --add safe.directory '*' - -WORKDIR /app -COPY pyproject.toml README.md ./ -COPY jq_collector ./jq_collector -RUN pip install --no-cache-dir . - -EXPOSE 9109 -CMD ["python", "-m", "jq_collector"] diff --git a/collector/tests/test_fleet.py b/collector/tests/test_fleet.py index 1475123..0bb64fc 100644 --- a/collector/tests/test_fleet.py +++ b/collector/tests/test_fleet.py @@ -436,44 +436,40 @@ def test_a_directory_that_is_not_a_checkout_fails(gen_repos, tmp_path): gen_repos.resolve({"path": str(tmp_path / "empty")}, 1) -def test_env_mode_prints_the_fleet_and_the_paths_and_writes_nothing( +def test_it_prints_the_fleet_and_the_paths_and_writes_nothing( gen_repos, tmp_path, monkeypatch, capsys ): """Both lines must come from repos.yml, not from anyone's memory. Retyping either is how they drift: a repo added here never reaches the - board and nothing reports the difference. The paths line is what lets a - collector running outside the container find a checkout that does not sit - at // - and a repo with no checkout must not appear in - it at all. --env must also leave the compose override alone, so it is safe - to run on a machine that has no stack. + board and nothing reports the difference. The paths line is what lets the + collector find a checkout that does not sit at //, and a + repo with no checkout must not appear in it at all. Nothing is written to + disk - scripts/collector.sh runs this at every launch and exports the + result, so there is no generated file in between to go stale. """ path = make_checkout(tmp_path, "Jebel-Quant", "rhiza") source = tmp_path / "repos.yml" source.write_text(f"repos:\n - path: {path}\n - repo: cvxgrp/cvxsimulator\n") - target = tmp_path / "docker-compose.repos.yml" monkeypatch.setattr(gen_repos, "SOURCE", source) - monkeypatch.setattr(gen_repos, "TARGET", target) - monkeypatch.setattr(sys, "argv", ["gen-repos.py", "--env"]) + monkeypatch.setattr(sys, "argv", ["gen-repos.py"]) gen_repos.main() - lines = capsys.readouterr().out.strip().splitlines() - assert lines == [ + assert capsys.readouterr().out.strip().splitlines() == [ "JQ_REPOS=Jebel-Quant/rhiza,cvxgrp/cvxsimulator", f"JQ_REPO_PATHS=Jebel-Quant/rhiza={path}", ] - assert not target.exists() + assert list(tmp_path.glob("docker-compose*")) == [] -def test_env_mode_refuses_a_path_it_cannot_express(gen_repos, tmp_path, monkeypatch, capsys): +def test_it_refuses_a_path_it_cannot_express(gen_repos, tmp_path, monkeypatch, capsys): """A comma is the separator, so a path holding one would read as two repos.""" path = make_checkout(tmp_path, "Jebel-Quant", "rhi,za") source = tmp_path / "repos.yml" source.write_text(f"repos:\n - path: {path}\n") monkeypatch.setattr(gen_repos, "SOURCE", source) - monkeypatch.setattr(gen_repos, "TARGET", tmp_path / "out.yml") - monkeypatch.setattr(sys, "argv", ["gen-repos.py", "--env"]) + monkeypatch.setattr(sys, "argv", ["gen-repos.py"]) with pytest.raises(SystemExit): gen_repos.main() diff --git a/docker-compose.yml b/docker-compose.yml index 71b232f..ae1e8e1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,37 +1,15 @@ -# The laptop stack. Not usable on its own - the fleet lives in repos.yml, -# so bring it up with scripts/up.sh, or with: +# Prometheus and Grafana. The collector is NOT here: it runs on the host, via +# scripts/collector.sh, because it reads your working copies and a container can +# only see them through bind mounts - which cannot name a checkout that sits +# somewhere other than //, and which are slow enough on +# macOS to have needed a caching layer to work around. # -# python3 scripts/gen-repos.py -# docker compose -f docker-compose.yml -f docker-compose.repos.yml up -d +# ./scripts/up.sh starts both halves +# docker compose up -d starts only this half name: jq-monitoring services: - collector: - build: ./collector - container_name: jq-collector - restart: unless-stopped - environment: - GITHUB_TOKEN: ${GITHUB_TOKEN:?set GITHUB_TOKEN in monitoring/.env - scripts/up.sh does it for you} - # JQ_REPOS and the per-repo bind mounts both come from - # docker-compose.repos.yml, generated from repos.yml. This file has no - # opinion about which repos are yours. - JQ_REPO_ROOT: /repos - JQ_TEMPLATE_REPO: ${JQ_TEMPLATE_REPO:-Jebel-Quant/rhiza} - JQ_GITHUB_INTERVAL: ${JQ_GITHUB_INTERVAL:-600} - JQ_LOCAL_INTERVAL: ${JQ_LOCAL_INTERVAL:-60} - JQ_MEASURE_MAX_AGE: ${JQ_MEASURE_MAX_AGE:-86400} - JQ_IGNORE: ${JQ_IGNORE:-} - JQ_PUBLIC_ONLY: ${JQ_PUBLIC_ONLY:-false} - JQ_LOG_LEVEL: ${JQ_LOG_LEVEL:-INFO} - # No volumes here on purpose. Every checkout is mounted individually by - # docker-compose.repos.yml, read-only: the collector reports on your working - # copies and must never be able to touch them, and mounting only what is - # listed means an unlisted repo is not merely filtered out - it is not even - # visible to the container. Every git call is --no-optional-locks too. - ports: - - "127.0.0.1:9109:9109" - prometheus: image: prom/prometheus:v2.55.1 container_name: jq-prometheus @@ -46,8 +24,10 @@ services: - prometheus-data:/prometheus ports: - "127.0.0.1:9090:9090" - depends_on: - - collector + # Docker Desktop resolves host.docker.internal on its own; plain Docker on + # Linux does not, and without this the collector target is simply down. + extra_hosts: + - "host.docker.internal:host-gateway" grafana: image: grafana/grafana:11.3.1 diff --git a/docs/configuration.md b/docs/configuration.md index 8526031..d0e1ef4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -87,27 +87,23 @@ better a refusal than a board that is quietly one repo short. | `JQ_IGNORE` | — | Repos to drop without editing `repos.yml`, as bare names or `owner/name`. Applies to both halves. | | `JQ_INCLUDE_ARCHIVED` | `false` | Archived repos are dropped from both halves. | | `JQ_PUBLIC_ONLY` | `false` | Drop private repos entirely — not just their details, their existence. | -| `JQ_REPO_PATHS` | — | Where each checkout really is, as `owner/name=path` pairs. Only needed outside the container — see [Checkout paths](#checkout-paths). | +| `JQ_REPO_PATHS` | — | Where each checkout really is, as `owner/name=path` pairs. `scripts/collector.sh` sets it from `repos.yml` — see [Checkout paths](#checkout-paths). | | `JQ_GITHUB_INTERVAL` | `300` | Seconds between GitHub refreshes. | | `JQ_MEASURE_MAX_AGE` | `86400` | Seconds an unchanged line/commit count may stand before it is retaken. See [Size and cadence](dashboard.md#size-and-cadence). | | `PROM_RETENTION` | `180d` | How much history to keep. | -`scripts/gen-repos.py` turns `repos.yml` into the `JQ_REPOS` list the -collector actually reads, so both halves see the same fleet. Setting `JQ_REPOS` -by hand works too, and skips `repos.yml` entirely — but then nothing mounts the -checkouts, so only the GitHub panels have anything to say. +`scripts/gen-repos.py` turns `repos.yml` into the `JQ_REPOS` list the collector +actually reads, so both halves see the same fleet. Setting `JQ_REPOS` by hand +works too and skips `repos.yml` entirely — but then no checkout paths are set +either, so only the GitHub panels have anything to say. ## Checkout paths The collector reads a repo at the path `JQ_REPO_PATHS` gives for it, and failing that at `//`. -**In the container only the fallback is used, and it is always right**, because -the generated bind mounts put every checkout at exactly that path. `repos.yml` -can say anything and the mount normalises it. - -**Outside the container it is often wrong**, because paths are whatever they are -on disk. An entry like +The collector runs on your machine, so paths are whatever they are on disk and +the fallback is often wrong. An entry like ```yaml - path: ~/repos/tschm/rhiza_projects/cs # this is tschm/cs @@ -118,11 +114,13 @@ working-copy panels while staying on the GitHub ones — present on the board, a quietly missing half its columns. Four repos in the fleet this was built against are laid out that way. -So when you run the collector on the host, take both lines from `repos.yml` -rather than writing them out: +You do not normally set this yourself: `scripts/collector.sh` runs +`scripts/gen-repos.py` at every launch and exports both lines, so `repos.yml` +stays the only place the fleet and the layout are written down. To see what it +resolves to: ```bash -uv run --with pyyaml python scripts/gen-repos.py --env >> .env +uv run --with pyyaml python scripts/gen-repos.py ``` A path containing a comma cannot be expressed — comma is the separator — and diff --git a/docs/dashboard.md b/docs/dashboard.md index 68df28c..2b190a7 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -107,6 +107,14 @@ name to `owner/name` when the cvxgrp repos were added, so history from before that point lives under the old names. Windows spanning the change show both generations; they age out. This is inherent to Prometheus, not a misconfiguration. +**The `instance` label is pinned on purpose.** There is one collector, so +`instance` says only where the process happened to be running — and left alone +it is the scrape address. When the collector moved out of its container the +address changed, every series forked in two, and for one stale window each repo +appeared on the board twice. `prometheus.yml` relabels it to a constant so that +cannot recur; series from before the move keep the old value and age out with +retention. + **Never `group_left` onto an `_info` metric without a `topk` guard.** The `_info` metrics carry a label that changes value — `ref`, `branch`, `conclusion`, `checks`. When one changes, Prometheus returns the old *and* the diff --git a/docs/operations.md b/docs/operations.md index d033001..02e4334 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -6,6 +6,42 @@ keywords: grafana login, no data, prometheus staleness, macos sleep, caffeinate # Running it day to day +## The collector runs on your machine + +Two halves, not three containers. Prometheus and Grafana are in Docker; +**the collector is an ordinary process on your Mac**, started by `up.sh` as a +launchd agent and stopped by `down.sh`. + +It is not containerised because it reads your working copies, and a container +can only reach those through bind mounts. Those mounts have to place every +checkout at `//`, which silently loses any repo that lives +somewhere else, and reading thousands of small files back through them on macOS +is slow enough that the line counts needed a cache to stay affordable. On the +host both problems disappear. + +| | | +|---|---| +| Its log | `.collector-logs/collector.log` | +| Restart it | `launchctl kickstart -k gui/$UID/com.jebel-quant.jq-collector` | +| Run it in the foreground instead | `./scripts/collector.sh` (Ctrl-C stops it) | +| Is Prometheus reaching it? | — the `jq-collector` job | + +Prometheus scrapes it at `host.docker.internal:9109`, which is how a container +reaches the machine it runs on. + +**The trade.** Inside the container the collector could only see the checkouts +mounted into it, so an unlisted repo was not merely filtered out — it was +invisible. It now runs as you and could read anything you can. It still never +writes: every git call is read-only and passes `--no-optional-locks`. But that +is now a property of the code rather than something the sandbox enforces. + +**If it will not start**, the usual cause is `PATH`. A launchd agent inherits +`/usr/bin:/bin:/usr/sbin:/sbin` and nothing else, so a `uv` under +`/opt/homebrew` or `~/.local` is invisible to it. `up.sh` pins `uv`'s directory +into the plist when it installs the agent, so re-running `./scripts/up.sh` +after moving or reinstalling `uv` is the fix. + + ## The "Sign in" button Grafana's own local login, against a SQLite file in the `grafana-data` volume on diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index 24ac892..9e47616 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -9,8 +9,22 @@ global: scrape_configs: - job_name: jq-collector + # The collector runs on the host, not in this stack - it reads your working + # copies at their real paths, which a bind mount cannot express and which + # is far faster than one anyway. host.docker.internal is how a container + # reaches the machine it runs on; docker-compose.yml maps it on Linux, + # where it is not built in. static_configs: - - targets: ["collector:9109"] + - targets: ["host.docker.internal:9109"] + relabel_configs: + # There is exactly one collector, so `instance` carries no information - + # only where the process happened to be running. Left alone it is the + # scrape address, and moving the collector (container -> host, or back) + # silently forks every series in two: the board then joins a repo against + # itself and shows each one twice until the old series go stale. Pin it, + # and the history stays continuous across the move. + - target_label: instance + replacement: jq-collector - job_name: prometheus static_configs: diff --git a/scripts/collector.sh b/scripts/collector.sh new file mode 100755 index 0000000..cd3b91a --- /dev/null +++ b/scripts/collector.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Run the collector on this machine, in the foreground. +# +# Not in a container, on purpose. It reads your working copies, and a container +# can only reach those through bind mounts - which cannot name a checkout that +# sits somewhere other than //, and which are slow enough on +# macOS that the line counts needed a caching layer to stay affordable. +# +# Prometheus, which is still in Docker, scrapes this at host.docker.internal:9109. +# +# Run it under launchd (./scripts/up.sh sets that up on macOS), in a terminal, +# or under tmux - it is an ordinary foreground process either way, and Ctrl-C +# stops it. +# +# The trade this makes: inside the container the collector could only see the +# checkouts that were mounted into it, so an unlisted repo was not merely +# filtered out, it was invisible. Here it runs as you and could read anything +# you can. It still never writes - every git call is read-only and passes +# --no-optional-locks - but that is now a property of the code rather than +# something the sandbox enforces. +set -euo pipefail +cd "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +die() { printf '\033[31merror\033[0m %s\n' "$1" >&2; exit 1; } + +[[ -f .env ]] || die ".env not found - run ./scripts/up.sh first" +[[ -f repos.yml ]] || die "repos.yml not found - run ./scripts/up.sh first" +# Says "on PATH", not "installed", because under launchd it is almost always +# the former: an agent inherits /usr/bin:/bin:/usr/sbin:/sbin and nothing else, +# so a perfectly good uv under /opt/homebrew or ~/.local is invisible. up.sh +# pins uv's directory into the plist for exactly this reason. +command -v uv >/dev/null 2>&1 || die "uv is not on PATH ($PATH) - see https://docs.astral.sh/uv/" + +set -a; . ./.env; set +a + +# repos.yml is the fleet and the layout, and it stays the only place either is +# written down: both lines are computed at every launch rather than generated +# into a file that can fall out of step with it. +# +# `export "$line"` and not `export $line` - a path may contain spaces, and +# unquoted word splitting would turn one repo into two broken ones. +while IFS= read -r line; do + [[ -n "$line" ]] && export "$line" +done < <(uv run --quiet --with pyyaml python scripts/gen-repos.py) + +# No container means no bind mounts and no /repos to fall back on. Every path +# is real and absolute, and JQ_REPO_PATHS carries all of them. +export JQ_REPO_ROOT="" + +cd collector +exec uv run --quiet python -m jq_collector diff --git a/scripts/down.sh b/scripts/down.sh index 7042271..cba596f 100755 --- a/scripts/down.sh +++ b/scripts/down.sh @@ -1,9 +1,15 @@ #!/usr/bin/env bash -# Stop the stack. Add --volumes to also throw away the Prometheus history. +# Stop both halves. Add --volumes to also throw away the Prometheus history. set -euo pipefail cd "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -# The generated override may be missing on a fresh clone; the project name in -# docker-compose.yml is enough to find the running containers either way. -files=(-f docker-compose.yml) -[[ -f docker-compose.repos.yml ]] && files+=(-f docker-compose.repos.yml) -docker compose "${files[@]}" down "$@" + +label="com.jebel-quant.jq-collector" +if [[ "$(uname -s)" == "Darwin" ]]; then + # KeepAlive means the agent restarts itself if merely killed, so unload it. + # Fails when it was never loaded, which is fine. + launchctl bootout "gui/$UID/$label" 2>/dev/null && echo "collector: launchd agent unloaded" || true +else + pkill -f "python -m jq_collector" 2>/dev/null && echo "collector: stopped" || true +fi + +docker compose down "$@" diff --git a/scripts/gen-repos.py b/scripts/gen-repos.py index dea4e0c..0b1965e 100755 --- a/scripts/gen-repos.py +++ b/scripts/gen-repos.py @@ -1,25 +1,25 @@ #!/usr/bin/env python3 -"""Turn repos.yml into the compose override that mounts the fleet. +"""Turn repos.yml into the two environment lines the collector reads. The collector never searches for checkouts. This script resolves every listed -path, asks its origin remote who it is, and writes one read-only bind mount per -repo at /repos// plus the matching JQ_REPOS list. Both halves of -the collector then read the same list, and the board's contents are decided by -repos.yml alone. +path, asks its origin remote who it is, and prints: -Run it after editing repos.yml; scripts/up.sh does that for you. + JQ_REPOS the fleet, as owner/name - read by both halves + JQ_REPO_PATHS where each checkout actually is - read by the local half -With ``--env`` it writes nothing and prints the environment instead - JQ_REPOS, -and JQ_REPO_PATHS naming where each checkout really is: + python3 scripts/gen-repos.py - python3 scripts/gen-repos.py --env +Nothing is written. scripts/collector.sh runs this at every launch and exports +the result, so repos.yml stays the only place the fleet and the layout are +written down - there is no generated file in between to go stale. -That second line is what makes a collector running outside the container agree -with repos.yml. In the container the bind mounts normalise every checkout onto -/repos//, so the path is implied; on the host it is not, because a -repos.yml entry like ~/repos/tschm/rhiza_projects/cs for tschm/cs cannot be -recovered by joining owner to name. Without the paths those repos silently -vanish from the working-copy panels while staying on the GitHub ones. +The paths line exists because a repos.yml entry like + + - path: ~/repos/tschm/rhiza_projects/cs # a checkout of tschm/cs + +cannot be recovered by joining owner to name. This used to be papered over by +bind-mounting every checkout onto /repos//, back when the collector +ran in a container; it does not, so the real path has to be carried. """ from __future__ import annotations @@ -31,7 +31,6 @@ ROOT = Path(__file__).resolve().parent.parent SOURCE = ROOT / "repos.yml" -TARGET = ROOT / "docker-compose.repos.yml" def fail(message: str) -> None: @@ -117,10 +116,9 @@ def resolve(entry: object, index: int) -> tuple[str, Path | None]: def main() -> None: - env_only = "--env" in sys.argv[1:] - unknown = [a for a in sys.argv[1:] if a != "--env"] + unknown = sys.argv[1:] if unknown: - fail(f"unknown argument {unknown[0]} - the only flag is --env") + fail(f"unknown argument {unknown[0]} - this script takes none") resolved: dict[str, Path | None] = {} for index, entry in enumerate(load_entries(), start=1): @@ -129,43 +127,20 @@ def main() -> None: fail(f"{full_name} is listed twice") resolved[full_name] = path - # Nothing but the lines themselves on stdout, so they can be piped or - # appended to a .env. - if env_only: - print(f'JQ_REPOS={",".join(resolved)}') - paths = {name: path for name, path in resolved.items() if path is not None} - # A comma is the separator, so a path containing one cannot be - # expressed. Refuse rather than emit a line the collector would reject - # or, worse, silently misread as two repos. - for name, path in paths.items(): - if "," in str(path): - fail(f"{name}: path contains a comma, which JQ_REPO_PATHS cannot express: {path}") - if paths: - joined = ",".join(f"{name}={path}" for name, path in paths.items()) - print(f"JQ_REPO_PATHS={joined}") - return - - lines = [ - "# Generated by scripts/gen-repos.py from repos.yml - do not edit.", - "#", - "# One read-only bind mount per monitored checkout, at the canonical", - "# /repos// the collector looks for, plus the matching", - "# JQ_REPOS list so the GitHub half sees exactly the same fleet.", - "", - "services:", - " collector:", - " environment:", - f' JQ_REPOS: "{",".join(resolved)}"', - ] - mounts = {name: path for name, path in resolved.items() if path is not None} - if mounts: - lines.append(" volumes:") - lines += [f' - "{path}:/repos/{name}:ro"' for name, path in mounts.items()] - TARGET.write_text("\n".join(lines) + "\n", encoding="utf-8") - - without = len(resolved) - len(mounts) - tail = f", {without} without a local checkout" if without else "" - print(f"gen-repos: wrote {TARGET.name} - {len(resolved)} repos{tail}") + # Nothing but the lines themselves on stdout, so they can be exported, + # piped, or appended to a .env. + print(f'JQ_REPOS={",".join(resolved)}') + + paths = {name: path for name, path in resolved.items() if path is not None} + # A comma is the separator, so a path containing one cannot be expressed. + # Refuse rather than print a line the collector would reject or, worse, + # silently misread as two repos. + for name, path in paths.items(): + if "," in str(path): + fail(f"{name}: path contains a comma, which JQ_REPO_PATHS cannot express: {path}") + if paths: + joined = ",".join(f"{name}={path}" for name, path in paths.items()) + print(f"JQ_REPO_PATHS={joined}") if __name__ == "__main__": diff --git a/scripts/up.sh b/scripts/up.sh index ac0579f..e16e40f 100755 --- a/scripts/up.sh +++ b/scripts/up.sh @@ -1,5 +1,10 @@ #!/usr/bin/env bash -# Bring the stack up, minting a GitHub token from the gh CLI if there isn't one. +# Bring both halves up: Prometheus and Grafana in Docker, the collector on this +# machine. Mints a GitHub token from the gh CLI if there isn't one. +# +# The collector is not containerised - see scripts/collector.sh for why. On +# macOS it is installed as a launchd agent so it comes back after a reboot; +# elsewhere this script says how to run it and leaves supervision to you. set -euo pipefail here="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" @@ -31,16 +36,66 @@ if ! grep -qE '^GITHUB_TOKEN=.+' .env; then echo "wrote a token from 'gh auth token' into monitoring/.env" fi -# repos.yml is the fleet. Regenerate the mounts every time, so editing the list -# and running ./scripts/up.sh is the whole workflow. -if command -v uv >/dev/null 2>&1; then - uv run --quiet --with pyyaml scripts/gen-repos.py +command -v uv >/dev/null 2>&1 || { + echo "uv is not installed - the collector needs it; see https://docs.astral.sh/uv/" >&2 + exit 1 +} + +# Fail on a broken repos.yml here, while there is still a human watching, rather +# than inside a launchd agent whose output nobody is reading. The collector +# recomputes this at every launch; this run is only a check. +uv run --quiet --with pyyaml python scripts/gen-repos.py >/dev/null + +docker compose up -d + +# -- the collector ----------------------------------------------------------- +label="com.jebel-quant.jq-collector" +if [[ "$(uname -s)" == "Darwin" ]]; then + plist="$HOME/Library/LaunchAgents/$label.plist" + # A launchd agent inherits almost no PATH - /usr/bin:/bin:/usr/sbin:/sbin and + # nothing else - so uv, which lives under /opt/homebrew or ~/.local, is simply + # not found. Pin the directory it is actually in rather than guessing at + # either location. + uv_dir="$(dirname "$(command -v uv)")" + logs="$here/.collector-logs" + mkdir -p "$HOME/Library/LaunchAgents" "$logs" + # Regenerated every time: the paths are absolute, so a moved checkout of this + # repo would otherwise leave a plist pointing at where it used to be. + cat > "$plist" < + + + + Label$label + ProgramArguments + $here/scripts/collector.sh + WorkingDirectory$here + EnvironmentVariables + + PATH$uv_dir:/usr/bin:/bin:/usr/sbin:/sbin + + RunAtLoad + KeepAlive + + ThrottleInterval10 + StandardOutPath$logs/collector.log + StandardErrorPath$logs/collector.log + + +PLIST + # bootout first so an edited plist is actually re-read; it fails when nothing + # is loaded, which is the normal first run. + launchctl bootout "gui/$UID/$label" 2>/dev/null || true + launchctl bootstrap "gui/$UID" "$plist" + echo "collector: loaded as a launchd agent - logs in .collector-logs/collector.log" else - python3 scripts/gen-repos.py + echo "collector: not started. Run it yourself, e.g." + echo " ./scripts/collector.sh # foreground" + echo " or install it as a systemd --user service running that script." fi -docker compose -f docker-compose.yml -f docker-compose.repos.yml up -d --build - echo echo "Grafana http://localhost:3000/d/jq-fleet (anonymous read-only; admin/admin to edit)" echo "Prometheus http://localhost:9090"