Read a checkout where it actually is

.github/workflows/ci.yml

@@ -65,35 +65,42 @@ jobs:
         run: python3 scripts/check-dashboard.py
 
   compose:
-    name: compose files parse
+    name: compose and scripts
     runs-on: ubuntu-latest
-    # The overlays only patch services, so they are not valid on their own -
-    # they are checked the way they are actually used. GITHUB_TOKEN is a
-    # required variable; any value satisfies interpolation.
+    # GITHUB_TOKEN is a required variable in the compose file; any value
+    # satisfies interpolation.
     env:
       GITHUB_TOKEN: dummy-value-for-interpolation-only
     steps:
       - uses: actions/checkout@v4
 
+      - name: Compose files parse
+        run: |
+          docker compose -f docker-compose.yml config --quiet
+          docker compose -f docker-compose.yml -f docker-compose.admin.yml config --quiet
+
+      # The collector runs on the host now, so these are the entry points a
+      # user actually touches. A syntax error in one of them used to be caught
+      # by nothing at all.
+      - name: Shell scripts parse
+        run: for f in scripts/*.sh; do bash -n "$f"; done
+
       # repos.yml is gitignored, so CI builds one the way a new user would and
-      # proves gen-repos.py emits an override that actually merges. Nothing
-      # else in CI reads it, so a malformed one would otherwise only surface
-      # on somebody's laptop.
-      - name: Generate the fleet override
+      # proves gen-repos.py emits the two lines the collector reads - including
+      # for a checkout that does NOT sit at <root>/<owner>/<name>, which is the
+      # case the paths line exists for. Nothing else in CI reads repos.yml, so
+      # a malformed one would otherwise only surface on somebody's laptop.
+      - name: The fleet resolves to an environment
         run: |
           pip install --quiet pyyaml
           printf 'repos:\n' > repos.yml
-          for full in Jebel-Quant/rhiza cvxgrp/cvxsimulator; do
-            git init -q -b main "ci-checkouts/$full"
-            git -C "ci-checkouts/$full" remote add origin "https://github.com/$full.git"
-            printf '  - path: ci-checkouts/%s\n' "$full" >> repos.yml
-          done
-          # A repo with no checkout: GitHub panels only, no bind mount.
+          git init -q -b main nested/somewhere/rhiza
+          git -C nested/somewhere/rhiza remote add origin https://github.com/Jebel-Quant/rhiza.git
+          printf '  - path: nested/somewhere/rhiza\n' >> repos.yml
+          # A repo with no checkout: GitHub panels only, no path.
           printf '  - repo: Jebel-Quant/actions\n' >> repos.yml
-          python3 scripts/gen-repos.py
-          cat docker-compose.repos.yml
 
-      - name: Base + fleet override
-        run: docker compose -f docker-compose.yml -f docker-compose.repos.yml config --quiet
-      - name: Base + admin overlay
-        run: docker compose -f docker-compose.yml -f docker-compose.repos.yml -f docker-compose.admin.yml config --quiet
+          python3 scripts/gen-repos.py | tee env.out
+          grep -qx 'JQ_REPOS=Jebel-Quant/rhiza,Jebel-Quant/actions' env.out
+          grep -q 'JQ_REPO_PATHS=Jebel-Quant/rhiza=.*/nested/somewhere/rhiza$' env.out
+          test "$(grep -c 'Jebel-Quant/actions=' env.out)" = 0

.gitignore

@@ -1,12 +1,14 @@
 .env
 
-# Your fleet and the mounts generated from it: both describe the folder layout
-# of one machine, which has no business in a public repo. Start from
-# repos.example.yml.
+# Your fleet: it describes the folder layout of one machine, which has no
+# business in a public repo. Start from repos.example.yml. Nothing is generated
+# from it any more - scripts/collector.sh reads it at every launch.
 repos.yml
-docker-compose.repos.yml
 
-# Left behind by running the collector natively instead of in its container.
+# Where the launchd agent's output goes.
+.collector-logs/
+
+# Left behind by running the collector, which is how it runs now.
 .venv/
 .ruff_cache/
 .pytest_cache/

README.md

@@ -15,8 +15,10 @@ the board because you listed it in `repos.yml`, and for no other reason.
 
 ## What you need
 
-Docker, and the [`gh` CLI](https://cli.github.com) signed in (`up.sh` mints the
-token from it — otherwise put a `GITHUB_TOKEN` in `.env` yourself).
+Docker (for Prometheus and Grafana), [`uv`](https://docs.astral.sh/uv/) (for the
+collector, which runs on your machine rather than in a container), and the
+[`gh` CLI](https://cli.github.com) signed in — `up.sh` mints the token from it,
+otherwise put a `GITHUB_TOKEN` in `.env` yourself.
 
 ## Recipe
 
@@ -34,7 +36,9 @@ repos:
   - repo: Jebel-Quant/actions           # monitored, but not cloned here
 ```
 
-`owner/name` comes from each checkout's `origin`, so the path is all you write.
+`owner/name` comes from each checkout's `origin`, so the path is all you write,
+and the path is used as written — a checkout does not have to live at
+`<root>/<owner>/<name>`.
 Then:
 
 ```bash
@@ -43,8 +47,9 @@ open http://localhost:3000/d/jq-fleet
 ```
 
 The board fills in within a minute — the local panels first, the GitHub panels
-after the first API refresh. Both `repos.yml` and the generated
-`docker-compose.repos.yml` are gitignored: they describe one machine's folders.
+after the first API refresh. `repos.yml` is gitignored: it describes one machine's folders. Nothing is
+generated from it — the collector reads it at every launch, so there is no
+second file to fall out of step.
 
 ## Then what
 
@@ -53,6 +58,7 @@ after the first API refresh. Both `repos.yml` and the generated
 | Add or drop a repo | edit `repos.yml`, `./scripts/up.sh` again — [details](docs/configuration.md) |
 | Erase a dropped repo's history | [`./scripts/purge-repo.sh owner/name`](docs/configuration.md#dropping-a-repo) (irreversible) |
 | Stop | `./scripts/down.sh` (add `--volumes` to discard the history too) |
+| See the collector's log | `tail -f .collector-logs/collector.log` — it runs on your machine, not in Docker ([why](docs/operations.md#the-collector-runs-on-your-machine)) |
 | Edit the board | change `grafana/dashboards/fleet.json`; it reloads in 30s — [read the traps first](docs/dashboard.md#traps-worth-not-re-introducing) |
 | Get notified | add a contact point under *Alerting → Contact points* — [why it is not provisioned](docs/dashboard.md#alerting) |
 

collector/jq_collector/config.py

@@ -20,6 +20,25 @@ def _csv(name: str, default: tuple[str, ...] = ()) -> tuple[str, ...]:
     return tuple(part.strip() for part in raw.split(",") if part.strip())
 
 
+def _pairs(name: str) -> dict[str, str]:
+    """``owner/name=path`` pairs, comma separated, into a mapping.
+
+    Raises rather than skipping a malformed entry. A dropped pair would take
+    one repo's working-copy panels off the board and say nothing about why,
+    which is the failure mode this whole module is shaped to avoid; refusing to
+    start is louder and cheaper to diagnose. A path containing a comma cannot
+    be expressed here - ``scripts/gen-repos.py`` refuses to emit one.
+    """
+    mapping: dict[str, str] = {}
+    for item in _csv(name):
+        key, separator, value = item.partition("=")
+        key, value = key.strip(), value.strip()
+        if not separator or not key or not value:
+            raise ValueError(f"{name}: expected owner/name=path, got {item!r}")
+        mapping[key] = os.path.expanduser(value)
+    return mapping
+
+
 @dataclass(frozen=True)
 class Config:
     """Everything the collector needs to know about its environment.
@@ -43,12 +62,21 @@ class Config:
     token: str = os.environ.get("GITHUB_TOKEN", "")
     api: str = os.environ.get("GITHUB_API", "https://api.github.com")
 
-    # Where the checkouts are mounted (read-only) inside the container, one per
-    # repo at <repo_root>/<owner>/<name>. Set empty to skip local scanning
-    # entirely, which is the right setting anywhere there are no working copies
-    # to report on - the local panels then simply have nothing to say.
+    # Where the checkouts are, for repos JQ_REPO_PATHS does not name: one per
+    # repo at <repo_root>/<owner>/<name>. That is exactly where the generated
+    # compose override bind-mounts them, so in the container this is all that
+    # is needed. Set empty to skip local scanning entirely, which is the right
+    # setting anywhere there are no working copies to report on - the local
+    # panels then simply have nothing to say.
     repo_root: str = os.environ.get("JQ_REPO_ROOT", "/repos")
 
+    # Explicit path per repo, as owner/name=path. Needed whenever a checkout
+    # does not sit at <repo_root>/<owner>/<name> - which is normal outside the
+    # container, where paths are whatever they are on disk rather than whatever
+    # a bind mount normalised them to. `scripts/gen-repos.py --env` emits this
+    # from repos.yml, so the layout is stated once and in one place.
+    repo_paths: dict[str, str] = field(default_factory=lambda: _pairs("JQ_REPO_PATHS"))
+
     # owner/name of the repo whose releases define "up to date".
     template_repo: str = os.environ.get("JQ_TEMPLATE_REPO", "Jebel-Quant/rhiza")
     template_pointer: str = os.environ.get("JQ_TEMPLATE_POINTER", ".rhiza/template.yml")

collector/jq_collector/localgit.py

@@ -17,10 +17,14 @@
 drift is the one thing that changes while the clone stands still, because it is
 the upstream that moved.
 
-Nothing here searches for checkouts either. Each monitored repo is expected at
-``<repo_root>/<owner>/<name>``, which is exactly where the generated compose
-override mounts it. The fleet is decided by ``repos.yml``, not by whatever
-happened to be lying around under a scanned directory.
+Nothing here searches for checkouts either. A repo is read at the path
+``JQ_REPO_PATHS`` gives for it, or failing that at ``<repo_root>/<owner>/<name>``
+- which is exactly where the generated compose override mounts it, so inside the
+container the default is always right. Outside it, paths are whatever they are on
+disk: ``repos.yml`` may well say ``~/repos/tschm/rhiza_projects/cs`` for
+``tschm/cs``, and no amount of joining owner to name will produce that. Either
+way the fleet is decided by ``repos.yml``, not by whatever happened to be lying
+around under a scanned directory.
 """
 
 from __future__ import annotations
@@ -354,20 +358,29 @@ def scan(
     """
     prior = previous or {}
     found: dict[str, LocalRepo] = {}
-    if not cfg.repo_root:
-        # Deliberate: with no repo root there are no working copies to report
-        # on, and an empty result is the honest answer rather than an error
-        # every minute.
-        return found
-    if not os.path.isdir(cfg.repo_root):
-        log.error("repo root %s is not a directory", cfg.repo_root)
+
+    # A configured root that does not exist is worth one error, not one per
+    # repo per minute. Explicit paths are unaffected by it, so this only
+    # withdraws the fallback rather than abandoning the whole scan.
+    root = cfg.repo_root
+    if root and not os.path.isdir(root):
+        log.error("repo root %s is not a directory", root)
+        root = ""
+    if not root and not cfg.repo_paths:
+        # Deliberate: nothing points at a working copy, so there are none to
+        # report on, and an empty result is the honest answer rather than an
+        # error every minute.
         return found
 
     for key in cfg.repos:
         if "/" not in key or key in skip:
             continue
         owner, repo_name = key.split("/", 1)
-        path = os.path.join(cfg.repo_root, owner, repo_name)
+        path = cfg.repo_paths.get(key)
+        if path is None:
+            if not root:
+                continue
+            path = os.path.join(root, owner, repo_name)
         # `.git` is a directory in a plain checkout and a file in a worktree.
         if not os.path.exists(os.path.join(path, ".git")):
             # Not mounted, or mounted somewhere else. Normal for a repo you