diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..0cbbd0a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,15 @@ +# Your machine's business, and it is mounted at runtime rather than baked in. +repos.yml +.env + +_book/ +docs/ +.git/ +.github/ +.collector-logs/ +**/.venv/ +**/.ruff_cache/ +**/.pytest_cache/ +**/__pycache__/ +**/_tests/ +**/.coverage diff --git a/.env.example b/.env.example index 26b44cb..001c6e2 100644 --- a/.env.example +++ b/.env.example @@ -1,9 +1,11 @@ -# Copy to .env, or let scripts/up.sh generate it from `gh auth token`. +# Only needed if you start the board with `docker compose up` rather than +# `docker run`; with docker run these are plain -e flags. # # Which repos are monitored is NOT set here - that lives in repos.yml, one -# entry per checkout. This file holds credentials and cadences only. +# entry per repo. This file holds credentials and cadences only. -# Needs `repo` (private repo metadata) and `read:org`. A gh OAuth token works. +# Needs `repo` (private repo metadata) and `read:org`. A gh OAuth token works: +# GITHUB_TOKEN=$(gh auth token) # It has to be able to read every repo listed in repos.yml. GITHUB_TOKEN= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd9a0f2..6995c53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,43 +81,81 @@ jobs: - name: Panel ids, link targets and grid overlaps run: python3 scripts/check-dashboard.py - compose: - name: compose and scripts + image: + name: image builds and comes up runs-on: ubuntu-latest - # GITHUB_TOKEN is a required variable in the compose file; any value - # satisfies interpolation. - env: - GITHUB_TOKEN: dummy-value-for-interpolation-only steps: - uses: actions/checkout@v4 - - name: Compose files parse - run: | - docker compose -f docker-compose.yml config --quiet - docker compose -f docker-compose.yml -f docker-compose.admin.yml config --quiet + - name: Build + run: docker build -t jq-monitoring:ci . - # The collector runs on the host now, so these are the entry points a - # user actually touches. A syntax error in one of them used to be caught - # by nothing at all. + # The entry points a user actually touches. A syntax error in either used + # to be caught by nothing at all - and now they are the container's PID 1 + # and its only maintenance command, so a broken one is a board that does + # not start. - name: Shell scripts parse - run: for f in scripts/*.sh; do bash -n "$f"; done - - # repos.yml is gitignored, so CI builds one the way a new user would and - # proves gen-repos.py emits the two lines the collector reads - including - # for a checkout that does NOT sit at //, which is the - # case the paths line exists for. Nothing else in CI reads repos.yml, so - # a malformed one would otherwise only surface on somebody's laptop. - - name: The fleet resolves to an environment + run: for f in image/*.sh; do bash -n "$f"; done + + - name: Compose file parses + env: + GITHUB_TOKEN: dummy-value-for-interpolation-only + run: docker compose -f docker-compose.yml config --quiet + + # repos.yml is gitignored, so CI writes one the way a new user would. + # This is the whole install path in one step: a checkout that does NOT sit + # at // - the case the paths in repos.yml exist for - + # plus a repo with no checkout at all, and the board has to come up + # serving metrics for both. Nothing else in CI reads repos.yml, so a + # regression here would otherwise only surface on somebody's laptop. + - name: It comes up and serves the fleet + run: | + mkdir -p home/nested/somewhere + git init -q -b main home/nested/somewhere/rhiza + git -C home/nested/somewhere/rhiza remote add origin \ + https://github.com/Jebel-Quant/rhiza.git + cat > repos.yml <<'YML' + repos: + - path: ~/nested/somewhere/rhiza + - repo: Jebel-Quant/actions + YML + + # No GITHUB_TOKEN on purpose: this asserts the container comes up and + # resolves the fleet off repos.yml, which needs no network. The + # GitHub half will fail (several endpoints 401 unauthenticated), so + # nothing below may depend on it having answered. + docker run -d --name fleet \ + -p 127.0.0.1:9109:9109 \ + -v "$PWD/repos.yml:/config/repos.yml:ro" \ + -v "$PWD/home:/host:ro" \ + jq-monitoring:ci + + for _ in $(seq 1 60); do + curl -fsS http://localhost:9109/metrics -o metrics.txt && break + docker ps -q -f name=fleet | grep -q . || { docker logs fleet; exit 1; } + sleep 5 + done + + docker logs fleet > logs.txt 2>&1 + cat logs.txt + + # The awkward path was found: `cloned` is 1 only if the collector + # reached the checkout through the /host mount, which is the whole + # reason repos.yml carries paths rather than deriving them. + grep -q 'jq_repo_cloned{repo="Jebel-Quant/rhiza"} 1.0' metrics.txt + + # Both entries parsed, one of them with no checkout. Asserted on the + # startup line rather than on a metric: a repo with no checkout + # reaches the board only through the GitHub half, which has no token + # here - so `repo="Jebel-Quant/actions"` is legitimately absent from + # /metrics and asserting on it would be testing GitHub, not this. + grep -q 'fleet: 2 repos, 1 with a checkout' logs.txt + + # A refusal at startup, not a board that is quietly one repo short. This + # is the failure mode the whole config path is shaped to avoid. + - name: A broken repos.yml stops the container run: | - pip install --quiet pyyaml - printf 'repos:\n' > repos.yml - git init -q -b main nested/somewhere/rhiza - git -C nested/somewhere/rhiza remote add origin https://github.com/Jebel-Quant/rhiza.git - printf ' - path: nested/somewhere/rhiza\n' >> repos.yml - # A repo with no checkout: GitHub panels only, no path. - printf ' - repo: Jebel-Quant/actions\n' >> repos.yml - - python3 scripts/gen-repos.py | tee env.out - grep -qx 'JQ_REPOS=Jebel-Quant/rhiza,Jebel-Quant/actions' env.out - grep -q 'JQ_REPO_PATHS=Jebel-Quant/rhiza=.*/nested/somewhere/rhiza$' env.out - test "$(grep -c 'Jebel-Quant/actions=' env.out)" = 0 + printf 'repos:\n - repo: no-slash\n' > broken.yml + docker run --rm -v "$PWD/broken.yml:/config/repos.yml:ro" \ + jq-monitoring:ci 2>&1 | tee out.txt || true + grep -q 'no-slash' out.txt diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml new file mode 100644 index 0000000..ae0cbee --- /dev/null +++ b/.github/workflows/image.yml @@ -0,0 +1,62 @@ +# Publishes the image the README tells people to run. Without this, the +# `docker run ghcr.io/jebel-quant/monitoring` in every doc points at nothing. +name: Image + +on: + push: + branches: [main] + # Prose cannot change the image. CI already builds it on every pull + # request, so this workflow only has to publish what main now holds. + paths-ignore: + - '**.md' + - 'docs/**' + - 'mkdocs.yml' + release: + types: [published] + workflow_dispatch: + +permissions: + contents: read + packages: write + +concurrency: + # A second push while the first is still building would race to the same + # `latest` tag, and the loser would win. + group: image-${{ github.ref }} + cancel-in-progress: true + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + # Apple silicon and CI runners are not the same architecture, and the + # README is written for a laptop. Both or the recipe does not work. + - uses: docker/setup-qemu-action@v3 + - uses: docker/setup-buildx-action@v3 + + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=semver,pattern={{version}} + type=sha,format=short + + - uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore index e6288dd..cb80515 100644 --- a/.gitignore +++ b/.gitignore @@ -2,13 +2,10 @@ # Your fleet: it describes the folder layout of one machine, which has no # business in a public repo. Start from repos.example.yml. Nothing is generated -# from it any more - scripts/collector.sh reads it at every launch. +# from it - the collector reads it at startup, inside the container. repos.yml -# Where the launchd agent's output goes. -.collector-logs/ - -# Left behind by running the collector, which is how it runs now. +# Left behind by running the collector's tests locally. .venv/ .ruff_cache/ .pytest_cache/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b3f4ee7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,76 @@ +# One image, one `docker run`. Prometheus, Grafana and the collector in a single +# container, with the dashboard, the datasource, the alert rules and the scrape +# config baked in - so nothing has to be cloned to run the board. +# +# The three used to be two containers plus a launchd agent on the host, because +# the collector reads your working copies and a bind mount could only name a +# checkout at /repos//. That constraint is gone: the home +# directory is mounted once, whole, at /host, and repos.yml names paths inside +# it. One mount expresses any layout. +# +# docker build -t jq-monitoring . + +FROM prom/prometheus:v2.55.1 AS prometheus +# The Ubuntu variant, not the default Alpine one: the runtime below is Debian, +# and a musl-linked grafana will not start there. +FROM grafana/grafana:11.3.1-ubuntu AS grafana + +FROM python:3.12-slim-bookworm + +# git for the working-copy half, curl for the healthcheck and purge-repo, +# ca-certificates for api.github.com. +RUN apt-get update \ + && apt-get install -y --no-install-recommends git curl ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=prometheus /bin/prometheus /bin/promtool /usr/local/bin/ +COPY --from=grafana /usr/share/grafana /usr/share/grafana +COPY --from=grafana /etc/grafana /etc/grafana + +# The mounted checkouts belong to your host user, not to root inside here, and +# git refuses to read a repository owned by someone else. Nothing here ever +# writes - every call passes --no-optional-locks and the mount is read-only - +# so the ownership check is protecting against nothing we do. +RUN git config --system safe.directory '*' + +# Baked in, so there is no second copy on your disk to edit by mistake: the +# only file you own is repos.yml. +COPY prometheus/prometheus.yml /etc/prometheus/prometheus.yml +COPY grafana/provisioning /etc/grafana/provisioning +COPY grafana/dashboards /etc/grafana/dashboards + +COPY collector /src/collector +RUN pip install --no-cache-dir /src/collector && rm -rf /src + +COPY image/entrypoint.sh /usr/local/bin/entrypoint +COPY image/purge-repo.sh /usr/local/bin/purge-repo +RUN chmod +x /usr/local/bin/entrypoint /usr/local/bin/purge-repo + +ENV GF_PATHS_HOME=/usr/share/grafana \ + GF_PATHS_CONFIG=/etc/grafana/grafana.ini \ + GF_PATHS_PROVISIONING=/etc/grafana/provisioning \ + GF_PATHS_DATA=/data/grafana \ + GF_PATHS_LOGS=/data/grafana/log \ + GF_PATHS_PLUGINS=/data/grafana/plugins \ + GF_AUTH_ANONYMOUS_ENABLED=true \ + GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer \ + GF_USERS_DEFAULT_THEME=dark \ + GF_ANALYTICS_REPORTING_ENABLED=false \ + GF_ANALYTICS_CHECK_FOR_UPDATES=false \ + GF_PLUGINS_PREINSTALL_DISABLED=true +# admin/admin is Grafana's own default and is deliberately not restated here: +# the board opens without signing in and the password is only for settings. +# Override with -e GF_SECURITY_ADMIN_PASSWORD=... if you expose port 3000. +ENV JQ_REPOS_FILE=/config/repos.yml \ + JQ_HOST_ROOT=/host \ + PROM_RETENTION=180d + +# /config is yours (repos.yml, read-only); /host is your home directory, +# read-only; /data is the history and Grafana's own database. +VOLUME /data +EXPOSE 3000 9090 9109 + +HEALTHCHECK --interval=30s --start-period=60s --retries=3 \ + CMD curl -fsS http://localhost:3000/api/health >/dev/null || exit 1 + +ENTRYPOINT ["/usr/local/bin/entrypoint"] diff --git a/README.md b/README.md index ff3373e..20c1fc7 100644 --- a/README.md +++ b/README.md @@ -10,24 +10,12 @@ A Grafana board for the state of your repo fleet — template drift, CI on the default branch, open pull requests, and the working copies on this machine — with Prometheus keeping the history and six alert rules on top. -Everything runs in Docker on `localhost`. Nothing is discovered: a repo is on -the board because you listed it in `repos.yml`, and for no other reason. - -## What you need - -Docker (for Prometheus and Grafana), [`uv`](https://docs.astral.sh/uv/) (for the -collector, which runs on your machine rather than in a container), and the -[`gh` CLI](https://cli.github.com) signed in — `up.sh` mints the token from it, -otherwise put a `GITHUB_TOKEN` in `.env` yourself. +One container, one file. Nothing is discovered: a repo is on the board because +you listed it in `repos.yml`, and for no other reason. ## Recipe -```bash -git clone https://github.com/Jebel-Quant/monitoring.git && cd monitoring -./scripts/up.sh # writes repos.yml + .env from the examples, then stops -``` - -Now **edit `repos.yml`** — one entry per repo you want on the board: +Write a `repos.yml` — one entry per repo you want on the board: ```yaml repos: @@ -38,39 +26,59 @@ repos: `owner/name` comes from each checkout's `origin`, so the path is all you write, and the path is used as written — a checkout does not have to live at -`//`. -Then: +`//`. Then: ```bash -./scripts/up.sh # builds and starts everything +docker run -d --name jq-fleet \ + -p 127.0.0.1:3000:3000 \ + -v "$PWD/repos.yml:/config/repos.yml:ro" \ + -v "$HOME:/host:ro" \ + -v jq-fleet-data:/data \ + -e GITHUB_TOKEN="$(gh auth token)" \ + ghcr.io/jebel-quant/monitoring:latest + open http://localhost:3000/d/jq-fleet ``` -The board fills in within a minute — the local panels first, the GitHub panels -after the first API refresh. `repos.yml` is gitignored: it describes one machine's folders. Nothing is -generated from it — the collector reads it at every launch, so there is no -second file to fall out of step. +The board fills in within a minute. That is the whole install — the dashboard, +the datasource, the alert rules and the scrape config are in the image, so +there is nothing to clone and nothing on your disk but `repos.yml`. + +### The four flags + +| | | +|---|---| +| `-v .../repos.yml:/config/repos.yml:ro` | Required. The fleet — [details](docs/configuration.md) | +| `-v "$HOME:/host:ro"` | Your home directory, read-only, so `~/...` in `repos.yml` resolves. Leave it out and the working-copy panels stay empty; everything GitHub reports still works | +| `-v jq-fleet-data:/data` | Prometheus history and Grafana's database. Leave it out and both start empty at every run | +| `-e GITHUB_TOKEN=...` | Needs `repo` and `read:org`, and must read every repo you listed. Without one GitHub allows 60 calls an hour, which is not a fleet | + +`docker compose up -d` does the same thing with the flags written down; see +[`docker-compose.yml`](docker-compose.yml). ## Then what | | | |---|---| -| Add or drop a repo | edit `repos.yml`, `./scripts/up.sh` again — [details](docs/configuration.md) | -| Erase a dropped repo's history | [`./scripts/purge-repo.sh owner/name`](docs/configuration.md#dropping-a-repo) (irreversible) | -| Stop | `./scripts/down.sh` (add `--volumes` to discard the history too) | -| See the collector's log | `tail -f .collector-logs/collector.log` — it runs on your machine, not in Docker ([why](docs/operations.md#the-collector-runs-on-your-machine)) | -| Edit the board | change `grafana/dashboards/fleet.json`; it reloads in 30s — [read the traps first](docs/dashboard.md#traps-worth-not-re-introducing) | +| Add or drop a repo | edit `repos.yml`, `docker restart jq-fleet` — [details](docs/configuration.md) | +| Erase a dropped repo's history | [`docker exec jq-fleet purge-repo owner/name`](docs/configuration.md#dropping-a-repo) (irreversible) | +| Stop | `docker rm -f jq-fleet` (add `docker volume rm jq-fleet-data` to discard the history too) | +| See what it is doing | `docker logs -f jq-fleet` — all three processes, prefixed ([why one container](docs/operations.md#one-container-three-processes)) | +| Edit the board | change `grafana/dashboards/fleet.json` and rebuild — [read the traps first](docs/dashboard.md#traps-worth-not-re-introducing) | | Get notified | add a contact point under *Alerting → Contact points* — [why it is not provisioned](docs/dashboard.md#alerting) | +| Migrate from the old two-container stack | [carry the Prometheus history over](docs/operations.md#coming-from-the-two-container-stack) | | | | |---|---| | Dashboard | | | Alert rules | | -| Prometheus | | -| Raw metrics | | +| Prometheus | — add `-p 127.0.0.1:9090:9090` | +| Raw metrics | — add `-p 127.0.0.1:9109:9109` | -All three ports bind to `127.0.0.1` only, because anonymous read access is on. -The board opens without signing in; `admin` / `admin` is only for settings. +Publish port 3000 to `127.0.0.1` only, as above, because anonymous read access +is on: the board opens without signing in, and `admin` / `admin` is only for +settings. A `0.0.0.0` binding would serve private repo names and pull request +titles to the whole LAN without a password. ## Docs @@ -78,7 +86,7 @@ Also published as a book: **** | | | |---|---| -| [Configuration](docs/configuration.md) | `repos.yml`, `.env`, the API budget | +| [Configuration](docs/configuration.md) | `repos.yml`, the environment, the API budget | | [What it watches](docs/metrics.md) | the four subjects, the metrics, and why each is shaped that way | | [The dashboard](docs/dashboard.md) | reading it, editing it, alerting, and the query traps | | [Day to day](docs/operations.md) | why panels say *No data*, and what the sign-in button is | diff --git a/collector/jq_collector/config.py b/collector/jq_collector/config.py index ed74d62..ed801b3 100644 --- a/collector/jq_collector/config.py +++ b/collector/jq_collector/config.py @@ -2,9 +2,14 @@ from __future__ import annotations +import logging import os from dataclasses import dataclass, field +from .repos import FleetError, load + +log = logging.getLogger(__name__) + def _int(name: str, default: int) -> int: raw = os.environ.get(name) @@ -27,7 +32,8 @@ def _pairs(name: str) -> dict[str, str]: one repo's working-copy panels off the board and say nothing about why, which is the failure mode this whole module is shaped to avoid; refusing to start is louder and cheaper to diagnose. A path containing a comma cannot - be expressed here - ``scripts/gen-repos.py`` refuses to emit one. + be expressed here at all, which is one more reason repos.yml is the place + the layout is normally written down. """ mapping: dict[str, str] = {} for item in _csv(name): @@ -43,18 +49,21 @@ def _pairs(name: str) -> dict[str, str]: class Config: """Everything the collector needs to know about its environment. - The fleet is an explicit list: ``JQ_REPOS`` names every monitored repo as - ``owner/name``, and nothing else is ever gathered. There used to be a - whole-org sweep as well, which meant the board's contents were decided by - GitHub rather than by you - a new repo in the org appeared unasked, and a - shared org like cvxgrp dragged in a hundred repos that were not yours. - - On a laptop the list is generated from ``repos.yml`` by - ``scripts/gen-repos.py``, which also mounts each checkout at - ``$JQ_REPO_ROOT//``. Setting it by hand works too, and then - nothing mounts the checkouts. Both halves read the same list, so the GitHub - panels and the working-copy panels can never disagree about who is in the - fleet. + The fleet is an explicit list, and nothing else is ever gathered. There + used to be a whole-org sweep as well, which meant the board's contents were + decided by GitHub rather than by you - a new repo in the org appeared + unasked, and a shared org like cvxgrp dragged in a hundred repos that were + not yours. + + ``repos.yml`` is where that list lives, and it is read here directly: both + the fleet and each checkout's path come out of the one file, at startup, + with nothing generated in between to go stale. ``JQ_REPOS`` and + ``JQ_REPO_PATHS`` remain for a deployment that has no file to mount - a + server, or CI - and ``JQ_REPO_PATHS`` still wins over the file, so one + awkward path can be corrected without editing it. + + Both halves read the same list, so the GitHub panels and the working-copy + panels can never disagree about who is in the fleet. """ repos: tuple[str, ...] = field(default_factory=lambda: _csv("JQ_REPOS")) @@ -62,19 +71,25 @@ class Config: token: str = os.environ.get("GITHUB_TOKEN", "") api: str = os.environ.get("GITHUB_API", "https://api.github.com") - # Where the checkouts are, for repos JQ_REPO_PATHS does not name: one per - # repo at //. That is exactly where the generated - # compose override bind-mounts them, so in the container this is all that - # is needed. Set empty to skip local scanning entirely, which is the right - # setting anywhere there are no working copies to report on - the local - # panels then simply have nothing to say. - repo_root: str = os.environ.get("JQ_REPO_ROOT", "/repos") - - # Explicit path per repo, as owner/name=path. Needed whenever a checkout - # does not sit at // - which is normal outside the - # container, where paths are whatever they are on disk rather than whatever - # a bind mount normalised them to. `scripts/gen-repos.py --env` emits this - # from repos.yml, so the layout is stated once and in one place. + # The fleet file. Read when it is there; when it is not, JQ_REPOS and + # JQ_REPO_PATHS are the whole story. /config/repos.yml is where the image + # expects it to be mounted. + repos_file: str = os.environ.get("JQ_REPOS_FILE", "/config/repos.yml") + + # Where the host's home directory is mounted, and therefore what `~` in + # repos.yml means. Set to /host in the image; empty when the collector runs + # natively, where `~` is just `~`. One mount for the whole fleet is what + # lets repos.yml name a checkout at any path at all - the per-repo bind + # mounts it replaced could only express //. + host_root: str = os.environ.get("JQ_HOST_ROOT", "") + + # Last-resort fallback for a repo no path is known for: // + # . Off by default, because repos.yml states every path outright. + # Setting it is for a deployment that checks its fleet out in that shape. + repo_root: str = os.environ.get("JQ_REPO_ROOT", "") + + # Explicit path per repo, as owner/name=path. Filled from repos.yml, and + # overridable from the environment for a deployment that has no file. repo_paths: dict[str, str] = field(default_factory=lambda: _pairs("JQ_REPO_PATHS")) # owner/name of the repo whose releases define "up to date". @@ -122,5 +137,25 @@ class Config: # its name, workflow names, PR titles and branch names are all disclosure. public_only: bool = os.environ.get("JQ_PUBLIC_ONLY", "false").lower() == "true" + def __post_init__(self) -> None: + """Fold repos.yml in, when there is one. + + Refuses to start on a file it cannot act on. The alternative - carrying + on with a short fleet - takes repos off the board and says nothing about + why, and a board that is quietly incomplete is worse than one that did + not come up. + """ + if not self.repos_file or not os.path.isfile(self.repos_file): + return + try: + fleet, paths = load(self.repos_file, self.host_root) + except FleetError as exc: + raise SystemExit(f"repos.yml: {exc}") from exc + object.__setattr__(self, "repos", fleet) + # The environment wins: it is the narrower, more deliberate statement, + # and it is how one path can be corrected without touching the file. + object.__setattr__(self, "repo_paths", {**paths, **self.repo_paths}) + log.info("fleet: %d repos, %d with a checkout", len(fleet), len(self.repo_paths)) + def is_ignored(self, owner: str, name: str) -> bool: return name in self.ignore or f"{owner}/{name}" in self.ignore diff --git a/collector/jq_collector/localgit.py b/collector/jq_collector/localgit.py index 43cd99d..d922de4 100644 --- a/collector/jq_collector/localgit.py +++ b/collector/jq_collector/localgit.py @@ -18,13 +18,13 @@ the upstream that moved. Nothing here searches for checkouts either. A repo is read at the path -``JQ_REPO_PATHS`` gives for it, or failing that at ``//`` -- which is exactly where the generated compose override mounts it, so inside the -container the default is always right. Outside it, paths are whatever they are on -disk: ``repos.yml`` may well say ``~/repos/tschm/rhiza_projects/cs`` for -``tschm/cs``, and no amount of joining owner to name will produce that. Either -way the fleet is decided by ``repos.yml``, not by whatever happened to be lying -around under a scanned directory. +``repos.py`` resolved for it out of ``repos.yml``, and those paths are whatever +they are on disk: ``repos.yml`` may well say ``~/repos/tschm/rhiza_projects/cs`` +for ``tschm/cs``, and no amount of joining owner to name will produce that. The +``//`` fallback is off by default and survives only for +a deployment laid out that way. Either way the fleet is decided by +``repos.yml``, not by whatever happened to be lying around under a scanned +directory. """ from __future__ import annotations diff --git a/collector/jq_collector/repos.py b/collector/jq_collector/repos.py new file mode 100644 index 0000000..3658585 --- /dev/null +++ b/collector/jq_collector/repos.py @@ -0,0 +1,173 @@ +"""The fleet, read from ``repos.yml``. + +One file names every monitored repo and, where there is one, the checkout on +disk. It is read at startup and nothing is generated from it: there is no +second file, and no environment round-trip, that could fall out of step. + +Paths are written the way you would write them on your own machine - ``~/repos/ +jebel-quant/rhiza``. Inside the container that home directory is a single +read-only bind mount, so ``JQ_HOST_ROOT`` (``/host`` in the image, unset when +the collector runs natively) is what ``~`` expands to. One mount covers the +whole fleet, which is what makes a checkout at an arbitrary path expressible +here at all - the old per-repo bind mounts could only name +``//``. + +A path that is not reachable is not fatal. It means the home directory was not +mounted, or that repo is not checked out here: the GitHub panels still report +on it and the working-copy panels have nothing to say. That is the same shape +as an entry with no ``path`` at all. +""" + +from __future__ import annotations + +import logging +import os +import subprocess +from typing import Any + +log = logging.getLogger(__name__) + + +class FleetError(Exception): + """``repos.yml`` says something the collector cannot act on.""" + + +def _origin_owner_name(path: str) -> tuple[str, str] | None: + """The ``(owner, name)`` a checkout's origin points at.""" + try: + url = subprocess.run( + ["git", "--no-optional-locks", "-C", path, "remote", "get-url", "origin"], + capture_output=True, + text=True, + timeout=20, + check=False, + ).stdout.strip() + except (OSError, subprocess.TimeoutExpired): + return None + if not url: + return None + url = url.removesuffix(".git") + if url.startswith("git@"): + tail = url.partition(":")[2] + elif "://" in url: + tail = url.split("://", 1)[1].split("/", 1)[-1] + else: + tail = url + parts = [p for p in tail.split("/") if p] + return (parts[-2], parts[-1]) if len(parts) >= 2 else None + + +def resolve_path(raw: str, host_root: str) -> str: + """Where a ``repos.yml`` path lands on *this* filesystem. + + ``~/x`` and the relative ``x`` both hang off ``host_root`` when it is set, + because that is the mount point of the home directory they were written + against. An absolute path is tried as written first - it is right when the + collector runs natively - and only then under the mount, which is what a + whole-root mount (``-v /:/host:ro``) makes work. + """ + raw = raw.strip() + if not host_root: + return os.path.abspath(os.path.expanduser(raw)) + + if raw.startswith("~"): + under = raw.removeprefix("~").lstrip("/") + elif not os.path.isabs(raw): + under = raw + elif os.path.exists(raw): + return raw + else: + under = raw.lstrip("/") + # normpath so a bare `~` gives /host and not /host/ - a trailing slash is + # harmless to open() but it reaches the dashboard as a repo's `path` label. + return os.path.normpath(os.path.join(host_root, under)) + + +def _is_checkout(path: str) -> bool: + # `.git` is a directory in a plain checkout and a file in a worktree. + return os.path.exists(os.path.join(path, ".git")) + + +def _entry(item: Any, index: int, host_root: str) -> tuple[str, str | None]: + """One entry -> ``(owner/name, checkout path or None)``.""" + # `- ~/repos/foo` is accepted as shorthand for `- path: ~/repos/foo`. + if isinstance(item, str): + item = {"path": item} + if not isinstance(item, dict): + raise FleetError(f"entry {index} is neither a path nor a mapping: {item!r}") + + named = str(item.get("repo") or "").strip() + raw_path = item.get("path") + + if raw_path is None: + # Name the offending value, not just the rule. "entry 7" in a + # twenty-five entry file means counting; the value is searchable. + if not named: + raise FleetError( + f"entry {index} ({item!r}) has neither a `path` nor a `repo: owner/name`" + ) + if "/" not in named: + raise FleetError(f"entry {index}: repo {named!r} is not of the form owner/name") + return named, None + + path = resolve_path(str(raw_path), host_root) + + if not _is_checkout(path): + # Not an error: the home directory may not be mounted, or this repo may + # simply not be checked out here. Either way the fleet keeps the repo + # and only the working-copy panels go quiet - but say so once, because + # a typo in repos.yml looks exactly like this from here. + if named and "/" in named: + log.warning("no checkout for %s at %s - GitHub panels only", named, path) + return named, None + raise FleetError( + f"entry {index}: {raw_path} is not a git checkout (looked in {path}). " + "Mount the home directory it lives under, or give the entry a " + "`repo: owner/name` so it can be monitored without one." + ) + + if "/" in named: + return named, path + + origin = _origin_owner_name(path) + if origin is None: + raise FleetError( + f"entry {index}: {path} has no usable origin remote - add an explicit `repo: owner/name`" + ) + return f"{origin[0]}/{origin[1]}", path + + +def load(source: str, host_root: str = "") -> tuple[tuple[str, ...], dict[str, str]]: + """Read ``repos.yml`` into ``(fleet, checkout paths)``. + + The fleet is every listed repo as ``owner/name``; the paths map holds only + those with a checkout this machine can actually read. + """ + import yaml + + try: + with open(source, encoding="utf-8") as handle: + text = handle.read() + except OSError as exc: + raise FleetError(f"cannot read {source}: {exc}") from exc + + try: + data = yaml.safe_load(text) or {} + except yaml.YAMLError as exc: + raise FleetError(f"{source} is not valid YAML: {exc}") from exc + + entries = data.get("repos") if isinstance(data, dict) else None + if not isinstance(entries, list) or not entries: + raise FleetError(f"{source} lists no repos under a top-level `repos:` key") + + fleet: list[str] = [] + paths: dict[str, str] = {} + for index, item in enumerate(entries, start=1): + full_name, path = _entry(item, index, host_root) + if full_name in fleet: + raise FleetError(f"{full_name} is listed twice") + fleet.append(full_name) + if path is not None: + paths[full_name] = path + + return tuple(fleet), paths diff --git a/collector/tests/test_fleet.py b/collector/tests/test_fleet.py index 0bb64fc..ad8ab3a 100644 --- a/collector/tests/test_fleet.py +++ b/collector/tests/test_fleet.py @@ -13,15 +13,13 @@ from __future__ import annotations import dataclasses -import importlib.util import os import pathlib import subprocess -import sys import pytest -from jq_collector import localgit +from jq_collector import localgit, repos from jq_collector.config import Config REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] @@ -375,108 +373,237 @@ def test_an_unreadable_repo_does_not_lose_the_others(make_client, cfg, caplog): assert "Jebel-Quant/typo" in caplog.text -# -- the generator ----------------------------------------------------------- +# -- reading repos.yml ------------------------------------------------------- +# +# One file, read at startup by the collector itself. It used to be turned into +# two environment lines by a script the launcher ran, which meant three places +# a repo could be lost between the file and the board. -@pytest.fixture -def gen_repos(): - """scripts/gen-repos.py, loaded by path - it is a script, not a package.""" - pytest.importorskip("yaml") - spec = importlib.util.spec_from_file_location( - "gen_repos", REPO_ROOT / "scripts" / "gen-repos.py" - ) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module +def write_fleet(tmp_path: pathlib.Path, body: str) -> str: + source = tmp_path / "repos.yml" + source.write_text(f"repos:\n{body}") + return str(source) -def test_the_generator_names_a_checkout_from_its_origin(gen_repos, tmp_path): +def test_a_checkout_is_named_by_its_origin(tmp_path): path = make_checkout(tmp_path, "cvxgrp", "cvxsimulator") - assert gen_repos.resolve({"path": str(path)}, 1) == ("cvxgrp/cvxsimulator", path) + fleet, paths = repos.load(write_fleet(tmp_path, f" - path: {path}\n")) + + assert fleet == ("cvxgrp/cvxsimulator",) + assert paths == {"cvxgrp/cvxsimulator": str(path)} -def test_an_explicit_repo_overrides_the_origin(gen_repos, tmp_path): +def test_an_explicit_repo_overrides_the_origin(tmp_path): """For a fork you want the board to follow upstream, not your copy.""" path = make_checkout(tmp_path, "me", "cvxpy") + body = f" - path: {path}\n repo: cvxpy/cvxpy\n" - assert gen_repos.resolve({"path": str(path), "repo": "cvxpy/cvxpy"}, 1) == ( - "cvxpy/cvxpy", - path, + assert repos.load(write_fleet(tmp_path, body)) == ( + ("cvxpy/cvxpy",), + {"cvxpy/cvxpy": str(path)}, ) -def test_an_entry_may_name_a_repo_with_no_checkout(gen_repos): - assert gen_repos.resolve({"repo": "Jebel-Quant/actions"}, 1) == ("Jebel-Quant/actions", None) +def test_an_entry_may_name_a_repo_with_no_checkout(tmp_path): + fleet, paths = repos.load(write_fleet(tmp_path, " - repo: Jebel-Quant/actions\n")) + assert fleet == ("Jebel-Quant/actions",) + assert paths == {} -def test_a_bare_string_entry_is_a_path(gen_repos, tmp_path): + +def test_a_bare_string_entry_is_a_path(tmp_path): path = make_checkout(tmp_path, "Jebel-Quant", "rhiza") - assert gen_repos.resolve(str(path), 1) == ("Jebel-Quant/rhiza", path) + assert repos.load(write_fleet(tmp_path, f" - {path}\n"))[0] == ("Jebel-Quant/rhiza",) + + +def test_a_named_repo_whose_checkout_is_missing_keeps_its_github_panels(tmp_path, caplog): + """The supported way to run without mounting a home directory. + + An unreachable path is not an error - the mount may simply not be there - + but it must be said out loud, because a typo in repos.yml looks identical + from in here and would otherwise silently empty half a repo's row. + """ + body = " - path: ~/nowhere/rhiza\n repo: Jebel-Quant/rhiza\n" + + with caplog.at_level("WARNING"): + fleet, paths = repos.load(write_fleet(tmp_path, body)) + + assert fleet == ("Jebel-Quant/rhiza",) + assert paths == {} + assert "no checkout for Jebel-Quant/rhiza" in caplog.text @pytest.mark.parametrize( - "entry", + "body", [ - {"repo": "no-slash"}, # not owner/name, and no path to derive it from - {}, # neither - {"path": "/definitely/not/here"}, + " - repo: no-slash\n", # not owner/name, and no path to derive it from + " - {}\n", # neither + " - path: /definitely/not/here\n", # unreachable, and nothing to fall back on + " - 42\n", # not a path and not a mapping ], ) -def test_a_broken_entry_fails_loudly(gen_repos, entry): - """Better a refusal at generate time than a board that is quietly short a repo.""" - with pytest.raises(SystemExit): - gen_repos.resolve(entry, 1) +def test_a_broken_entry_refuses_to_start(tmp_path, body): + """Better a refusal at startup than a board that is quietly short a repo.""" + with pytest.raises(repos.FleetError): + repos.load(write_fleet(tmp_path, body)) + + +def test_the_refusal_names_the_offending_value(tmp_path): + """`entry 7` in a twenty-five entry file means counting lines. The value is + what you can search the file for.""" + with pytest.raises(repos.FleetError, match="no-slash"): + repos.load(write_fleet(tmp_path, " - repo: no-slash\n")) -def test_a_directory_that_is_not_a_checkout_fails(gen_repos, tmp_path): +def test_a_directory_that_is_not_a_checkout_is_refused(tmp_path): (tmp_path / "empty").mkdir() - with pytest.raises(SystemExit): - gen_repos.resolve({"path": str(tmp_path / "empty")}, 1) + with pytest.raises(repos.FleetError): + repos.load(write_fleet(tmp_path, f" - path: {tmp_path / 'empty'}\n")) -def test_it_prints_the_fleet_and_the_paths_and_writes_nothing( - gen_repos, tmp_path, monkeypatch, capsys -): - """Both lines must come from repos.yml, not from anyone's memory. +def test_the_same_repo_twice_is_refused(tmp_path): + """Two entries, one row on the board: the second would silently win.""" + body = " - repo: a/b\n - repo: a/b\n" + with pytest.raises(repos.FleetError): + repos.load(write_fleet(tmp_path, body)) - Retyping either is how they drift: a repo added here never reaches the - board and nothing reports the difference. The paths line is what lets the - collector find a checkout that does not sit at //, and a - repo with no checkout must not appear in it at all. Nothing is written to - disk - scripts/collector.sh runs this at every launch and exports the - result, so there is no generated file in between to go stale. - """ - path = make_checkout(tmp_path, "Jebel-Quant", "rhiza") + +@pytest.mark.parametrize( + "body", + [" - repo: [\n", ""], # unparseable, and no `repos:` list at all +) +def test_an_unusable_file_refuses_to_start(tmp_path, body): source = tmp_path / "repos.yml" - source.write_text(f"repos:\n - path: {path}\n - repo: cvxgrp/cvxsimulator\n") - monkeypatch.setattr(gen_repos, "SOURCE", source) - monkeypatch.setattr(sys, "argv", ["gen-repos.py"]) + source.write_text(body) + with pytest.raises(repos.FleetError): + repos.load(str(source)) - gen_repos.main() - assert capsys.readouterr().out.strip().splitlines() == [ - "JQ_REPOS=Jebel-Quant/rhiza,cvxgrp/cvxsimulator", - f"JQ_REPO_PATHS=Jebel-Quant/rhiza={path}", - ] - assert list(tmp_path.glob("docker-compose*")) == [] +def test_a_missing_file_refuses_to_start(tmp_path): + with pytest.raises(repos.FleetError): + repos.load(str(tmp_path / "absent.yml")) -def test_it_refuses_a_path_it_cannot_express(gen_repos, tmp_path, monkeypatch, capsys): - """A comma is the separator, so a path holding one would read as two repos.""" - path = make_checkout(tmp_path, "Jebel-Quant", "rhi,za") - source = tmp_path / "repos.yml" - source.write_text(f"repos:\n - path: {path}\n") - monkeypatch.setattr(gen_repos, "SOURCE", source) - monkeypatch.setattr(sys, "argv", ["gen-repos.py"]) +# -- host paths -------------------------------------------------------------- +# +# In the container the home directory is one read-only mount, and repos.yml is +# written against the host's view of it. This is the whole translation, and it +# is the reason a checkout at any path can be named at all - the per-repo bind +# mounts it replaced could only express //. - with pytest.raises(SystemExit): - gen_repos.main() - assert "contains a comma" in capsys.readouterr().err + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("~/repos/rhiza", "/host/repos/rhiza"), + ("~", "/host"), + ("repos/rhiza", "/host/repos/rhiza"), # relative: also relative to home + ("/nowhere/rhiza", "/host/nowhere/rhiza"), # absolute, and not on this fs + ], +) +def test_a_path_is_read_through_the_host_mount(raw, expected): + assert repos.resolve_path(raw, "/host") == expected + + +def test_an_absolute_path_that_exists_is_taken_as_written(tmp_path): + """The collector running natively, where there is no mount to look under.""" + assert repos.resolve_path(str(tmp_path), "/host") == str(tmp_path) + + +def test_without_a_mount_a_path_is_just_a_path(monkeypatch, tmp_path): + monkeypatch.setenv("HOME", str(tmp_path)) + + assert repos.resolve_path("~/repos/rhiza", "") == str(tmp_path / "repos" / "rhiza") + + +# -- and how Config folds it in ---------------------------------------------- -def test_an_unknown_flag_is_refused(gen_repos, monkeypatch): - monkeypatch.setattr(sys, "argv", ["gen-repos.py", "--all"]) +def test_config_reads_the_fleet_out_of_the_file(tmp_path, monkeypatch): + path = make_checkout(tmp_path, "Jebel-Quant", "rhiza") + source = write_fleet(tmp_path, f" - path: {path}\n - repo: cvxgrp/cvxsimulator\n") + monkeypatch.setenv("JQ_REPOS_FILE", source) + + cfg = Config(repos_file=source) + + assert cfg.repos == ("Jebel-Quant/rhiza", "cvxgrp/cvxsimulator") + assert cfg.repo_paths == {"Jebel-Quant/rhiza": str(path)} + + +def test_an_explicit_path_in_the_environment_beats_the_file(tmp_path): + """The escape hatch: one awkward path corrected without editing the file.""" + path = make_checkout(tmp_path, "Jebel-Quant", "rhiza") + source = write_fleet(tmp_path, f" - path: {path}\n") + + cfg = Config(repos_file=source, repo_paths={"Jebel-Quant/rhiza": "/elsewhere"}) + + assert cfg.repo_paths == {"Jebel-Quant/rhiza": "/elsewhere"} + + +def test_no_file_leaves_the_environment_in_charge(tmp_path, monkeypatch): + """A deployment with nothing to mount - a server, or CI.""" + monkeypatch.setenv("JQ_REPOS", "a/b,c/d") + + cfg = Config(repos_file=str(tmp_path / "absent.yml")) + + assert cfg.repos == ("a/b", "c/d") + + +def test_a_broken_file_stops_the_collector(tmp_path): + """SystemExit, not a short fleet: a board that came up missing repos and + said nothing is worse than one that did not come up.""" + source = write_fleet(tmp_path, " - repo: no-slash\n") + with pytest.raises(SystemExit): - gen_repos.main() + Config(repos_file=source) + + +# -- who a checkout says it is ----------------------------------------------- +# +# The origin URL is the only thing that can name a checkout, and git writes it +# in several shapes. Getting one wrong puts a repo on the board under the wrong +# name, or drops it - neither of which the URL itself would ever hint at. + + +@pytest.mark.parametrize( + "origin", + [ + "git@github.com:Jebel-Quant/rhiza.git", + "https://github.com/Jebel-Quant/rhiza.git", + "https://github.com/Jebel-Quant/rhiza", + "ssh://git@github.com/Jebel-Quant/rhiza.git", + "/srv/mirrors/Jebel-Quant/rhiza", # a local clone of a local clone + ], +) +def test_every_shape_of_origin_url_names_the_same_repo(tmp_path, origin): + path = make_checkout(tmp_path, "somewhere", "else", origin=origin) + + assert repos.load(write_fleet(tmp_path, f" - path: {path}\n"))[0] == ("Jebel-Quant/rhiza",) + + +@pytest.mark.parametrize("origin", ["rhiza", ""]) +def test_an_origin_that_names_no_owner_is_refused(tmp_path, origin): + """No owner means no `owner/name`, and guessing one would file the repo + under a name that does not exist on GitHub.""" + path = make_checkout(tmp_path, "somewhere", "else") + if origin: + subprocess.run(["git", "-C", str(path), "remote", "set-url", "origin", origin], check=True) + else: + subprocess.run(["git", "-C", str(path), "remote", "remove", "origin"], check=True) + + with pytest.raises(repos.FleetError, match="origin"): + repos.load(write_fleet(tmp_path, f" - path: {path}\n")) + + +def test_git_being_unrunnable_is_refused_not_guessed_at(tmp_path, monkeypatch): + """Same answer as a missing remote: the collector will not invent a name.""" + path = make_checkout(tmp_path, "Jebel-Quant", "rhiza") + monkeypatch.setattr( + repos.subprocess, "run", lambda *_a, **_k: (_ for _ in ()).throw(OSError("no git")) + ) + + with pytest.raises(repos.FleetError, match="origin"): + repos.load(write_fleet(tmp_path, f" - path: {path}\n")) diff --git a/docker-compose.admin.yml b/docker-compose.admin.yml deleted file mode 100644 index 62c9d9b..0000000 --- a/docker-compose.admin.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Temporary override that enables Prometheus's admin API, which can delete -# series. Applied only for the duration of scripts/purge-repo.sh and reverted -# immediately after. -# -# It is NOT left on: Grafana's datasource proxy lets any viewer - and anonymous -# access is enabled here - reach arbitrary paths on the datasource, so a -# permanently enabled admin API would put "delete every metric" one request -# away from a read-only visitor. -# -# compose replaces `command` wholesale rather than merging, so the base flags -# are repeated here. Keep them in step with docker-compose.yml. -services: - prometheus: - command: - - --config.file=/etc/prometheus/prometheus.yml - - --storage.tsdb.path=/prometheus - - --storage.tsdb.retention.time=${PROM_RETENTION:-180d} - - --web.enable-lifecycle - - --web.enable-admin-api diff --git a/docker-compose.yml b/docker-compose.yml index ae1e8e1..43f01cc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,59 +1,46 @@ -# Prometheus and Grafana. The collector is NOT here: it runs on the host, via -# scripts/collector.sh, because it reads your working copies and a container can -# only see them through bind mounts - which cannot name a checkout that sits -# somewhere other than //, and which are slow enough on -# macOS to have needed a caching layer to work around. +# Optional. `docker run` is the documented way to start the board - see the +# README - and this file is the same thing with the flags written down, for +# when you would rather not retype them. # -# ./scripts/up.sh starts both halves -# docker compose up -d starts only this half +# docker compose up -d --build +# +# One service: Prometheus, Grafana and the collector share a container. See the +# Dockerfile for why they are not three. name: jq-monitoring services: - prometheus: - image: prom/prometheus:v2.55.1 - container_name: jq-prometheus - restart: unless-stopped - command: - - --config.file=/etc/prometheus/prometheus.yml - - --storage.tsdb.path=/prometheus - - --storage.tsdb.retention.time=${PROM_RETENTION:-180d} - - --web.enable-lifecycle - volumes: - - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro - - prometheus-data:/prometheus - ports: - - "127.0.0.1:9090:9090" - # Docker Desktop resolves host.docker.internal on its own; plain Docker on - # Linux does not, and without this the collector target is simply down. - extra_hosts: - - "host.docker.internal:host-gateway" - - grafana: - image: grafana/grafana:11.3.1 - container_name: jq-grafana + fleet: + build: . + image: ghcr.io/jebel-quant/monitoring:latest + container_name: jq-fleet restart: unless-stopped environment: + GITHUB_TOKEN: ${GITHUB_TOKEN:?set GITHUB_TOKEN, e.g. export GITHUB_TOKEN=$(gh auth token)} + JQ_TEMPLATE_REPO: ${JQ_TEMPLATE_REPO:-Jebel-Quant/rhiza} + JQ_GITHUB_INTERVAL: ${JQ_GITHUB_INTERVAL:-600} + JQ_LOCAL_INTERVAL: ${JQ_LOCAL_INTERVAL:-60} + JQ_IGNORE: ${JQ_IGNORE:-} + JQ_PUBLIC_ONLY: ${JQ_PUBLIC_ONLY:-false} + PROM_RETENTION: ${PROM_RETENTION:-180d} GF_SECURITY_ADMIN_USER: ${GF_ADMIN_USER:-admin} GF_SECURITY_ADMIN_PASSWORD: ${GF_ADMIN_PASSWORD:-admin} - # A single-user board on localhost; skip the login wall but keep the - # anonymous role read-only so a stray click cannot edit provisioning. - GF_AUTH_ANONYMOUS_ENABLED: "true" - GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer - GF_USERS_DEFAULT_THEME: dark - GF_FEATURE_TOGGLES_ENABLE: "" volumes: - - ./grafana/provisioning:/etc/grafana/provisioning:ro - - ./grafana/dashboards:/var/lib/grafana/dashboards:ro - - grafana-data:/var/lib/grafana + - ./repos.yml:/config/repos.yml:ro + # Your home directory, whole and read-only. One mount, because repos.yml + # names checkouts at whatever paths they actually have - see the + # Dockerfile. Drop this line and the working-copy panels stay empty; + # everything the GitHub half reports still works. + - ${HOME}:/host:ro + - fleet-data:/data ports: - # Loopback only: anonymous access is enabled, so a 0.0.0.0 binding would + # Loopback only: anonymous read access is on, so a 0.0.0.0 binding would # serve private repo names and PR titles to the whole LAN without a # password. Change to "3000:3000" only if you deliberately want that. - "127.0.0.1:3000:3000" - depends_on: - - prometheus + # Prometheus and the raw metrics, for when you are debugging a panel. + - "127.0.0.1:9090:9090" + - "127.0.0.1:9109:9109" volumes: - prometheus-data: - grafana-data: + fleet-data: diff --git a/docs/configuration.md b/docs/configuration.md index 62941fd..ee68d30 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,13 +1,13 @@ --- title: Configuration -description: repos.yml and .env, dropping and purging a repo, and the GitHub API budget. -keywords: repos.yml, dotenv, github token, prometheus retention, api rate limit, purge series +description: repos.yml and the environment, dropping and purging a repo, and the GitHub API budget. +keywords: repos.yml, docker run, github token, prometheus retention, api rate limit, purge series --- # Configuration -**Which repos** is `repos.yml`. **Everything else** is `.env` (see -`.env.example`). +**Which repos** is `repos.yml`, the one file you own. **Everything else** is +environment variables on the `docker run`. ## The fleet is an explicit list @@ -23,10 +23,9 @@ repos: `owner/name` is read from each checkout's `origin` remote, so the path is all you write. Nothing is discovered: a repo is on the board because it is in this -file, and for no other reason. `scripts/up.sh` turns the file into -`docker-compose.repos.yml`, which mounts each checkout **read-only** at -`/repos//` — so an unlisted repo is not merely filtered out, it is -never visible to the container at all. +file, and for no other reason. The collector reads the file itself, at startup, +from `/config/repos.yml` — nothing is generated from it, so there is no second +file to fall out of step. This replaced a whole-org GitHub sweep plus a directory walk under one mounted root. Both decided membership on their own: a new repo in the org arrived @@ -34,9 +33,9 @@ unasked, a shared org like cvxgrp dragged in 100+ repos that were not yours, and any checkout that happened to sit under the root joined the board because its origin looked right. -Edit `repos.yml`, run `./scripts/up.sh` again, and the fleet is whatever you -just wrote. Both halves of the collector read the same list, so the GitHub -panels and the working-copy panels can never disagree about who is in scope. +Edit `repos.yml`, `docker restart jq-fleet`, and the fleet is whatever you just +wrote. Both halves of the collector read the same list, so the GitHub panels +and the working-copy panels can never disagree about who is in scope. **Archived repos are never monitored.** They are dropped from the GitHub half *and* their local checkouts are skipped, so a checkout left on disk cannot keep a @@ -50,15 +49,17 @@ Dropping a repo stops new samples but leaves its **history**, so it still appears in time windows that reach back before the change. To erase that too: ```bash -./scripts/purge-repo.sh Jebel-Quant/rhiza-brainbug # irreversible +docker exec jq-fleet purge-repo Jebel-Quant/rhiza-brainbug # irreversible ``` -The script enables Prometheus's admin API, deletes the series, cleans -tombstones, and turns the admin API straight back off. It is not left enabled: -Grafana's datasource proxy lets any viewer — and anonymous access is on — reach -arbitrary paths on the datasource, which would put "delete every metric" one -request away from a read-only visitor. Both label generations are purged, since -repos predating the `owner/name` rename have series under a bare name too. +This needs Prometheus's admin API, which is off unless the container was started +with `-e JQ_PROM_ADMIN_API=true` — so a purge is: recreate the container with +that flag, purge, recreate it without. It is deliberately not left on: Grafana's +datasource proxy lets any viewer — and anonymous access is on — reach arbitrary +paths on the datasource, which would put "delete every metric" one request away +from a read-only visitor. `purge-repo` says exactly this if you run it with the +API disabled. Both label generations are purged, since repos predating the +`owner/name` rename have series under a bare name too. Right after a purge, `/api/v1/series` may still list the series while queries return nothing: that is stale head-block index metadata, cleared at the next @@ -71,62 +72,71 @@ alarming. | Key | | | |---|---|---| -| `path` | | A checkout on this machine. `~` and paths relative to the repo both work. | +| `path` | | A checkout on this machine, written as you would write it yourself. `~` is your home directory — which the container sees as the single `-v "$HOME:/host:ro"` mount — and a relative path is relative to it too. | | `repo` | | `owner/name`. Optional next to a `path` — it overrides the origin, which is what you want for a fork whose board should follow upstream. On its own it monitors a repo you have not cloned: GitHub panels are gathered, the working-copy panels stay empty for that row. | A bare string is shorthand for `path`. Duplicate entries, a path that is not a -checkout, and an entry with neither key are all refused at generate time — +checkout, and an entry with neither key all stop the collector at startup — better a refusal than a board that is quietly one repo short. -## .env +The one thing that is *not* fatal is a `path` that cannot be reached alongside +an explicit `repo:`. That is what running without the `$HOME` mount looks like, +and it is a supported way to use this: the repo keeps its GitHub panels and the +working-copy panels stay empty. It is logged as a warning, because a typo looks +identical from inside the container. + +## Environment + +Passed as `-e NAME=value` on the `docker run`, or in `.env` if you use +`docker compose`. | Variable | Default | | |---|---|---| -| `GITHUB_TOKEN` | — | Must be able to read every repo in `repos.yml`. `up.sh` mints one from `gh auth token`. | +| `GITHUB_TOKEN` | — | Must be able to read every repo in `repos.yml`. `gh auth token` prints a usable one. | | `JQ_TEMPLATE_REPO` | `Jebel-Quant/rhiza` | Whose releases define "up to date", as `owner/name`. | | `JQ_IGNORE` | — | Repos to drop without editing `repos.yml`, as bare names or `owner/name`. Applies to both halves. | | `JQ_INCLUDE_ARCHIVED` | `false` | Archived repos are dropped from both halves. | | `JQ_PUBLIC_ONLY` | `false` | Drop private repos entirely — not just their details, their existence. | -| `JQ_REPO_PATHS` | — | Where each checkout really is, as `owner/name=path` pairs. `scripts/collector.sh` sets it from `repos.yml` — see [Checkout paths](#checkout-paths). | +| `JQ_REPO_PATHS` | — | Where each checkout really is, as `owner/name=path` pairs. Filled from `repos.yml`; set it to override one entry — see [Checkout paths](#checkout-paths). | +| `JQ_HOST_ROOT` | `/host` | Where your home directory is mounted, and therefore what `~` in `repos.yml` means. | +| `JQ_PROM_ADMIN_API` | `false` | Enables the API [`purge-repo`](#dropping-a-repo) needs. Leave it off. | | `JQ_GITHUB_INTERVAL` | `300` | Seconds between GitHub refreshes. | | `JQ_MEASURE_MAX_AGE` | `86400` | Seconds an unchanged line/commit count may stand before it is retaken. See [Size and cadence](dashboard.md#size-and-cadence). | | `PROM_RETENTION` | `180d` | How much history to keep. | -`scripts/gen-repos.py` turns `repos.yml` into the `JQ_REPOS` list the collector -actually reads, so both halves see the same fleet. Setting `JQ_REPOS` by hand -works too and skips `repos.yml` entirely — but then no checkout paths are set -either, so only the GitHub panels have anything to say. +`repos.yml` is read at startup and turned into the fleet the collector actually +uses, so both halves see the same list. Setting `JQ_REPOS` by hand works too and +is what a deployment with no file to mount — a server, or CI — does instead. ## Checkout paths -The collector reads a repo at the path `JQ_REPO_PATHS` gives for it, and failing -that at `//`. - -The collector runs on your machine, so paths are whatever they are on disk and -the fallback is often wrong. An entry like +The collector reads a repo at the path `repos.yml` gave for it, and nowhere +else. Paths are whatever they are on disk, so an entry like ```yaml - path: ~/repos/tschm/rhiza_projects/cs # this is tschm/cs ``` -cannot be recovered by joining owner to name, so that repo drops off the -working-copy panels while staying on the GitHub ones — present on the board, and -quietly missing half its columns. Four repos in the fleet this was built against -are laid out that way. +means exactly that. Joining owner to name would produce `tschm/cs` and find +nothing, and that repo would drop off the working-copy panels while staying on +the GitHub ones — present on the board, and quietly missing half its columns. +Four repos in the fleet this was built against are laid out that way, which is +the whole reason `repos.yml` carries paths rather than deriving them. -You do not normally set this yourself: `scripts/collector.sh` runs -`scripts/gen-repos.py` at every launch and exports both lines, so `repos.yml` -stays the only place the fleet and the layout are written down. To see what it -resolves to: +Inside the container `~` is `$JQ_HOST_ROOT`, the mount point of your home +directory. That single mount is what makes an arbitrary layout expressible at +all: the per-repo bind mounts it replaced could only ever name +`//`. To see what a path resolved to, read `jq_local_*` +series' `path` label at , or the startup line: -```bash -uv run --with pyyaml python scripts/gen-repos.py +``` +collector | INFO jq_collector.config fleet: 25 repos, 24 with a checkout ``` -A path containing a comma cannot be expressed — comma is the separator — and -`gen-repos.py` refuses to emit one rather than produce a line that would be -misread as two repos. A malformed pair stops the collector at startup instead of -silently shrinking the board. +`JQ_REPO_PATHS` overrides the file, one repo at a time, for the case where a +path is right everywhere except in the container. A path containing a comma +cannot be expressed there — comma is the separator — which is one more reason +`repos.yml` is where the layout is normally written down. ## API budget diff --git a/docs/index.md b/docs/index.md index 073caf6..c365294 100644 --- a/docs/index.md +++ b/docs/index.md @@ -10,22 +10,17 @@ A Grafana board for the state of your repo fleet — template drift, CI on the default branch, open pull requests, and the working copies on your machine — with Prometheus keeping the history and six alert rules on top. -Everything runs in Docker on `localhost`. **Nothing is discovered:** a repo is -on the board because you listed it in `repos.yml`, and for no other reason. +One container, one file. **Nothing is discovered:** a repo is on the board +because you listed it in `repos.yml`, and for no other reason. ## What you need -Docker, and the [`gh` CLI](https://cli.github.com) signed in — `up.sh` mints the -token from it. Otherwise put a `GITHUB_TOKEN` in `.env` yourself. +Docker, and the [`gh` CLI](https://cli.github.com) signed in — or any GitHub +token that can read the repos you list. ## The recipe -```bash -git clone https://github.com/Jebel-Quant/monitoring.git && cd monitoring -./scripts/up.sh # writes repos.yml + .env from the examples, then stops -``` - -Now edit `repos.yml` — one entry per repo you want on the board: +Write a `repos.yml` — one entry per repo you want on the board: ```yaml repos: @@ -38,19 +33,39 @@ repos: Then: ```bash -./scripts/up.sh # builds and starts everything +docker run -d --name jq-fleet \ + -p 127.0.0.1:3000:3000 \ + -v "$PWD/repos.yml:/config/repos.yml:ro" \ + -v "$HOME:/host:ro" \ + -v jq-fleet-data:/data \ + -e GITHUB_TOKEN="$(gh auth token)" \ + ghcr.io/jebel-quant/monitoring:latest + open http://localhost:3000/d/jq-fleet ``` -The board fills in within a minute — the local panels first, the GitHub panels -after the first API refresh. +That is the whole install. The dashboard, the datasource, the alert rules and +the scrape config are baked into the image, so there is nothing to clone and +nothing on your disk but `repos.yml`. + +The board fills in within a minute or two — the GitHub panels first, the +working-copy panels once the first scan of the mount completes. + +### The four flags + +| | | +|---|---| +| `-v .../repos.yml:/config/repos.yml:ro` | Required. The fleet — see [Configuration](configuration.md). | +| `-v "$HOME:/host:ro"` | Your home directory, read-only, so `~/...` in `repos.yml` resolves. Leave it out and the working-copy panels stay empty; everything the GitHub half reports still works. | +| `-v jq-fleet-data:/data` | Prometheus history and Grafana's database. Leave it out and both start empty at every run. | +| `-e GITHUB_TOKEN=...` | Needs `repo` and `read:org`. Without one GitHub allows 60 calls an hour, which is not a fleet. | -!!! warning "All three ports bind to `127.0.0.1` only" +!!! warning "Publish the ports to `127.0.0.1` only" Anonymous read access is on, so the board opens without signing in and - `admin` / `admin` is only for settings. Published on `0.0.0.0` the stack - would serve private repo names, PR titles and local branch names to anyone - on the network with no password. This stack is built to run on one + `admin` / `admin` is only for settings. Published on `0.0.0.0` the + container would serve private repo names, PR titles and local branch names + to anyone on the network with no password. This is built to run on one machine; putting it on the internet is not a supported path. ## Where to go next @@ -59,7 +74,8 @@ after the first API refresh. - :material-tune: **[Configuration](configuration.md)** - `repos.yml` and `.env`, dropping and purging a repo, and the API budget. + `repos.yml` and the environment, dropping and purging a repo, and the + API budget. - :material-eye-outline: **[What it watches](metrics.md)** diff --git a/docs/operations.md b/docs/operations.md index 02e4334..200e413 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -6,55 +6,111 @@ keywords: grafana login, no data, prometheus staleness, macos sleep, caffeinate # Running it day to day -## The collector runs on your machine +## One container, three processes -Two halves, not three containers. Prometheus and Grafana are in Docker; -**the collector is an ordinary process on your Mac**, started by `up.sh` as a -launchd agent and stopped by `down.sh`. - -It is not containerised because it reads your working copies, and a container -can only reach those through bind mounts. Those mounts have to place every -checkout at `//`, which silently loses any repo that lives -somewhere else, and reading thousands of small files back through them on macOS -is slow enough that the line counts needed a cache to stay affordable. On the -host both problems disappear. +Prometheus, Grafana and the collector share a container, and `docker logs -f +jq-fleet` shows all three with a prefix telling you which is talking. | | | |---|---| -| Its log | `.collector-logs/collector.log` | -| Restart it | `launchctl kickstart -k gui/$UID/com.jebel-quant.jq-collector` | -| Run it in the foreground instead | `./scripts/collector.sh` (Ctrl-C stops it) | -| Is Prometheus reaching it? | — the `jq-collector` job | +| What it is doing | `docker logs -f jq-fleet` | +| Restart it | `docker restart jq-fleet` — also how you pick up an edited `repos.yml` | +| Is Prometheus reaching the collector? | — the `jq-collector` job (needs `-p 127.0.0.1:9090:9090`) | +| Raw metrics | (needs `-p 127.0.0.1:9109:9109`) | + +Nothing here restarts a dead process. The three are one board — a dead +collector means empty panels, a dead Prometheus means no history — so any of +them exiting takes the container down and Docker's own `--restart` policy +handles it. That keeps `docker ps` honest about whether the board is up, which +a supervisor quietly restarting one process inside a still-healthy container +would not. + +### How the working copies get in + +They used to keep the collector out of Docker entirely. A bind mount had to +place every checkout at `/repos//`, which silently lost any repo +living somewhere else — and four repos in the fleet this was built against are +laid out that way. So the collector ran on the host as a launchd agent, and +Prometheus scraped it at `host.docker.internal:9109`. + +The fix was to stop mounting repos one at a time. `-v "$HOME:/host:ro"` mounts +the home directory once, whole, and `~/...` in `repos.yml` is read relative to +that mount — so `~/repos/tschm/rhiza_projects/cs` means exactly what it says. +One mount expresses any layout, and the collector came back inside. + +**The trade.** The collector can now see everything under your home directory, +not only the checkouts you listed — the mount is the same width either way. +It still never writes: every git call is read-only and passes +`--no-optional-locks`, and the mount is `:ro` so the kernel enforces it too. +Leave the mount off entirely and the GitHub half still works; the working-copy +panels simply stay empty. + +**The first scan is slow.** Reading through a Docker Desktop bind mount is cold +the first time — on a 24-repo fleet the opening pass took about two minutes, +and about three milliseconds per git call once the mount was warm. This is also +why line counts are cached against a fingerprint of the clone rather than +retaken every minute (see [Size and cadence](dashboard.md#size-and-cadence)). + +## Coming from the two-container stack + +The board used to be `jq-prometheus` and `jq-grafana` plus a launchd agent, with +history in the `jq-monitoring_prometheus-data` and `jq-monitoring_grafana-data` +volumes. Both carry over — Prometheus's `instance` label is pinned to +`jq-collector` by a relabel rule (see `prometheus/prometheus.yml`) precisely so +that moving the collector does not fork every series in two. + +Stop the old stack **cleanly** first. `docker stop` sends SIGTERM and Prometheus +flushes its WAL on it; killing it instead loses whatever had not been compacted +into a block yet. -Prometheus scrapes it at `host.docker.internal:9109`, which is how a container -reaches the machine it runs on. +```bash +launchctl bootout "gui/$UID/com.jebel-quant.jq-collector" # macOS +docker stop jq-prometheus jq-grafana + +docker volume create jq-fleet-data +docker run --rm \ + -v jq-monitoring_prometheus-data:/old-prom:ro \ + -v jq-monitoring_grafana-data:/old-graf:ro \ + -v jq-fleet-data:/data \ + alpine sh -euc ' + mkdir -p /data/prometheus /data/grafana + cd /old-prom; for f in *; do case "$f" in lock|queries.active) continue;; esac + cp -a "$f" /data/prometheus/; done + cd /old-graf; for f in *; do case "$f" in dashboards) continue;; esac + cp -a "$f" /data/grafana/; done' +``` -**The trade.** Inside the container the collector could only see the checkouts -mounted into it, so an unlisted repo was not merely filtered out — it was -invisible. It now runs as you and could read anything you can. It still never -writes: every git call is read-only and passes `--no-optional-locks`. But that -is now a property of the code rather than something the sandbox enforces. +Two paths change: the TSDB moves from `/prometheus` to `/data/prometheus` and +Grafana's database from `/var/lib/grafana` to `/data/grafana`, which is why the +copy is into subdirectories rather than into the volume root. `lock` and +`queries.active` are runtime files Prometheus rebuilds, and `dashboards/` is an +empty leftover from the old compose file bind-mounting over that path — the +dashboards are in the image now. -**If it will not start**, the usual cause is `PATH`. A launchd agent inherits -`/usr/bin:/bin:/usr/sbin:/sbin` and nothing else, so a `uv` under -`/opt/homebrew` or `~/.local` is invisible to it. `up.sh` pins `uv`'s directory -into the plist when it installs the agent, so re-running `./scripts/up.sh` -after moving or reinstalling `uv` is the fix. +Then start the container [as in the recipe](index.md#the-recipe) with +`-v jq-fleet-data:/data`. This is a copy, so the old volumes are untouched and +rolling back is `docker start jq-prometheus jq-grafana`. Once you are satisfied: +```bash +docker rm jq-prometheus jq-grafana +docker volume rm jq-monitoring_prometheus-data jq-monitoring_grafana-data +rm -rf .collector-logs # the launchd agent's log; nothing writes here now +``` ## The "Sign in" button -Grafana's own local login, against a SQLite file in the `grafana-data` volume on -this machine. There is one account, `admin` / `admin` (override in `.env`). No +Grafana's own local login, against a SQLite file under `/data` on this +machine. There is one account, `admin` / `admin` (override with +`-e GF_SECURITY_ADMIN_PASSWORD=...`). No Grafana Cloud, no external account, nothing leaves the box. Anonymous access is enabled with the `Viewer` role, so the board opens without signing in; the dashboard is provisioned and read-only anyway, so you would only sign in to add a contact point or poke at settings. -Because anonymous access is on, **all three ports bind to `127.0.0.1` only**. -Published on `0.0.0.0` they would serve private repo names, PR titles and local -branch names to anyone on the same network with no password. If you genuinely -want that, drop the `127.0.0.1:` prefix in `docker-compose.yml`. +Because anonymous access is on, **publish the ports to `127.0.0.1` only** — +`-p 127.0.0.1:3000:3000`, as every example here does. A bare `-p 3000:3000` +would serve private repo names, PR titles and local branch names to anyone on +the same network with no password. ## "No data" usually means the Mac was asleep @@ -78,6 +134,7 @@ was green during hours nobody observed. That is the same failure as the papering over them, keep the machine awake while the stack matters: ```bash -caffeinate -s docker compose up -d # or Energy Saver -> prevent sleeping +caffeinate -s -w "$(docker inspect -f '{{.State.Pid}}' jq-fleet)" # or + # Energy Saver -> prevent sleeping ``` diff --git a/grafana/provisioning/dashboards/dashboards.yml b/grafana/provisioning/dashboards/dashboards.yml index d35d78b..f1f68b4 100644 --- a/grafana/provisioning/dashboards/dashboards.yml +++ b/grafana/provisioning/dashboards/dashboards.yml @@ -11,5 +11,8 @@ providers: allowUiUpdates: false updateIntervalSeconds: 30 options: - path: /var/lib/grafana/dashboards + # Beside the rest of the provisioning, not under /var/lib/grafana: that + # is GF_PATHS_DATA, which the image points at the /data volume, and a + # dashboard shipped in the image must not depend on a mount to exist. + path: /etc/grafana/dashboards foldersFromFilesStructure: false diff --git a/grafana/provisioning/datasources/prometheus.yml b/grafana/provisioning/datasources/prometheus.yml index 86e8f1d..a06f3cb 100644 --- a/grafana/provisioning/datasources/prometheus.yml +++ b/grafana/provisioning/datasources/prometheus.yml @@ -5,7 +5,9 @@ datasources: uid: jq-prometheus type: prometheus access: proxy - url: http://prometheus:9090 + # Same container as Grafana; this was http://prometheus:9090 when + # Prometheus was a service of its own. + url: http://localhost:9090 isDefault: true jsonData: timeInterval: 30s diff --git a/image/entrypoint.sh b/image/entrypoint.sh new file mode 100755 index 0000000..9efd764 --- /dev/null +++ b/image/entrypoint.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Supervise the three processes that make up the board. +# +# Not an init system, on purpose: there is nothing here to restart. The three +# are one board - a dead collector means empty panels, a dead Prometheus means +# no history, a dead Grafana means no board at all - so any of them exiting +# takes the container down and Docker's own restart policy handles it. That +# keeps `docker ps` honest about whether the board is up, which a supervisor +# quietly restarting one process in a still-"healthy" container would not. +set -uo pipefail + +die() { printf '\033[31mfleet:\033[0m %s\n' "$1" >&2; exit 1; } +say() { printf '\033[36mfleet:\033[0m %s\n' "$1"; } + +# -- what you have to have provided ------------------------------------------ +if [[ ! -f "${JQ_REPOS_FILE:-/config/repos.yml}" ]]; then + die "no repos.yml at ${JQ_REPOS_FILE:-/config/repos.yml} - mount one: + -v \"\$PWD/repos.yml:/config/repos.yml:ro\" + It lists the repos to monitor; nothing is discovered." +fi +if [[ -z "${GITHUB_TOKEN:-}" ]]; then + say "no GITHUB_TOKEN - GitHub allows 60 unauthenticated calls an hour, which + is not enough for a fleet. Pass -e GITHUB_TOKEN=\"\$(gh auth token)\"." +fi +# Not fatal: a board with no working copies mounted is a supported way to run +# this. Saying so once beats a user wondering why half the panels are empty. +if [[ ! -d "${JQ_HOST_ROOT:-/host}" ]]; then + say "${JQ_HOST_ROOT:-/host} is not mounted - the working-copy panels will stay + empty. Add -v \"\$HOME:/host:ro\" to fill them in." +fi + +mkdir -p /data/prometheus /data/grafana + +# -- shut down together ------------------------------------------------------ +pids=() +stop() { + trap - TERM INT + # Signal the group rather than each pid: promtool-style children and + # Grafana's own subprocesses are otherwise left behind holding /data. + kill -TERM "${pids[@]}" 2>/dev/null + wait + exit 0 +} +trap stop TERM INT + +# -- prometheus -------------------------------------------------------------- +prom_args=( + --config.file=/etc/prometheus/prometheus.yml + --storage.tsdb.path=/data/prometheus + --storage.tsdb.retention.time="${PROM_RETENTION:-180d}" + --web.enable-lifecycle +) +# Deleting series is irreversible and the API needs no authentication, so it is +# off unless you asked for it - `purge-repo` says so when it is not there. +if [[ "${JQ_PROM_ADMIN_API:-false}" == "true" ]]; then + prom_args+=(--web.enable-admin-api) + say "Prometheus admin API is ENABLED - series can be deleted without a password" +fi +# Process substitution, not a pipe: a pipeline's $! is the *last* command in +# it, so `kill $!` would reap the sed and leave Prometheus running. +# `sed -u` because a block-buffered log is a log you cannot tail. +prometheus "${prom_args[@]}" > >(sed -u 's/^/prometheus | /') 2>&1 & +pids+=($!) + +# -- grafana ----------------------------------------------------------------- +/usr/share/grafana/bin/grafana server \ + --homepath=/usr/share/grafana \ + --config="${GF_PATHS_CONFIG}" > >(sed -u 's/^/grafana | /') 2>&1 & +pids+=($!) + +# -- the collector ----------------------------------------------------------- +# Started last because it is the one that refuses to start on a repos.yml it +# cannot act on, and that error is what you want at the bottom of +# `docker logs` rather than buried above Grafana's startup banner. +# -u because this log is the only window into the container: block-buffered +# output would hold a startup error until enough of it accumulated to flush. +python -u -m jq_collector > >(sed -u 's/^/collector | /') 2>&1 & +pids+=($!) + +say "Grafana http://localhost:3000/d/jq-fleet · Prometheus :9090 · metrics :9109" + +# First exit wins: report which, then bring the rest down with it. +wait -n +code=$? +say "a process exited ($code) - stopping the others" +kill -TERM "${pids[@]}" 2>/dev/null +wait +exit "$code" diff --git a/scripts/purge-repo.sh b/image/purge-repo.sh similarity index 76% rename from scripts/purge-repo.sh rename to image/purge-repo.sh index 21127f9..cbe26a6 100755 --- a/scripts/purge-repo.sh +++ b/image/purge-repo.sh @@ -1,23 +1,26 @@ #!/usr/bin/env bash # Permanently delete a repo's history from Prometheus. # -# ./scripts/purge-repo.sh Jebel-Quant/rhiza-brainbug +# docker exec jq-fleet purge-repo Jebel-Quant/rhiza-brainbug # # Use after archiving or deleting a repo, when you do not want its old rows # lingering in long time windows. THIS IS IRREVERSIBLE - the samples are gone. # +# The admin API this needs is off unless the container was started with +# -e JQ_PROM_ADMIN_API=true, because it deletes series with no authentication +# at all and the board is otherwise a read-only thing. +# # Both label generations are purged: this board once used a bare repo name and # now uses owner/name, so a repo that predates that change has series under both. set -euo pipefail -cd "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" - if [[ $# -eq 0 ]]; then - echo "usage: $0 [more...]" >&2 + echo "usage: purge-repo [more...]" >&2 exit 64 fi PROM=http://localhost:9090 + selectors=() for repo in "$@"; do # A bare name would match nothing dangerous, but an empty or wildcard @@ -30,7 +33,23 @@ for repo in "$@"; do [[ "$repo" == */* ]] && selectors+=("{repo=\"${repo##*/}\"}") done -since() { date -u -v-3650d +%s 2>/dev/null || date -u -d '10 years ago' +%s; } +# Fail here, with the fix in hand, rather than after printing a delete plan and +# then reporting three HTTP 404s that look like a bug. +if [[ "$(curl -s -o /dev/null -w '%{http_code}' -X POST \ + "$PROM/api/v1/admin/tsdb/clean_tombstones")" == "404" ]]; then + cat >&2 <<'MSG' +the Prometheus admin API is disabled, so nothing can be deleted. + +Restart the container with it on, purge, then restart it without: + + docker rm -f jq-fleet + docker run ... -e JQ_PROM_ADMIN_API=true ghcr.io/jebel-quant/monitoring + docker exec jq-fleet purge-repo owner/name +MSG + exit 69 +fi + +since() { date -u -d '10 years ago' +%s 2>/dev/null || date -u -v-3650d +%s; } # Series metadata: what the index still lists. count() { @@ -71,10 +90,6 @@ if [[ "$total" -eq 0 ]]; then exit 0 fi -echo "enabling the admin API..." -docker compose -f docker-compose.yml -f docker-compose.admin.yml up -d prometheus >/dev/null -until [[ "$(curl -s -o /dev/null -w '%{http_code}' "$PROM/-/ready")" == "200" ]]; do sleep 1; done - for sel in "${selectors[@]}"; do curl -s -X POST -G "$PROM/api/v1/admin/tsdb/delete_series" --data-urlencode "match[]=$sel" \ -o /dev/null -w " delete $sel -> HTTP %{http_code}\n" @@ -83,10 +98,6 @@ done # delete_series only tombstones; this reclaims the blocks on disk. curl -s -X POST "$PROM/api/v1/admin/tsdb/clean_tombstones" -o /dev/null -w " clean_tombstones -> HTTP %{http_code}\n" -echo "disabling the admin API again..." -docker compose up -d prometheus >/dev/null -until [[ "$(curl -s -o /dev/null -w '%{http_code}' "$PROM/-/ready")" == "200" ]]; do sleep 1; done - echo "remaining:" stale=0 for sel in "${selectors[@]}"; do diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index 9e47616..2e0498b 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -9,13 +9,12 @@ global: scrape_configs: - job_name: jq-collector - # The collector runs on the host, not in this stack - it reads your working - # copies at their real paths, which a bind mount cannot express and which - # is far faster than one anyway. host.docker.internal is how a container - # reaches the machine it runs on; docker-compose.yml maps it on Linux, - # where it is not built in. + # Same container, so this is a loopback scrape. It used to be + # host.docker.internal: the collector ran on your machine because it reads + # your working copies and a per-repo bind mount could not name a checkout + # at an arbitrary path. One whole-home mount can, so it moved back in. static_configs: - - targets: ["host.docker.internal:9109"] + - targets: ["localhost:9109"] relabel_configs: # There is exactly one collector, so `instance` carries no information - # only where the process happened to be running. Left alone it is the diff --git a/repos.example.yml b/repos.example.yml index a55d074..ac2340b 100644 --- a/repos.example.yml +++ b/repos.example.yml @@ -1,17 +1,18 @@ # The fleet. Copy to repos.yml and edit - repos.yml is gitignored, because the # folder layout of your machine is nobody else's business. # -# One entry per monitored repo. `scripts/gen-repos.py` reads this file and -# writes docker-compose.repos.yml, which mounts each checkout read-only at -# /repos// and hands the collector the matching JQ_REPOS list. -# scripts/up.sh runs it for you; run it again after editing this file. +# This is the only file you own. It is mounted into the container at +# /config/repos.yml and read at startup; nothing is generated from it, so there +# is no second file to fall out of step. # # Nothing is discovered. A repo appears on the board because it is listed here, # and for no other reason. repos: - # The usual entry: a path to a checkout. `~` and paths relative to this file - # both work, and owner/name is read from the checkout's origin remote. + # The usual entry: a path to a checkout, written the way you would write it + # on your own machine. `~` is your home directory, which the container sees + # as a single read-only mount, and owner/name is read from the checkout's + # origin remote. - path: ~/repos/jebel-quant/monitoring - path: ~/repos/jebel-quant/rhiza @@ -24,5 +25,7 @@ repos: # repo: cvxpy/cvxpy # A repo with no checkout on this machine. GitHub panels (CI, pull requests, - # template drift) are gathered; the working-copy panels stay empty for it. + # template drift, coverage) are gathered; the working-copy panels stay empty + # for it. This is also the form to use for every entry if you would rather + # not mount your home directory at all. # - repo: Jebel-Quant/actions diff --git a/scripts/collector.sh b/scripts/collector.sh deleted file mode 100755 index cd3b91a..0000000 --- a/scripts/collector.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash -# Run the collector on this machine, in the foreground. -# -# Not in a container, on purpose. It reads your working copies, and a container -# can only reach those through bind mounts - which cannot name a checkout that -# sits somewhere other than //, and which are slow enough on -# macOS that the line counts needed a caching layer to stay affordable. -# -# Prometheus, which is still in Docker, scrapes this at host.docker.internal:9109. -# -# Run it under launchd (./scripts/up.sh sets that up on macOS), in a terminal, -# or under tmux - it is an ordinary foreground process either way, and Ctrl-C -# stops it. -# -# The trade this makes: inside the container the collector could only see the -# checkouts that were mounted into it, so an unlisted repo was not merely -# filtered out, it was invisible. Here it runs as you and could read anything -# you can. It still never writes - every git call is read-only and passes -# --no-optional-locks - but that is now a property of the code rather than -# something the sandbox enforces. -set -euo pipefail -cd "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" - -die() { printf '\033[31merror\033[0m %s\n' "$1" >&2; exit 1; } - -[[ -f .env ]] || die ".env not found - run ./scripts/up.sh first" -[[ -f repos.yml ]] || die "repos.yml not found - run ./scripts/up.sh first" -# Says "on PATH", not "installed", because under launchd it is almost always -# the former: an agent inherits /usr/bin:/bin:/usr/sbin:/sbin and nothing else, -# so a perfectly good uv under /opt/homebrew or ~/.local is invisible. up.sh -# pins uv's directory into the plist for exactly this reason. -command -v uv >/dev/null 2>&1 || die "uv is not on PATH ($PATH) - see https://docs.astral.sh/uv/" - -set -a; . ./.env; set +a - -# repos.yml is the fleet and the layout, and it stays the only place either is -# written down: both lines are computed at every launch rather than generated -# into a file that can fall out of step with it. -# -# `export "$line"` and not `export $line` - a path may contain spaces, and -# unquoted word splitting would turn one repo into two broken ones. -while IFS= read -r line; do - [[ -n "$line" ]] && export "$line" -done < <(uv run --quiet --with pyyaml python scripts/gen-repos.py) - -# No container means no bind mounts and no /repos to fall back on. Every path -# is real and absolute, and JQ_REPO_PATHS carries all of them. -export JQ_REPO_ROOT="" - -cd collector -exec uv run --quiet python -m jq_collector diff --git a/scripts/down.sh b/scripts/down.sh deleted file mode 100755 index cba596f..0000000 --- a/scripts/down.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -# Stop both halves. Add --volumes to also throw away the Prometheus history. -set -euo pipefail -cd "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" - -label="com.jebel-quant.jq-collector" -if [[ "$(uname -s)" == "Darwin" ]]; then - # KeepAlive means the agent restarts itself if merely killed, so unload it. - # Fails when it was never loaded, which is fine. - launchctl bootout "gui/$UID/$label" 2>/dev/null && echo "collector: launchd agent unloaded" || true -else - pkill -f "python -m jq_collector" 2>/dev/null && echo "collector: stopped" || true -fi - -docker compose down "$@" diff --git a/scripts/gen-repos.py b/scripts/gen-repos.py deleted file mode 100755 index 0b1965e..0000000 --- a/scripts/gen-repos.py +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env python3 -"""Turn repos.yml into the two environment lines the collector reads. - -The collector never searches for checkouts. This script resolves every listed -path, asks its origin remote who it is, and prints: - - JQ_REPOS the fleet, as owner/name - read by both halves - JQ_REPO_PATHS where each checkout actually is - read by the local half - - python3 scripts/gen-repos.py - -Nothing is written. scripts/collector.sh runs this at every launch and exports -the result, so repos.yml stays the only place the fleet and the layout are -written down - there is no generated file in between to go stale. - -The paths line exists because a repos.yml entry like - - - path: ~/repos/tschm/rhiza_projects/cs # a checkout of tschm/cs - -cannot be recovered by joining owner to name. This used to be papered over by -bind-mounting every checkout onto /repos//, back when the collector -ran in a container; it does not, so the real path has to be carried. -""" - -from __future__ import annotations - -import os -import subprocess -import sys -from pathlib import Path - -ROOT = Path(__file__).resolve().parent.parent -SOURCE = ROOT / "repos.yml" - - -def fail(message: str) -> None: - print(f"gen-repos: {message}", file=sys.stderr) - sys.exit(1) - - -def origin_owner_name(path: Path) -> tuple[str, str] | None: - """The ``(owner, name)`` a checkout's origin points at.""" - try: - url = subprocess.run( - ["git", "--no-optional-locks", "-C", str(path), "remote", "get-url", "origin"], - capture_output=True, - text=True, - timeout=20, - check=False, - ).stdout.strip() - except (OSError, subprocess.TimeoutExpired): - return None - if not url: - return None - url = url.removesuffix(".git") - if url.startswith("git@"): - tail = url.partition(":")[2] - elif "://" in url: - tail = url.split("://", 1)[1].split("/", 1)[-1] - else: - tail = url - parts = [p for p in tail.split("/") if p] - return (parts[-2], parts[-1]) if len(parts) >= 2 else None - - -def load_entries() -> list[dict]: - if not SOURCE.exists(): - fail(f"{SOURCE.name} not found - copy repos.example.yml to repos.yml and list your repos") - try: - import yaml - except ModuleNotFoundError: - fail( - "PyYAML is not installed - `uv run --with pyyaml scripts/gen-repos.py`, or pip install pyyaml" - ) - data = yaml.safe_load(SOURCE.read_text(encoding="utf-8")) or {} - entries = data.get("repos") if isinstance(data, dict) else None - if not isinstance(entries, list) or not entries: - fail(f"{SOURCE.name} lists no repos under a top-level `repos:` key") - return entries - - -def resolve(entry: object, index: int) -> tuple[str, Path | None]: - """One entry -> ``(owner/name, checkout path or None)``.""" - # `- ~/repos/foo` is accepted as shorthand for `- path: ~/repos/foo`. - if isinstance(entry, str): - entry = {"path": entry} - if not isinstance(entry, dict): - fail(f"entry {index} is neither a path nor a mapping: {entry!r}") - - named = str(entry.get("repo") or "").strip() - raw_path = entry.get("path") - - if raw_path is None: - if "/" not in named: - fail(f"entry {index} needs a `path`, or a `repo:` of the form owner/name") - return named, None - - # Relative paths are relative to this repo, so a checked-in repos.yml on a - # colleague's machine means the same thing as it does here. - path = Path(os.path.expanduser(str(raw_path))) - path = (path if path.is_absolute() else ROOT / path).resolve() - - if not path.is_dir(): - fail(f"{raw_path} is not a directory") - # `.git` is a directory in a plain checkout and a file in a worktree. - if not (path / ".git").exists(): - fail(f"{path} is not a git checkout") - - if "/" in named: - return named, path - - origin = origin_owner_name(path) - if origin is None: - fail(f"{path} has no usable origin remote - add an explicit `repo: owner/name`") - return f"{origin[0]}/{origin[1]}", path - - -def main() -> None: - unknown = sys.argv[1:] - if unknown: - fail(f"unknown argument {unknown[0]} - this script takes none") - - resolved: dict[str, Path | None] = {} - for index, entry in enumerate(load_entries(), start=1): - full_name, path = resolve(entry, index) - if full_name in resolved: - fail(f"{full_name} is listed twice") - resolved[full_name] = path - - # Nothing but the lines themselves on stdout, so they can be exported, - # piped, or appended to a .env. - print(f'JQ_REPOS={",".join(resolved)}') - - paths = {name: path for name, path in resolved.items() if path is not None} - # A comma is the separator, so a path containing one cannot be expressed. - # Refuse rather than print a line the collector would reject or, worse, - # silently misread as two repos. - for name, path in paths.items(): - if "," in str(path): - fail(f"{name}: path contains a comma, which JQ_REPO_PATHS cannot express: {path}") - if paths: - joined = ",".join(f"{name}={path}" for name, path in paths.items()) - print(f"JQ_REPO_PATHS={joined}") - - -if __name__ == "__main__": - main() diff --git a/scripts/up.sh b/scripts/up.sh deleted file mode 100755 index e16e40f..0000000 --- a/scripts/up.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env bash -# Bring both halves up: Prometheus and Grafana in Docker, the collector on this -# machine. Mints a GitHub token from the gh CLI if there isn't one. -# -# The collector is not containerised - see scripts/collector.sh for why. On -# macOS it is installed as a launchd agent so it comes back after a reboot; -# elsewhere this script says how to run it and leaves supervision to you. -set -euo pipefail - -here="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$here" - -if [[ ! -f .env ]]; then - cp .env.example .env - echo "created monitoring/.env from the example" -fi - -if [[ ! -f repos.yml ]]; then - cp repos.example.yml repos.yml - echo "created monitoring/repos.yml from the example - EDIT IT, then run this again" - echo "it lists the checkouts to monitor; the example points at paths you may not have" - exit 1 -fi - -# Only fill the token if it is still blank - never clobber one you set by hand. -if ! grep -qE '^GITHUB_TOKEN=.+' .env; then - if ! command -v gh >/dev/null 2>&1; then - echo "no GITHUB_TOKEN in .env and no gh CLI to mint one; set it by hand" >&2 - exit 1 - fi - token="$(gh auth token)" - # Portable in-place edit: BSD sed and GNU sed disagree about -i. - tmp="$(mktemp)" - sed "s|^GITHUB_TOKEN=.*|GITHUB_TOKEN=${token}|" .env > "$tmp" && mv "$tmp" .env - chmod 600 .env - echo "wrote a token from 'gh auth token' into monitoring/.env" -fi - -command -v uv >/dev/null 2>&1 || { - echo "uv is not installed - the collector needs it; see https://docs.astral.sh/uv/" >&2 - exit 1 -} - -# Fail on a broken repos.yml here, while there is still a human watching, rather -# than inside a launchd agent whose output nobody is reading. The collector -# recomputes this at every launch; this run is only a check. -uv run --quiet --with pyyaml python scripts/gen-repos.py >/dev/null - -docker compose up -d - -# -- the collector ----------------------------------------------------------- -label="com.jebel-quant.jq-collector" -if [[ "$(uname -s)" == "Darwin" ]]; then - plist="$HOME/Library/LaunchAgents/$label.plist" - # A launchd agent inherits almost no PATH - /usr/bin:/bin:/usr/sbin:/sbin and - # nothing else - so uv, which lives under /opt/homebrew or ~/.local, is simply - # not found. Pin the directory it is actually in rather than guessing at - # either location. - uv_dir="$(dirname "$(command -v uv)")" - logs="$here/.collector-logs" - mkdir -p "$HOME/Library/LaunchAgents" "$logs" - # Regenerated every time: the paths are absolute, so a moved checkout of this - # repo would otherwise leave a plist pointing at where it used to be. - cat > "$plist" < - - - - Label$label - ProgramArguments - $here/scripts/collector.sh - WorkingDirectory$here - EnvironmentVariables - - PATH$uv_dir:/usr/bin:/bin:/usr/sbin:/sbin - - RunAtLoad - KeepAlive - - ThrottleInterval10 - StandardOutPath$logs/collector.log - StandardErrorPath$logs/collector.log - - -PLIST - # bootout first so an edited plist is actually re-read; it fails when nothing - # is loaded, which is the normal first run. - launchctl bootout "gui/$UID/$label" 2>/dev/null || true - launchctl bootstrap "gui/$UID" "$plist" - echo "collector: loaded as a launchd agent - logs in .collector-logs/collector.log" -else - echo "collector: not started. Run it yourself, e.g." - echo " ./scripts/collector.sh # foreground" - echo " or install it as a systemd --user service running that script." -fi - -echo -echo "Grafana http://localhost:3000/d/jq-fleet (anonymous read-only; admin/admin to edit)" -echo "Prometheus http://localhost:9090" -echo "Collector http://localhost:9109/metrics"