diff --git a/README.md b/README.md index 20c1fc7..167849d 100644 --- a/README.md +++ b/README.md @@ -22,11 +22,17 @@ repos: - path: ~/repos/jebel-quant/rhiza # a checkout on this machine - path: ~/repos/cvxgrp/cvxsimulator - repo: Jebel-Quant/actions # monitored, but not cloned here + - repo: acme/platform/infra/web # GitLab, on the same board + forge: gitlab ``` -`owner/name` comes from each checkout's `origin`, so the path is all you write, -and the path is used as written — a checkout does not have to live at -`//`. Then: +`namespace/name` comes from each checkout's `origin`, so the path is all you +write, and the path is used as written — a checkout does not have to live at +`//`. **GitHub and GitLab repos share one board**; the forge +is read off the origin's host where there is a checkout, and stated with +`forge: gitlab` where there is not — see +[Configuration](docs/configuration.md#github-and-gitlab-in-the-same-fleet). +Then: ```bash docker run -d --name jq-fleet \ @@ -52,6 +58,7 @@ there is nothing to clone and nothing on your disk but `repos.yml`. | `-v "$HOME:/host:ro"` | Your home directory, read-only, so `~/...` in `repos.yml` resolves. Leave it out and the working-copy panels stay empty; everything GitHub reports still works | | `-v jq-fleet-data:/data` | Prometheus history and Grafana's database. Leave it out and both start empty at every run | | `-e GITHUB_TOKEN=...` | Needs `repo` and `read:org`, and must read every repo you listed. Without one GitHub allows 60 calls an hour, which is not a fleet | +| `-e GITLAB_TOKEN=...` | Only if the fleet has a GitLab repo in it. Needs `read_api` | `docker compose up -d` does the same thing with the flags written down; see [`docker-compose.yml`](docker-compose.yml). diff --git a/collector/jq_collector/__main__.py b/collector/jq_collector/__main__.py index bdcda6c..76a75c3 100644 --- a/collector/jq_collector/__main__.py +++ b/collector/jq_collector/__main__.py @@ -1,8 +1,14 @@ """Entry point: two refresh loops feeding one /metrics endpoint. The loops run on their own cadences and write into a shared store, so a scrape -never waits on the GitHub API. Prometheus can poll every 15s while GitHub is +never waits on a forge's API. Prometheus can poll every 15s while the forges are only asked every five minutes. + +There are two loops and not three, though there are now two forges. The remote +loop collects both and publishes once, because `Store.update(remote=...)` +replaces the whole map: a loop per forge would have each thread publish only its +own share, and the board would show half the fleet flickering against the other +half. Health and errors are still recorded per forge inside that one pass. """ from __future__ import annotations @@ -17,10 +23,11 @@ from prometheus_client import REGISTRY, start_http_server from . import github as gh +from . import gitlab as gl from . import localgit from .config import Config from .metrics import FleetCollector -from .state import SourceHealth, Store +from .state import RemoteRepo, SourceHealth, Store log = logging.getLogger("jq_collector") @@ -36,7 +43,18 @@ def _refresh_local(cfg: Config, store: Store) -> None: ) -def _refresh_github(cfg: Config, store: Store) -> None: +def _refresh_remote(cfg: Config, store: Store) -> None: + """Collect every forge in the fleet and publish the result as one snapshot. + + One pass over both forges rather than a loop each, because + ``Store.update(remote=...)`` replaces the whole map: two threads writing it + would each publish only their own share, and the board would show half the + fleet flickering against the other half. + + GitHub is collected even when no repo lives there, because it is where the + template's releases are published and therefore where drift is measured + from, whichever forge the repo pinning it is on. + """ snap = store.snapshot() # (default_branch_sha, ref) per repo: the pointer can only have changed if # the branch head moved, so this keeps the drift read correct without @@ -46,7 +64,7 @@ def _refresh_github(cfg: Config, store: Store) -> None: } # (artifact id, percent) per repo: an unchanged artifact means the report # behind it is byte-identical, so there is nothing to gain from pulling the - # zip down again. + # zip down again. GitLab reports coverage as a field and ignores this. coverage_cache = { name: (repo.coverage_artifact, (repo.coverage, repo.coverage_lines)) if repo.coverage is not None @@ -54,18 +72,53 @@ def _refresh_github(cfg: Config, store: Store) -> None: for name, repo in snap.remote.items() if repo.coverage_artifact } - remote, api, latest, excluded = gh.collect(cfg, ref_cache, coverage_cache) - try: - store.update( - remote=remote, - excluded=excluded, - latest_template_ref=latest, - rate_limit_remaining=api.rate_remaining, - rate_limit_limit=api.rate_limit, - rate_limit_reset=api.rate_reset, + + by_forge = cfg.repos_by_forge() + remote: dict[str, RemoteRepo] = {} + excluded: set[str] = set() + # The template's release tags, fetched by the GitHub pass and needed by the + # GitLab one to measure drift against the same list. + template_tags: list[str] = [] + + def gather_github() -> None: + result, api, tags, dropped = gh.collect( + cfg, ref_cache, coverage_cache, fleet=by_forge.get("github", ()) ) - finally: - api.close() + try: + remote.update(result) + excluded.update(dropped) + template_tags[:] = tags + store.update( + latest_template_ref=tags[0] if tags else "", + rate_limit_remaining=api.rate_remaining, + rate_limit_limit=api.rate_limit, + rate_limit_reset=api.rate_reset, + ) + finally: + api.close() + + def gather_gitlab() -> None: + result, api, dropped = gl.collect( + cfg, + ref_cache, + coverage_cache, + fleet=by_forge["gitlab"], + tags=template_tags, + ) + try: + remote.update(result) + excluded.update(dropped) + finally: + api.close() + + # Health is recorded per forge, so a GitLab outage shows up as + # jq_collector_errors{source="gitlab"} and leaves the GitHub repos standing + # rather than shortening the fleet silently. + _run_once("github", store, gather_github) + if by_forge.get("gitlab"): + _run_once("gitlab", store, gather_gitlab) + + store.update(remote=remote, excluded=frozenset(excluded)) def _run_once(source: str, store: Store, work: Callable[[], None]) -> None: @@ -99,19 +152,37 @@ def _run_once(source: str, store: Store, work: Callable[[], None]) -> None: log.info("%s refresh finished in %.1fs", source, elapsed) +def _tick_local(cfg: Config, store: Store) -> None: + """One local refresh, health recorded.""" + _run_once("local", store, lambda: _refresh_local(cfg, store)) + + def _loop( - source: str, interval: int, - store: Store, - work: Callable[[], None], + tick: Callable[[], None], stop: threading.Event, ) -> None: - """Refresh forever. Waits first, because main() has already seeded once.""" + """Refresh forever. Waits first, because main() has already seeded once. + + The tick owns its own health bookkeeping. That moved out of here when the + remote pass grew from one source to one per forge: a single `source` name + per loop could no longer describe what the tick had actually talked to. + + Which leaves this loop holding the last resort. A tick records what each + forge did, but the work between forges - merging their results and + publishing the snapshot - belongs to no single source and would otherwise + escape. An exception reaching a thread's target kills the thread silently, + and the loop is the only thing that will ever try again, so nothing may get + out of here. + """ while not stop.is_set(): stop.wait(interval) if stop.is_set(): return - _run_once(source, store, work) + try: + tick() + except Exception: + log.exception("refresh tick failed") def main() -> None: @@ -122,19 +193,23 @@ def main() -> None: cfg = Config() if not cfg.token: log.warning("no GITHUB_TOKEN set - unauthenticated GitHub calls are limited to 60/hour") + # Only warn about the token a fleet actually needs. A GitHub-only deployment + # must not start complaining about GitLab credentials it has no use for. + if cfg.repos_by_forge().get("gitlab") and not cfg.gitlab_token: + log.warning("no GITLAB_TOKEN set - only public GitLab projects will be readable") store = Store() # Seed both sources BEFORE opening the port. The local pass takes ~2s and - # the GitHub pass ~11s, and serving in between would publish a snapshot with + # the remote pass ~11s, and serving in between would publish a snapshot with # working copies but no CI, drift or pull requests - which Prometheus stores # as a real observation, leaving a permanent dip in the history that reads # as "nothing was behind the template" rather than "we had not looked yet". # A refused connection for those few seconds is an honest target-down. - # GitHub runs first: the local pass needs its default-branch names to know - # which local ref to compare against, and nothing now flows the other way. - _run_once("github", store, lambda: _refresh_github(cfg, store)) - _run_once("local", store, lambda: _refresh_local(cfg, store)) + # The forges run first: the local pass needs their default-branch names to + # know which local ref to compare against, and nothing flows the other way. + _refresh_remote(cfg, store) + _tick_local(cfg, store) # The default registry already carries the process and GC collectors, so # adding ours to it keeps exporter self-monitoring on the same endpoint. @@ -146,27 +221,15 @@ def main() -> None: threads = [ threading.Thread( target=_loop, - args=( - "local", - cfg.local_interval, - store, - lambda: _refresh_local(cfg, store), - stop, - ), + args=(cfg.local_interval, lambda: _tick_local(cfg, store), stop), daemon=True, name="local", ), threading.Thread( target=_loop, - args=( - "github", - cfg.github_interval, - store, - lambda: _refresh_github(cfg, store), - stop, - ), + args=(cfg.github_interval, lambda: _refresh_remote(cfg, store), stop), daemon=True, - name="github", + name="remote", ), ] # The seed above already ran both, so hold each loop off for one interval. diff --git a/collector/jq_collector/config.py b/collector/jq_collector/config.py index ed801b3..d5c690d 100644 --- a/collector/jq_collector/config.py +++ b/collector/jq_collector/config.py @@ -6,7 +6,8 @@ import os from dataclasses import dataclass, field -from .repos import FleetError, load +from .origin import GITHUB +from .repos import KNOWN_FORGES, FleetError, load log = logging.getLogger(__name__) @@ -45,6 +46,25 @@ def _pairs(name: str) -> dict[str, str]: return mapping +def _forges(name: str) -> dict[str, str]: + """``owner/name=forge`` pairs, comma separated, into a mapping. + + Raises on an unknown forge for the same reason `repos._declared_forge` + does: reading a name nobody implements as GitHub would put the repo on the + board with every remote panel wrong and nothing saying why. + """ + mapping: dict[str, str] = {} + for item in _csv(name): + key, separator, value = item.partition("=") + key, value = key.strip(), value.strip().lower() + if not separator or not key or not value: + raise ValueError(f"{name}: expected owner/name=forge, got {item!r}") + if value not in KNOWN_FORGES: + raise ValueError(f"{name}: forge {value!r} is not one of {', '.join(KNOWN_FORGES)}") + mapping[key] = value + return mapping + + @dataclass(frozen=True) class Config: """Everything the collector needs to know about its environment. @@ -71,6 +91,13 @@ class Config: token: str = os.environ.get("GITHUB_TOKEN", "") api: str = os.environ.get("GITHUB_API", "https://api.github.com") + # The second forge. Kept as its own pair rather than a per-forge mapping + # because gitlab.com is the only GitLab this supports: a fleet spanning a + # self-hosted instance as well would need the base URL per entry, and there + # is no point paying for that generality until somebody has one. + gitlab_token: str = os.environ.get("GITLAB_TOKEN", "") + gitlab_api: str = os.environ.get("GITLAB_API", "https://gitlab.com/api/v4") + # The fleet file. Read when it is there; when it is not, JQ_REPOS and # JQ_REPO_PATHS are the whole story. /config/repos.yml is where the image # expects it to be mounted. @@ -92,6 +119,11 @@ class Config: # overridable from the environment for a deployment that has no file. repo_paths: dict[str, str] = field(default_factory=lambda: _pairs("JQ_REPO_PATHS")) + # Which forge each repo is read through, as owner/name=forge. Filled from + # repos.yml; anything unlisted is GitHub, which is what makes a fleet that + # predates GitLab support keep working untouched. + forges: dict[str, str] = field(default_factory=lambda: _forges("JQ_REPO_FORGES")) + # owner/name of the repo whose releases define "up to date". template_repo: str = os.environ.get("JQ_TEMPLATE_REPO", "Jebel-Quant/rhiza") template_pointer: str = os.environ.get("JQ_TEMPLATE_POINTER", ".rhiza/template.yml") @@ -148,14 +180,41 @@ def __post_init__(self) -> None: if not self.repos_file or not os.path.isfile(self.repos_file): return try: - fleet, paths = load(self.repos_file, self.host_root) + fleet, paths, forges = load(self.repos_file, self.host_root) except FleetError as exc: raise SystemExit(f"repos.yml: {exc}") from exc object.__setattr__(self, "repos", fleet) # The environment wins: it is the narrower, more deliberate statement, # and it is how one path can be corrected without touching the file. object.__setattr__(self, "repo_paths", {**paths, **self.repo_paths}) + object.__setattr__(self, "forges", {**forges, **self.forges}) log.info("fleet: %d repos, %d with a checkout", len(fleet), len(self.repo_paths)) + # A second line rather than a clause on the first, and only when there + # is a split worth reporting: a GitHub-only fleet logs exactly what it + # logged before GitLab support existed, which is what CI asserts on and + # what anybody reading these logs already recognises. + by_forge = self.repos_by_forge() + if len(by_forge) > 1: + log.info( + "forges: %s", + ", ".join(f"{len(names)} on {forge}" for forge, names in sorted(by_forge.items())), + ) def is_ignored(self, owner: str, name: str) -> bool: return name in self.ignore or f"{owner}/{name}" in self.ignore + + def forge_for(self, full_name: str) -> str: + """Which forge a repo is read through. Unlisted means GitHub.""" + return self.forges.get(full_name, GITHUB) + + def repos_by_forge(self) -> dict[str, tuple[str, ...]]: + """The fleet grouped by forge, so each API is asked once for its share. + + Only forges that actually have repos appear, which is what keeps a + GitHub-only fleet from ever building a GitLab client or warning about a + token it has no use for. + """ + grouped: dict[str, list[str]] = {} + for full_name in self.repos: + grouped.setdefault(self.forge_for(full_name), []).append(full_name) + return {forge: tuple(names) for forge, names in grouped.items()} diff --git a/collector/jq_collector/forge.py b/collector/jq_collector/forge.py new file mode 100644 index 0000000..378d590 --- /dev/null +++ b/collector/jq_collector/forge.py @@ -0,0 +1,86 @@ +"""The seam between "a forge" and the rest of the collector. + +``RemoteRepo`` asks the same questions of every forge - what is the default +branch, is it protected, is CI green, what is open - so the only thing that +varies is which URLs answer them and in what words. This module holds the +protocol both collectors satisfy and the one translation that matters: CI +verdicts. + +GitHub and GitLab disagree on the vocabulary. GitHub conclusions are ``success``, +``failure``, ``cancelled``, ``skipped``, ``neutral``, ``stale``; GitLab pipeline +and job statuses are ``success``, ``failed``, ``canceled`` (one l), ``skipped``, +``manual``, ``running``, plus a handful of pre-run states. ``metrics.py`` and the +dashboard's value mappings are written against GitHub's set, and that set is +also what the stored history says. So the GitLab collector normalises on the way +in and nothing downstream has to know there was ever a second spelling. +""" + +from __future__ import annotations + +from typing import Protocol + +from .state import RemoteRepo + +# Conclusions that mean the run did its job. +GOOD_CONCLUSIONS = frozenset({"success", "neutral", "skipped"}) + +# Conclusions that are no verdict at all - cancelled by hand or superseded by a +# newer push, and `stale` for a run that never really happened. Neither is green +# or red, so they are left out of the exposition rather than counted as failures. +INCONCLUSIVE_CONCLUSIONS = frozenset({"cancelled", "stale"}) + +# GitLab's spelling -> the vocabulary the board already speaks. +# +# `manual` is a job waiting for somebody to press the button - a deploy gate, +# typically. It is not a failure and not a success, so it maps onto the same +# inconclusive bucket as a cancelled run; counting held deploys as red would +# make a correctly-configured pipeline permanently angry. +# +# The pre-run states (`created`, `pending`, `preparing`, `waiting_for_resource`, +# `scheduled`, `running`) describe a pipeline still in flight. They are also +# inconclusive: the board reports the last *completed* state of the default +# branch, so a run in progress must not overwrite the verdict of the one before. +_GITLAB_STATUS = { + "success": "success", + "failed": "failure", + "canceled": "cancelled", + "canceling": "cancelled", + "skipped": "skipped", + "manual": "cancelled", + "created": "stale", + "pending": "stale", + "preparing": "stale", + "waiting_for_resource": "stale", + "waiting_for_callback": "stale", + "scheduled": "stale", + "running": "stale", +} + + +def normalise_gitlab_status(status: str) -> str: + """One GitLab status as a GitHub conclusion. + + An unrecognised status becomes ``stale`` rather than ``failure``: GitLab has + added states before and will again, and a new one appearing as a fleet-wide + red is a worse failure mode than it appearing as "no verdict yet". + """ + return _GITLAB_STATUS.get((status or "").strip().lower(), "stale") + + +class RemoteSource(Protocol): + """What ``__main__`` needs of a forge client. + + ``collect`` returns the repos it was able to read, keyed by + ``namespace/name``, plus the set it deliberately dropped (archived, or named + in ``JQ_IGNORE``). Both caches are keyed the same way and may hold entries + for repos on other forges; an implementation is expected to ignore those + rather than trip over them. + """ + + def collect( + self, + ref_cache: dict[str, tuple[str, str]], + coverage_cache: dict[str, tuple[int, tuple[float, int] | None]], + ) -> tuple[dict[str, RemoteRepo], frozenset[str]]: ... + + def close(self) -> None: ... diff --git a/collector/jq_collector/github.py b/collector/jq_collector/github.py index 532c3e1..034ad22 100644 --- a/collector/jq_collector/github.py +++ b/collector/jq_collector/github.py @@ -28,19 +28,15 @@ import httpx from .config import Config +from .forge import GOOD_CONCLUSIONS, INCONCLUSIVE_CONCLUSIONS from .state import MergedPull, PullRequest, RemoteRepo, WorkflowRun log = logging.getLogger(__name__) -# Conclusions that mean the workflow did its job. Kept here as well as in -# metrics.py because the representative-run choice depends on it. -GOOD_CONCLUSIONS = frozenset({"success", "neutral", "skipped"}) - -# Conclusions that are no verdict at all. A run cancelled by hand, or by a -# concurrency group when a newer push superseded it, says nothing about the -# branch - and `stale` means it never really ran. Treating either as red made a -# repo failing for the sole reason that somebody stopped a job. -INCONCLUSIVE_CONCLUSIONS = frozenset({"cancelled", "stale"}) +# Both re-exported. They used to be defined here and again in metrics.py, and a +# second forge would have made three copies of one contract. They live in +# forge.py now, next to the table that translates GitLab's spelling into them. +__all__ = ["GOOD_CONCLUSIONS", "INCONCLUSIVE_CONCLUSIONS", "GitHub", "collect"] _ACCEPT = "application/vnd.github+json" _MAX_WORKERS = 8 @@ -145,17 +141,21 @@ def _paginate(self, path: str, **params: object) -> list[dict]: # -- fleet-level ----------------------------------------------------- - def list_repos(self) -> list[dict]: + def list_repos(self, fleet: tuple[str, ...] | None = None) -> list[dict]: """The repos named in the config, in the order they were listed. One call each, and no org sweep: the fleet is whatever you wrote down. A repo that cannot be read is dropped with a warning rather than failing the refresh, so one bad line does not blank the whole board. + + ``fleet`` narrows that list to this forge's share. Without it every + GitLab repo in a mixed fleet would be asked of GitHub, which answers 404 + and logs each one as unreadable. """ repos: list[dict] = [] seen: set[str] = set() - for full_name in self._cfg.repos: + for full_name in fleet if fleet is not None else self._cfg.repos: if "/" not in full_name or full_name in seen: continue raw = self._json(f"/repos/{full_name}") @@ -516,8 +516,17 @@ def collect( cfg: Config, ref_cache: dict[str, tuple[str, str]], coverage_cache: dict[str, tuple[int, tuple[float, int] | None]] | None = None, -) -> tuple[dict[str, RemoteRepo], GitHub, str, frozenset[str]]: - """Build the remote half of the snapshot, keyed by ``owner/name``. + fleet: tuple[str, ...] | None = None, +) -> tuple[dict[str, RemoteRepo], GitHub, list[str], frozenset[str]]: + """Build the GitHub part of the snapshot, keyed by ``owner/name``. + + Returns the template repo's release tags along with the repos, because the + GitLab collector needs the same list to measure its own repos' drift: the + template lives on GitHub wherever the repo pinning it lives, so fetching the + tags once here and passing them on keeps it to one call per refresh. + + ``fleet`` is this forge's share of the repos. It defaults to the whole + fleet, which is right for a GitHub-only deployment and for every test. The template pointer is always read from GitHub's default branch, never from the local clone. Drift is a property of the *repo*, and a clone can be @@ -539,7 +548,6 @@ def collect( coverage_cache = coverage_cache or {} api = GitHub(cfg) tags = api.release_tags(cfg.template_repo) - latest = tags[0] if tags else "" # Dropped repos are reported back so the local scan can skip their clones # too - otherwise a checkout keeps a dead repo on the board after the GitHub @@ -549,7 +557,7 @@ def collect( # honoured only by the scan, via Config.wants(), so ignoring a repo silently # removed its working-copy rows while leaving every CI, drift and # pull-request series in place. - listing = api.list_repos() + listing = api.list_repos(fleet) excluded = frozenset( r["full_name"] for r in listing @@ -618,6 +626,9 @@ def one(raw: dict) -> RemoteRepo: return RemoteRepo( name=raw["name"], owner=owner, + forge="github", + url=raw.get("html_url") or "", + pulls_url=f"{raw['html_url']}/pulls" if raw.get("html_url") else "", default_branch=branch, visibility=raw.get("visibility") or "unknown", archived=bool(raw.get("archived")), @@ -656,4 +667,4 @@ def one(raw: dict) -> RemoteRepo: except Exception as exc: # noqa: BLE001 - one bad repo must not sink the refresh log.warning("repo %s failed: %s", full_name, exc) - return result, api, latest, excluded + return result, api, tags, excluded diff --git a/collector/jq_collector/gitlab.py b/collector/jq_collector/gitlab.py new file mode 100644 index 0000000..6c709c0 --- /dev/null +++ b/collector/jq_collector/gitlab.py @@ -0,0 +1,524 @@ +"""Read the state of the repos as GitLab sees them. + +The GitHub twin of this module (``github.py``) is the reference; this one answers +the same questions of a different API. Two differences are worth knowing before +reading it. + +**The cost.** Roughly ``8 * repos + open_mrs`` calls per refresh (nine on the +first pass, before the template pointer is cached), against GitHub's +``6 * repos + workflows + open_pull_requests``. It is not the bargain it first +looked like: + +- coverage genuinely is cheaper. It is a field on the pipeline, so there is no + artifact to list and no zip to download (``github.GitHub.coverage_percent``). + The coverage cache this module's ``collect`` still accepts is therefore + unused - it is in the protocol because GitHub needs it, and honouring the + shape keeps ``__main__`` from special-casing either forge. +- MR checks are not. GitLab documents ``head_pipeline`` on the merge-request + response, but only the *single* MR endpoint returns it - the listing omits the + key entirely, and reading that absence as "no pipeline" reported every GitLab + MR as unchecked. So there is one follow-up call per open MR, exactly as on + GitHub. See ``head_pipeline_status``. +- The pipeline itself costs two calls where GitHub's workflow runs cost one: the + listing carries a status but not coverage, so the newest one is fetched again + in full. + +**Projects are addressed by URL-encoded path.** ``acme/platform/infra/web`` +becomes ``acme%2Fplatform%2Finfra%2Fweb``. GitLab namespaces nest, which is why +``origin.parse`` keeps the whole path rather than the last two segments. + +One panel cannot be filled. Dependabot's counterpart is GitLab's vulnerability +report, an Ultimate-tier feature whose REST API is being retired in favour of +GraphQL, so ``alerts_enabled`` is always False here. That is the honest answer +rather than a misleading one: ``state.RemoteRepo`` keeps "the feature is off" +apart from "zero open alerts" precisely so an unscanned repo does not render as +a green tile. +""" + +from __future__ import annotations + +import concurrent.futures +import logging +from datetime import datetime +from urllib.parse import quote + +import httpx + +from .config import Config +from .forge import GOOD_CONCLUSIONS, normalise_gitlab_status +from .state import MergedPull, PullRequest, RemoteRepo, WorkflowRun + +log = logging.getLogger(__name__) + +_MAX_WORKERS = 8 + +# GitLab's own cap. Asking for more is not an error, it is silently clamped, so +# there is nothing to gain by trying. +_PER_PAGE = 100 + + +def _ts(value: str | None) -> float: + """One GitLab timestamp as epoch seconds, or 0.0 if it is unusable. + + GitLab returns ``2026-08-31T06:05:36.000Z``, where GitHub's has no + milliseconds. Both parse as they stand: 3.11 taught ``fromisoformat`` the + whole of ISO 8601, including the trailing ``Z``, and 3.11 is this package's + floor - so this is `github._ts` with a different docstring. + """ + if not value: + return 0.0 + try: + return datetime.fromisoformat(value).timestamp() + except ValueError: + return 0.0 + + +def _pid(full_name: str) -> str: + """A project path as GitLab's ``:id`` path segment.""" + return quote(full_name, safe="") + + +class GitLab: + def __init__(self, cfg: Config) -> None: + self._cfg = cfg + headers = {"Accept": "application/json"} + if cfg.gitlab_token: + # PRIVATE-TOKEN takes a personal or project access token; Bearer + # would be an OAuth token, which is not what GITLAB_TOKEN holds. + headers["PRIVATE-TOKEN"] = cfg.gitlab_token + self._client = httpx.Client( + base_url=cfg.gitlab_api, + headers=headers, + timeout=cfg.http_timeout, + follow_redirects=True, + ) + + def close(self) -> None: + self._client.close() + + def _json(self, path: str, **params: object) -> object | None: + """GET returning parsed JSON, or None for the expected empty cases. + + The same three states the GitHub client tolerates, for the same reason: + 404 is a missing file or a project the token cannot see, 403 is a + feature the tier does not include, and both are things a real fleet + contains. Failing the refresh over one would blank the whole board. + """ + response = self._get(path, **params) + if response is None: + return None + return response.json() + + def _get(self, path: str, **params: object) -> httpx.Response | None: + """GET, or None for the statuses a real fleet legitimately produces.""" + response = self._client.get(path, params=params or None) + if response.status_code in (401, 403, 404): + log.info("%s -> %s", path, response.status_code) + return None + response.raise_for_status() + return response + + def _text(self, path: str, **params: object) -> str | None: + """GET returning the body as text, for the routes that serve a file. + + ``repository/files/.../raw`` answers with the file itself, so parsing it + as JSON would fail on any ordinary YAML pointer. + """ + response = self._get(path, **params) + return response.text if response is not None else None + + def _paginate(self, path: str, **params: object) -> list[dict]: + items: list[dict] = [] + page = 1 + while True: + batch = self._json(path, per_page=_PER_PAGE, page=page, **params) + if not isinstance(batch, list) or not batch: + break + items.extend(batch) + if len(batch) < _PER_PAGE: + break + page += 1 + if page > 10: # a fleet this size never needs more + break + return items + + # -- fleet-level ----------------------------------------------------- + + def list_projects(self, fleet: tuple[str, ...]) -> list[dict]: + """The projects named in the config, in the order they were listed. + + One call each, no group sweep - the same contract as + ``github.GitHub.list_repos``, and for the same reason: the board's + contents are decided by repos.yml and by nothing else. + """ + projects: list[dict] = [] + seen: set[str] = set() + + for full_name in fleet: + if "/" not in full_name or full_name in seen: + continue + raw = self._json(f"/projects/{_pid(full_name)}") + if isinstance(raw, dict) and raw.get("path_with_namespace"): + seen.add(raw["path_with_namespace"]) + projects.append(raw) + else: + log.warning( + "listed repo %s is not readable on GitLab - " + "check the path and the token's scopes", + full_name, + ) + + return projects + + def protected_branch(self, full_name: str, branch: str) -> dict | None: + """The branch's protection entry, or None if it is not protected. + + Unlike GitHub's, this endpoint needs no admin rights, so there is no + third "we were not allowed to look" state to model: a 404 here really + does mean the branch is unprotected. + """ + raw = self._json(f"/projects/{_pid(full_name)}/protected_branches/{quote(branch, safe='')}") + return raw if isinstance(raw, dict) else None + + def branch_sha(self, full_name: str, branch: str) -> str: + data = self._json( + f"/projects/{_pid(full_name)}/repository/branches/{quote(branch, safe='')}" + ) + if not isinstance(data, dict): + return "" + return ((data.get("commit") or {}).get("id")) or "" + + def template_ref(self, full_name: str, branch: str) -> str: + """The ref pinned in the repo's template pointer, if it is managed.""" + import yaml + + pointer = quote(self._cfg.template_pointer, safe="") + raw = self._text( + f"/projects/{_pid(full_name)}/repository/files/{pointer}/raw", + ref=branch, + ) + if not raw: + return "" + try: + data = yaml.safe_load(raw) or {} + except yaml.YAMLError as exc: + # Data, not a crash: a hand-edited pointer is one repo's problem and + # must not take the rest of the refresh down with it. + log.warning("%s has a malformed template pointer: %s", full_name, exc) + return "" + return str(data.get("ref") or "") if isinstance(data, dict) else "" + + def latest_pipeline(self, full_name: str, branch: str) -> dict | None: + """The newest pipeline on *branch*, with its coverage. + + The listing carries a status but not coverage, so the one worth having + is fetched in full. Two calls, and only for the newest pipeline. + """ + listing = self._json( + f"/projects/{_pid(full_name)}/pipelines", + ref=branch, + per_page=1, + order_by="id", + sort="desc", + ) + if not isinstance(listing, list) or not listing: + return None + pipeline_id = listing[0].get("id") + if pipeline_id is None: + return None + full = self._json(f"/projects/{_pid(full_name)}/pipelines/{pipeline_id}") + return full if isinstance(full, dict) else listing[0] + + def pipeline_jobs(self, full_name: str, pipeline_id: int) -> list[dict]: + """Every job in one pipeline. + + A GitLab pipeline is one run containing many jobs, where GitHub has many + independent workflows. The job is the closest analogue to a workflow - + it is the thing that is individually green or red and individually worth + naming on the board - so one job becomes one `WorkflowRun`. + """ + return self._paginate( + f"/projects/{_pid(full_name)}/pipelines/{pipeline_id}/jobs", + include_retried="false", + ) + + def head_pipeline_status(self, full_name: str, iid: int) -> str | None: + """The status of the pipeline on one MR's head commit. + + A call per MR, which the listing looked like it would save: GitLab + documents ``head_pipeline`` on the merge-request response, but only the + *single* MR endpoint actually returns it - the listing omits it, and + reading the absence as "no pipeline" reported every GitLab MR as having + no checks at all. So this mirrors ``github.GitHub.checks_state``: one + follow-up per MR, capped by ``max_prs_per_repo``. + """ + raw = self._json(f"/projects/{_pid(full_name)}/merge_requests/{iid}") + if not isinstance(raw, dict): + return None + pipeline = raw.get("head_pipeline") or raw.get("pipeline") or {} + return pipeline.get("status") if isinstance(pipeline, dict) else None + + def open_merge_requests(self, full_name: str) -> tuple[int, list[PullRequest]]: + """``(total open, the first N)`` as ``PullRequest`` records. + + The total is reported separately because the detail list is clipped - + the tile stays right on a repo with an MR flood. + """ + raw = self._paginate( + f"/projects/{_pid(full_name)}/merge_requests", + state="opened", + order_by="updated_at", + sort="desc", + ) + total = len(raw) + pulls: list[PullRequest] = [] + for item in raw[: self._cfg.max_prs_per_repo]: + # iid, not id: iid is the number shown in the UI and in the MR's own + # URL. id is globally unique and means nothing to anyone reading the + # board. + iid = int(item.get("iid") or 0) + status = self.head_pipeline_status(full_name, iid) if iid else None + pulls.append( + PullRequest( + number=iid, + # Clipped like GitHub's, so one essay of a title cannot + # stretch the table's column past everything else. + title=(item.get("title") or "")[:120], + author=(item.get("author") or {}).get("username") or "unknown", + draft=bool(item.get("draft")), + created_at=_ts(item.get("created_at")), + updated_at=_ts(item.get("updated_at")), + checks=_checks_state(status), + url=item.get("web_url") or "", + ) + ) + return total, pulls + + def recent_merges(self, full_name: str, limit: int) -> list[MergedPull]: + raw = self._json( + f"/projects/{_pid(full_name)}/merge_requests", + state="merged", + order_by="updated_at", + sort="desc", + per_page=limit, + ) + if not isinstance(raw, list): + return [] + merged = [] + for item in raw: + at = _ts(item.get("merged_at")) + if not at: + continue + merged.append( + MergedPull( + number=int(item.get("iid") or 0), + title=item.get("title") or "", + author=(item.get("author") or {}).get("username") or "", + merged_at=at, + url=item.get("web_url") or "", + ) + ) + return merged + + +def _checks_state(status: str | None) -> str: + """One MR's pipeline status in the vocabulary the board's PR table uses. + + That vocabulary is `success | failure | cancelled | pending | none`, which is + GitHub's check-run summary rather than GitLab's pipeline status, so the + in-flight states collapse to `pending` here rather than to the `stale` that + `normalise_gitlab_status` gives a *completed* run. + """ + if not status: + return "none" + status = status.strip().lower() + if status in ( + "running", + "pending", + "created", + "preparing", + "waiting_for_resource", + "scheduled", + ): + return "pending" + if status == "success": + return "success" + if status in ("canceled", "canceling", "manual", "skipped"): + return "cancelled" + if status == "failed": + return "failure" + return "none" + + +def _visibility(raw: dict) -> str: + """The project's visibility: ``public``, ``internal`` or ``private``. + + Named rather than inlined because ``public_only`` tests it too, and + ``internal`` must fall on the private side of that test - it means "every + signed-in user on the instance", which is not public but reads like it. + """ + return raw.get("visibility") or "unknown" + + +def _protection(entry: dict | None) -> tuple[bool | None, bool, int]: + """``(protected, allows_force_push, required_reviews)`` from one entry. + + ``protected`` is never None on GitLab. GitHub's third state exists because + its protection endpoint 404s both for an unprotected branch and for a token + without admin; GitLab's needs no special rights, so a 404 is unambiguous. + + ``required_reviews`` is always 0. MR approval rules are a Premium feature + and the free tier has no equivalent to report, so `protected` carries the + signal and the review count stays honestly empty. + """ + if entry is None: + return False, False, 0 + return True, bool(entry.get("allow_force_push")), 0 + + +def _behind_count(tags: list[str], ref: str) -> int | None: + """How many template releases *ref* is behind, or None if it is not one. + + Deliberately identical in meaning to ``github._behind_count``: the template + lives on GitHub whichever forge the repo pinning it lives on, so drift is + measured against the same tag list either way. + """ + if not ref or ref not in tags: + return None + return tags.index(ref) + + +def collect( + cfg: Config, + ref_cache: dict[str, tuple[str, str]], + coverage_cache: dict[str, tuple[int, tuple[float, int] | None]] | None = None, + fleet: tuple[str, ...] = (), + tags: list[str] | None = None, +) -> tuple[dict[str, RemoteRepo], GitLab, frozenset[str]]: + """Build the GitLab part of the snapshot, keyed by ``namespace/name``. + + ``tags`` is the template repo's releases, fetched once by the GitHub + collector and passed in - see this module's docstring. Without them drift is + reported as unknown rather than as zero, which is the same thing an + unmanaged repo reports and is the honest answer when the comparison could + not be made. + + ``ref_cache`` maps ``full_name -> (default_branch_sha, ref)`` from the + previous refresh, exactly as in ``github.collect``: the pointer cannot have + changed if the branch head has not moved. ``coverage_cache`` is accepted and + ignored, because coverage here is a field rather than a download. + """ + tags = tags or [] + api = GitLab(cfg) + + listing = api.list_projects(fleet) + excluded = frozenset( + r["path_with_namespace"] + for r in listing + if (r.get("archived") and not cfg.include_archived) + or cfg.is_ignored(*r["path_with_namespace"].rsplit("/", 1)) + or (cfg.public_only and _visibility(r) != "public") + ) + projects = [r for r in listing if r["path_with_namespace"] not in excluded] + + def one(raw: dict) -> RemoteRepo: + full_name = raw["path_with_namespace"] + branch = raw.get("default_branch") or "main" + head_sha = api.branch_sha(full_name, branch) + + cached = ref_cache.get(full_name) + if cached is not None and head_sha and cached[0] == head_sha: + ref = cached[1] + else: + ref = api.template_ref(full_name, branch) + + pipeline = api.latest_pipeline(full_name, branch) or {} + coverage = pipeline.get("coverage") + # GitLab reports coverage as a string percentage, and as null when the + # project has no coverage regex configured. + try: + coverage = float(coverage) if coverage is not None else None + except (TypeError, ValueError): + coverage = None + + workflows: tuple[WorkflowRun, ...] = () + if pipeline.get("id") is not None: + workflows = tuple( + WorkflowRun( + name=job.get("name") or "unnamed", + conclusion=normalise_gitlab_status(job.get("status") or ""), + finished_at=_ts(job.get("finished_at")), + duration=float(job.get("duration") or 0.0), + url=job.get("web_url") or "", + ) + for job in api.pipeline_jobs(full_name, pipeline["id"]) + ) + + # Identical rule to github.collect: the branch is red if ANY job's + # latest run is red, and the run worth showing is that failure rather + # than whichever job happens to have finished most recently. + failing = sorted( + (w for w in workflows if w.conclusion and w.conclusion not in GOOD_CONCLUSIONS), + key=lambda w: w.finished_at, + reverse=True, + ) + rest = sorted(workflows, key=lambda w: w.finished_at, reverse=True) + representative = failing[0] if failing else (rest[0] if rest else None) + + protected, force_push, reviews = _protection(api.protected_branch(full_name, branch)) + + pulls_total, pulls = api.open_merge_requests(full_name) + merged = api.recent_merges(full_name, cfg.recent_merges_per_repo) + # Unlike GitHub's, GitLab's open_issues_count already excludes merge + # requests, so there is nothing to subtract. + open_issues = max(0, int(raw.get("open_issues_count") or 0)) + + return RemoteRepo( + name=raw.get("path") or full_name.rsplit("/", 1)[-1], + owner=full_name.rsplit("/", 1)[0], + forge="gitlab", + url=raw.get("web_url") or "", + pulls_url=f"{raw['web_url']}/-/merge_requests" if raw.get("web_url") else "", + default_branch=branch, + visibility=_visibility(raw), + archived=bool(raw.get("archived")), + head_sha=head_sha, + pushed_at=_ts(raw.get("last_activity_at")), + protected=protected, + required_reviews=reviews, + allows_force_push=force_push, + # No counterpart at most tiers - see the module docstring. + alerts_enabled=False, + alerts=(), + rhiza_managed=bool(ref), + rhiza_ref=ref, + rhiza_behind=_behind_count(tags, ref), + ci_conclusion=representative.conclusion if representative else "", + ci_workflow=representative.name if representative else "", + ci_finished_at=representative.finished_at if representative else 0.0, + ci_duration=representative.duration if representative else 0.0, + ci_url=representative.url if representative else "", + workflows=workflows, + coverage=coverage, + # Lines are not reported by GitLab's coverage field. Zero here means + # the dashboard shows a percentage without a line count rather than + # inventing one. + coverage_lines=0, + coverage_artifact=0, + open_issues=open_issues, + open_pulls_total=pulls_total, + pulls=tuple(pulls), + merged=tuple(merged), + ) + + result: dict[str, RemoteRepo] = {} + with concurrent.futures.ThreadPoolExecutor(max_workers=_MAX_WORKERS) as pool: + futures = {pool.submit(one, raw): raw["path_with_namespace"] for raw in projects} + for future in concurrent.futures.as_completed(futures): + full_name = futures[future] + try: + result[full_name] = future.result() + except Exception as exc: # noqa: BLE001 - one bad repo must not sink the refresh + log.warning("repo %s failed: %s", full_name, exc) + + return result, api, excluded diff --git a/collector/jq_collector/localgit.py b/collector/jq_collector/localgit.py index d922de4..aebd089 100644 --- a/collector/jq_collector/localgit.py +++ b/collector/jq_collector/localgit.py @@ -34,6 +34,7 @@ import subprocess import time +from . import origin from .config import Config from .state import LocalRepo @@ -60,22 +61,17 @@ def _git(path: str, *args: str) -> str | None: return result.stdout.strip() -def origin_owner_name(path: str) -> tuple[str, str] | None: - """The ``(owner, name)`` this clone points at, or None if there is no origin.""" +def origin_full_name(path: str) -> str | None: + """The ``namespace/name`` this clone points at, or None without an origin. + + A thin wrapper over `origin.parse` so the scan keeps using this module's + logging `_git`; the URL grammar itself lives in one place now. + """ url = _git(path, "remote", "get-url", "origin") if not url: return None - url = url.removesuffix(".git") - if url.startswith("git@"): - _, _, tail = url.partition(":") - elif "://" in url: - tail = url.split("://", 1)[1].split("/", 1)[-1] - else: - tail = url - parts = [p for p in tail.split("/") if p] - if len(parts) < 2: - return None - return parts[-2], parts[-1] + parsed = origin.parse(url) + return parsed.full_name if parsed else None def _template_ref(path: str, pointer: str) -> str: @@ -375,12 +371,14 @@ def scan( for key in cfg.repos: if "/" not in key or key in skip: continue - owner, repo_name = key.split("/", 1) + # rsplit, not split: a GitLab namespace nests, so everything before the + # last slash is the owner and only the tail is the repo's own name. + owner, repo_name = key.rsplit("/", 1) path = cfg.repo_paths.get(key) if path is None: if not root: continue - path = os.path.join(root, owner, repo_name) + path = os.path.join(root, *key.split("/")) # `.git` is a directory in a plain checkout and a file in a worktree. if not os.path.exists(os.path.join(path, ".git")): # Not mounted, or mounted somewhere else. Normal for a repo you @@ -391,16 +389,12 @@ def scan( # The mount point claims to be this repo; the origin remote is the only # thing that can confirm it. A wrong path in repos.yml would otherwise # report one repo's dirty files under another repo's name. - origin = origin_owner_name(path) - if origin is not None and (origin[0].lower(), origin[1].lower()) != ( - owner.lower(), - repo_name.lower(), - ): + actual = origin_full_name(path) + if actual is not None and actual.lower() != key.lower(): log.warning( - "%s is a checkout of %s/%s, not %s - check repos.yml", + "%s is a checkout of %s, not %s - check repos.yml", path, - origin[0], - origin[1], + actual, key, ) continue diff --git a/collector/jq_collector/metrics.py b/collector/jq_collector/metrics.py index 946e128..bbad081 100644 --- a/collector/jq_collector/metrics.py +++ b/collector/jq_collector/metrics.py @@ -5,28 +5,37 @@ churn - a merged PR, a deleted repo, a branch that goes away - and a registry of persistent gauges would keep exporting those series forever. -The ``repo`` label is the full ``owner/name``. A bare name is only unique -within one owner, and the dashboard joins frames on this label and builds GitHub -links out of it - both of which need it to be unique on its own. ``owner`` is -carried on ``jq_repo_info`` alone, purely so the dashboard can offer it as a -filter, rather than as a duplicate column on every joined table. +The ``repo`` label is the full ``namespace/name`` - which on GitLab can be more +than two segments deep, since namespaces nest. A bare name is only unique within +one owner, and the dashboard joins frames on this label, so it has to be unique +on its own. ``owner`` is carried on ``jq_repo_info`` alone, purely so the +dashboard can offer it as a filter, rather than as a duplicate column on every +joined table. + +The forge is carried the same way, on ``jq_repo_info`` and nowhere else. That is +deliberate: adding a label to a metric family starts a new series and orphans +the old one, and the thirty families keyed on ``repo`` alone are the history +behind every trend panel and alert rule. Putting the forge on the identity +metric - already an ``Always 1`` join target whose label set churns anyway - +means a fleet that predates GitLab support keeps all of it. + +For the same reason the ``url`` labels carry what the API actually returned +rather than something the dashboard rebuilds: a per-forge URL cannot be built by +pasting the repo label onto a fixed prefix, and never had to be. + +The repo-level one is ``repo_url`` rather than ``url`` so that a panel can pull +it in with ``group_left`` alongside a pull request's or a run's own ``url`` +without the two colliding - which is exactly what the drill-down tables do. """ from __future__ import annotations from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily +from .forge import GOOD_CONCLUSIONS as _GOOD_CONCLUSIONS +from .forge import INCONCLUSIVE_CONCLUSIONS as _INCONCLUSIVE_CONCLUSIONS from .state import Snapshot -# Conclusions that mean "this pipeline did its job". -_GOOD_CONCLUSIONS = {"success", "neutral", "skipped"} - -# Conclusions that are no verdict at all - a cancelled or superseded run. They -# are neither green nor red, so they are left out of the exposition entirely, -# the same way a workflow that never ran is. github.py already drops them when -# it picks each workflow's latest run; this layer owns the exposition contract. -_INCONCLUSIVE_CONCLUSIONS = {"cancelled", "stale"} - def _gauge(name: str, doc: str, labels: list[str] | None = None) -> GaugeMetricFamily: return GaugeMetricFamily(name, doc, labels=labels or []) @@ -51,10 +60,10 @@ def render(snap: Snapshot): "Refreshes that raised, per source.", labels=["source"], ) - for source in ("github", "local"): - health = snap.health.get(source) - if health is None: - continue + # Whatever actually reported, rather than a fixed list: the set of sources + # is now the set of forges in the fleet plus `local`, so a hardcoded tuple + # would silently drop gitlab's health. + for source, health in sorted(snap.health.items()): last_success.add_metric([source], health.last_success) duration.add_metric([source], health.last_duration) errors.add_metric([source], health.errors) @@ -91,7 +100,7 @@ def render(snap: Snapshot): repo_info = _gauge( "jq_repo_info", "Always 1; labels carry the repo's identity for joining.", - ["repo", "owner", "default_branch", "visibility"], + ["repo", "owner", "default_branch", "visibility", "forge", "repo_url", "pulls_url"], ) cloned = _gauge( "jq_repo_cloned", @@ -156,7 +165,7 @@ def render(snap: Snapshot): ci_info = _gauge( "jq_ci_last_run_info", "Always 1; labels carry the last completed default-branch run.", - ["repo", "conclusion", "workflow"], + ["repo", "conclusion", "workflow", "url"], ) ci_ok = _gauge( "jq_ci_last_run_success", @@ -179,6 +188,14 @@ def render(snap: Snapshot): "Per workflow: when that run finished.", ["repo", "workflow"], ) + # Its own family rather than a label on jq_ci_workflow_success, which is a + # real gauge with 180 days behind it - adding a label there would start a + # new series and orphan all of it. + wf_info = _gauge( + "jq_ci_workflow_info", + "Always 1; the label carries the URL of that workflow's latest run.", + ["repo", "workflow", "url"], + ) coverage = _gauge( "jq_ci_coverage_percent", "Line coverage from the newest default-branch coverage-report artifact. " @@ -212,13 +229,22 @@ def render(snap: Snapshot): pr_info = _gauge( "jq_pull_request_info", "Always 1; one series per open pull request.", - ["repo", "number", "title", "author", "checks", "draft"], + ["repo", "number", "title", "author", "checks", "draft", "url"], ) merged_at = _gauge( "jq_merged_pull_request_timestamp_seconds", "Unix time a pull request was merged. topk() over this gives the newest.", ["repo", "number", "title", "author"], ) + # The URL rides on its own family rather than on the timestamp above, + # because that timestamp *is* the merged-PR timeline - the history the + # "Recently merged" panel reads. A label there would have orphaned every + # existing series and, until they went stale, shown each merged PR twice. + merged_info = _gauge( + "jq_merged_pull_request_info", + "Always 1; the label carries the merged pull request's URL.", + ["repo", "number", "url"], + ) pr_created = _gauge( "jq_pull_request_created_timestamp_seconds", "Unix time the pull request was opened.", @@ -311,12 +337,23 @@ def render(snap: Snapshot): for key in keys: remote = snap.remote.get(key) local = snap.local.get(key) - owner = key.partition("/")[0] + # rpartition, not partition: a GitLab namespace nests, so the owner is + # everything before the last slash rather than the first segment. + owner = key.rpartition("/")[0] ident = [key] default_branch = remote.default_branch if remote else "main" repo_info.add_metric( - [key, owner, default_branch, remote.visibility if remote else "unknown"], 1 + [ + key, + owner, + default_branch, + remote.visibility if remote else "unknown", + remote.forge if remote else "github", + remote.url if remote else "", + remote.pulls_url if remote else "", + ], + 1, ) cloned.add_metric(ident, 1 if local else 0) @@ -362,9 +399,12 @@ def render(snap: Snapshot): bad += 0 if good else 1 wf_ok.add_metric([*ident, wf.name], 1 if good else 0) wf_at.add_metric([*ident, wf.name], wf.finished_at) + wf_info.add_metric([*ident, wf.name, wf.url], 1) if remote.ci_conclusion and remote.ci_conclusion not in _INCONCLUSIVE_CONCLUSIONS: - ci_info.add_metric([*ident, remote.ci_conclusion, remote.ci_workflow], 1) + ci_info.add_metric( + [*ident, remote.ci_conclusion, remote.ci_workflow, remote.ci_url], 1 + ) # Green only when no workflow is red. Deriving this from a single # run made a repo look green whenever some other workflow had run # more recently than the failing one. @@ -396,6 +436,7 @@ def render(snap: Snapshot): pull.author, pull.checks, str(pull.draft).lower(), + pull.url, ], 1, ) @@ -411,6 +452,7 @@ def render(snap: Snapshot): continue seen.add(m.number) merged_at.add_metric([*ident, str(m.number), m.title, m.author], m.merged_at) + merged_info.add_metric([*ident, str(m.number), m.url], 1) if local is not None: local_branch.add_metric([*ident, local.branch or "unknown"], 1) @@ -458,6 +500,7 @@ def render(snap: Snapshot): ci_dur, wf_ok, wf_at, + wf_info, wf_failing, coverage, coverage_lines, @@ -467,6 +510,7 @@ def render(snap: Snapshot): pr_info, pr_created, merged_at, + merged_info, local_branch, on_default, dirty, diff --git a/collector/jq_collector/origin.py b/collector/jq_collector/origin.py new file mode 100644 index 0000000..80b2cce --- /dev/null +++ b/collector/jq_collector/origin.py @@ -0,0 +1,120 @@ +"""What a clone's ``origin`` remote says it is. + +There used to be two near-identical copies of this parser - one in ``repos.py`` +to key the fleet, one in ``localgit.py`` to confirm a mount point is the repo it +claims to be - and both threw the host away and kept only the last two path +segments. That was fine while every repo was on GitHub, where ``owner/name`` is +the whole of an identity. It is wrong twice over now: + +- the host is what says which forge a checkout belongs to, and inferring it is + free here where the URL is already in hand; +- GitLab namespaces nest. ``gitlab.com/acme/platform/infra/web`` is the project + ``web`` in the namespace ``acme/platform/infra``, and keeping the last two + segments gives ``infra/web`` - a repo that does not exist, and an API path + that 404s. + +So the namespace is the whole path when there is a host to hang it off. A path +remote - a clone of a clone, ``/srv/mirrors/o/r`` - has no host and no +namespace, only a filesystem trail, so there the last two segments remain the +best available guess. +""" + +from __future__ import annotations + +import logging +import subprocess +from dataclasses import dataclass + +log = logging.getLogger(__name__) + +_TIMEOUT = 20 + +GITHUB = "github" +GITLAB = "gitlab" + + +@dataclass(frozen=True) +class Origin: + """A parsed ``origin`` URL. ``host`` is empty for a path remote.""" + + host: str + namespace: str + name: str + + @property + def full_name(self) -> str: + return f"{self.namespace}/{self.name}" + + @property + def forge(self) -> str: + return forge_for_host(self.host) + + +def forge_for_host(host: str) -> str: + """Which forge a host is. + + GitHub is the default for anything unrecognised, including an empty host. + That keeps a self-hosted GitHub Enterprise and every path remote behaving + exactly as they did before this module existed; a fleet on some other forge + says so with an explicit ``forge:`` in repos.yml rather than relying on a + guess from a hostname. + """ + host = host.lower() + if host == "gitlab.com" or host.startswith("gitlab."): + return GITLAB + return GITHUB + + +def parse(url: str) -> Origin | None: + """Split a git remote URL, or None if it cannot name a repo. + + Handles the four shapes a remote is written in: ``scheme://host/path``, + the scp-like ``[user@]host:path``, and a bare absolute or relative path. + """ + url = url.strip().removesuffix(".git") + if not url: + return None + + if "://" in url: + authority, _, path = url.split("://", 1)[1].partition("/") + elif ":" in url and "/" not in url.partition(":")[0]: + # scp-like. The `/` test is what keeps a Windows-style or + # colon-bearing path from being mistaken for host:path. + authority, _, path = url.partition(":") + else: + authority, path = "", url + + # Strip any `user@` and any `:port` to leave the bare hostname. + host = authority.rpartition("@")[2].partition(":")[0].lower() + + parts = [part for part in path.split("/") if part] + if len(parts) < 2: + # One segment cannot be split into a namespace and a name. + return None + if not host: + # A filesystem trail is not a namespace; only its tail is meaningful. + parts = parts[-2:] + return Origin(host=host, namespace="/".join(parts[:-1]), name=parts[-1]) + + +def read(path: str) -> Origin | None: + """The parsed ``origin`` of the checkout at *path*. + + Read-only and lock-free, like every other git call the collector makes: the + clones are somebody's live working copies and a monitor has no business + taking a lock in one. + """ + try: + result = subprocess.run( + ["git", "--no-optional-locks", "-C", path, "remote", "get-url", "origin"], + capture_output=True, + text=True, + timeout=_TIMEOUT, + check=False, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + log.warning("reading origin of %s failed: %s", path, exc) + return None + if result.returncode != 0: + return None + return parse(result.stdout) diff --git a/collector/jq_collector/repos.py b/collector/jq_collector/repos.py index 3658585..089d465 100644 --- a/collector/jq_collector/repos.py +++ b/collector/jq_collector/repos.py @@ -22,41 +22,19 @@ import logging import os -import subprocess from typing import Any +from . import origin + log = logging.getLogger(__name__) +KNOWN_FORGES = (origin.GITHUB, origin.GITLAB) + class FleetError(Exception): """``repos.yml`` says something the collector cannot act on.""" -def _origin_owner_name(path: str) -> tuple[str, str] | None: - """The ``(owner, name)`` a checkout's origin points at.""" - try: - url = subprocess.run( - ["git", "--no-optional-locks", "-C", path, "remote", "get-url", "origin"], - capture_output=True, - text=True, - timeout=20, - check=False, - ).stdout.strip() - except (OSError, subprocess.TimeoutExpired): - return None - if not url: - return None - url = url.removesuffix(".git") - if url.startswith("git@"): - tail = url.partition(":")[2] - elif "://" in url: - tail = url.split("://", 1)[1].split("/", 1)[-1] - else: - tail = url - parts = [p for p in tail.split("/") if p] - return (parts[-2], parts[-1]) if len(parts) >= 2 else None - - def resolve_path(raw: str, host_root: str) -> str: """Where a ``repos.yml`` path lands on *this* filesystem. @@ -88,8 +66,25 @@ def _is_checkout(path: str) -> bool: return os.path.exists(os.path.join(path, ".git")) -def _entry(item: Any, index: int, host_root: str) -> tuple[str, str | None]: - """One entry -> ``(owner/name, checkout path or None)``.""" +def _declared_forge(item: dict, index: int) -> str | None: + """The entry's explicit ``forge:``, validated, or None if it has none. + + An unknown value is refused rather than defaulted. Silently reading + ``forge: gitbucket`` as GitHub would put a repo on the board with every + remote panel wrong and nothing saying why. + """ + declared = str(item.get("forge") or "").strip().lower() + if not declared: + return None + if declared not in KNOWN_FORGES: + raise FleetError( + f"entry {index}: forge {declared!r} is not one of {', '.join(KNOWN_FORGES)}" + ) + return declared + + +def _entry(item: Any, index: int, host_root: str) -> tuple[str, str | None, str]: + """One entry -> ``(namespace/name, checkout path or None, forge)``.""" # `- ~/repos/foo` is accepted as shorthand for `- path: ~/repos/foo`. if isinstance(item, str): item = {"path": item} @@ -98,6 +93,7 @@ def _entry(item: Any, index: int, host_root: str) -> tuple[str, str | None]: named = str(item.get("repo") or "").strip() raw_path = item.get("path") + declared = _declared_forge(item, index) if raw_path is None: # Name the offending value, not just the rule. "entry 7" in a @@ -108,7 +104,9 @@ def _entry(item: Any, index: int, host_root: str) -> tuple[str, str | None]: ) if "/" not in named: raise FleetError(f"entry {index}: repo {named!r} is not of the form owner/name") - return named, None + # No checkout means no origin to infer from, so an entry on any forge + # but the default has to say so itself. + return named, None, declared or origin.GITHUB path = resolve_path(str(raw_path), host_root) @@ -118,30 +116,37 @@ def _entry(item: Any, index: int, host_root: str) -> tuple[str, str | None]: # and only the working-copy panels go quiet - but say so once, because # a typo in repos.yml looks exactly like this from here. if named and "/" in named: - log.warning("no checkout for %s at %s - GitHub panels only", named, path) - return named, None + log.warning("no checkout for %s at %s - remote panels only", named, path) + return named, None, declared or origin.GITHUB raise FleetError( f"entry {index}: {raw_path} is not a git checkout (looked in {path}). " "Mount the home directory it lives under, or give the entry a " "`repo: owner/name` so it can be monitored without one." ) + parsed = origin.read(path) + if "/" in named: - return named, path + # An explicit `repo:` overrides the origin - that is what it is for, on + # a fork - but the origin's host is still the best evidence of which + # forge the repo lives on when the entry did not say. + return named, path, declared or (parsed.forge if parsed else origin.GITHUB) - origin = _origin_owner_name(path) - if origin is None: + if parsed is None: raise FleetError( f"entry {index}: {path} has no usable origin remote - add an explicit `repo: owner/name`" ) - return f"{origin[0]}/{origin[1]}", path + return parsed.full_name, path, declared or parsed.forge -def load(source: str, host_root: str = "") -> tuple[tuple[str, ...], dict[str, str]]: - """Read ``repos.yml`` into ``(fleet, checkout paths)``. +def load( + source: str, host_root: str = "" +) -> tuple[tuple[str, ...], dict[str, str], dict[str, str]]: + """Read ``repos.yml`` into ``(fleet, checkout paths, forge per repo)``. - The fleet is every listed repo as ``owner/name``; the paths map holds only - those with a checkout this machine can actually read. + The fleet is every listed repo as ``namespace/name``; the paths map holds + only those with a checkout this machine can actually read; the forge map + says which API each one is read through. """ import yaml @@ -162,12 +167,21 @@ def load(source: str, host_root: str = "") -> tuple[tuple[str, ...], dict[str, s fleet: list[str] = [] paths: dict[str, str] = {} + forges: dict[str, str] = {} for index, item in enumerate(entries, start=1): - full_name, path = _entry(item, index, host_root) + full_name, path, forge = _entry(item, index, host_root) if full_name in fleet: - raise FleetError(f"{full_name} is listed twice") + # Two forges can host the same `namespace/name`, and the board's + # whole label scheme is that one `repo` value is one repo. Refusing + # is the same choice the duplicate case has always made: a merged + # pair would report one repo's CI under the other's name, and read + # as a working board while doing it. + clash = forges[full_name] + detail = f" - on {clash} and on {forge}" if clash != forge else "" + raise FleetError(f"{full_name} is listed twice{detail}") fleet.append(full_name) + forges[full_name] = forge if path is not None: paths[full_name] = path - return tuple(fleet), paths + return tuple(fleet), paths, forges diff --git a/collector/jq_collector/state.py b/collector/jq_collector/state.py index 713e7cf..18d2079 100644 --- a/collector/jq_collector/state.py +++ b/collector/jq_collector/state.py @@ -47,16 +47,36 @@ class MergedPull: @dataclass(frozen=True) class RemoteRepo: - """What GitHub says about a repo.""" + """What the forge says about a repo. + + Nothing in this shape is GitHub-specific, which is why adding GitLab needed + a second collector rather than a second record: both forges answer the same + questions, in different words, at different URLs. + """ name: str owner: str = "" + # Which forge answered. GitHub by default, so a snapshot built before this + # field existed reads correctly. + forge: str = "github" default_branch: str = "main" visibility: str = "unknown" archived: bool = False head_sha: str = "" pushed_at: float = 0.0 + # The repo's page, as the forge itself reported it. The dashboard used to + # rebuild these by pasting the repo label onto `https://github.com/`, which + # no longer works once a repo can live somewhere else - and never needed to, + # since every API hands the URL over. + url: str = "" + + # The repo's pull-request listing. Built rather than reported, because no + # API hands this one over - and built here, in the collector that already + # knows the forge, rather than in a dashboard that would have to learn that + # GitHub spells it /pulls and GitLab /-/merge_requests. + pulls_url: str = "" + # Default-branch protection. None means "GitHub would not tell us" - the # protection endpoint needs admin on the repo, and a token without it gets # the same 404 as a genuinely unprotected branch. Reporting that as diff --git a/collector/tests/conftest.py b/collector/tests/conftest.py index ab693db..da33e11 100644 --- a/collector/tests/conftest.py +++ b/collector/tests/conftest.py @@ -6,6 +6,7 @@ from jq_collector.config import Config from jq_collector.github import GitHub +from jq_collector.gitlab import GitLab class FakeGitHub(GitHub): @@ -38,6 +39,34 @@ def _bytes(self, path: str) -> bytes | None: return self.blobs.get(path) +class FakeGitLab(GitLab): + """A GitLab client whose HTTP layer is a dict of canned responses. + + The GitHub twin above is faked at ``_json``; this one has to fake ``_text`` + as well, because the template pointer is served by a raw-file route that + answers with YAML rather than JSON. + """ + + def __init__( + self, + cfg: Config, + responses: dict[str, object], + texts: dict[str, str] | None = None, + ) -> None: + super().__init__(cfg) + self.responses = responses + self.texts = texts or {} + self.calls: list[str] = [] + + def _json(self, path: str, **params: object) -> object | None: + self.calls.append(path) + return self.responses.get(path) + + def _text(self, path: str, **params: object) -> str | None: + self.calls.append(path) + return self.texts.get(path) + + @pytest.fixture def cfg() -> Config: return Config() @@ -51,6 +80,44 @@ def _make(responses: dict[str, object], blobs: dict[str, bytes] | None = None) - return _make +@pytest.fixture +def make_gitlab(cfg): + def _make(responses: dict[str, object], texts: dict[str, str] | None = None) -> FakeGitLab: + return FakeGitLab(cfg, responses, texts) + + return _make + + +def job(name: str, status: str, finished: str | None = None, duration: float = 1.0): + """A minimal GitLab pipeline-job payload.""" + return { + "name": name, + "status": status, + "finished_at": finished or "2026-08-31T06:05:36.000Z", + "duration": duration, + "web_url": f"https://gitlab.com/o/r/-/jobs/{name}", + } + + +def project(full_name: str = "o/r", **overrides): + """A minimal GitLab project payload.""" + namespace, _, name = full_name.rpartition("/") + base = { + "path_with_namespace": full_name, + "path": name, + "name": name, + "default_branch": "main", + "visibility": "public", + "archived": False, + "open_issues_count": 0, + "web_url": f"https://gitlab.com/{full_name}", + "last_activity_at": "2026-08-31T06:05:36.000Z", + "namespace": {"full_path": namespace}, + } + base.update(overrides) + return base + + def run(workflow_id: int, name: str, conclusion: str, updated: str, started: str | None = None): """A minimal workflow_run payload.""" return { diff --git a/collector/tests/test_config.py b/collector/tests/test_config.py index 8d3d7de..f68b855 100644 --- a/collector/tests/test_config.py +++ b/collector/tests/test_config.py @@ -153,3 +153,94 @@ def test_is_ignored_accepts_a_bare_name_or_owner_slash_name(monkeypatch, owner, monkeypatch.setenv("JQ_IGNORE", "bare,o/full") assert Config().is_ignored(owner, name) is ignored + + +# -- which forge each repo is read through ----------------------------------- + + +def test_an_unlisted_repo_is_github(monkeypatch): + """What keeps a fleet that predates GitLab support working untouched.""" + cfg = Config() + object.__setattr__(cfg, "repos", ("o/r",)) + object.__setattr__(cfg, "forges", {}) + + assert cfg.forge_for("o/r") == "github" + + +def test_the_fleet_groups_by_forge(monkeypatch): + """Each API is asked only for its own share; asking GitHub about a GitLab + repo answers 404 and logs it as unreadable.""" + cfg = Config() + object.__setattr__(cfg, "repos", ("o/gh", "acme/platform/web", "o/gh2")) + object.__setattr__(cfg, "forges", {"acme/platform/web": "gitlab"}) + + assert cfg.repos_by_forge() == { + "github": ("o/gh", "o/gh2"), + "gitlab": ("acme/platform/web",), + } + + +def test_a_github_only_fleet_lists_no_gitlab_at_all(monkeypatch): + """Absence, not an empty tuple: it is what stops a GitHub-only deployment + building a GitLab client or warning about a token it has no use for.""" + cfg = Config() + object.__setattr__(cfg, "repos", ("o/r",)) + object.__setattr__(cfg, "forges", {}) + + assert "gitlab" not in cfg.repos_by_forge() + + +def test_forges_can_be_set_from_the_environment(monkeypatch): + """For a deployment with no repos.yml to mount - a server, or CI.""" + monkeypatch.setenv("JQ_REPO_FORGES", "acme/web=gitlab, o/r=github") + + assert Config().forges == {"acme/web": "gitlab", "o/r": "github"} + + +@pytest.mark.parametrize("raw", ["acme/web", "acme/web=", "=gitlab"]) +def test_a_malformed_forge_pair_refuses_to_start(monkeypatch, raw): + monkeypatch.setenv("JQ_REPO_FORGES", raw) + + with pytest.raises(ValueError, match="owner/name=forge"): + Config() + + +def test_an_unknown_forge_in_the_environment_refuses_to_start(monkeypatch): + monkeypatch.setenv("JQ_REPO_FORGES", "acme/web=gitea") + + with pytest.raises(ValueError, match="not one of"): + Config() + + +def test_the_gitlab_api_defaults_to_gitlab_com(monkeypatch): + monkeypatch.delenv("GITLAB_API", raising=False) + + assert Config().gitlab_api == "https://gitlab.com/api/v4" + + +def test_a_github_only_fleet_logs_exactly_what_it_always_did(tmp_path, caplog): + """CI greps this line, and so does anybody reading the logs. Adding a clause + to it would have broken both for a fleet that has not changed.""" + source = tmp_path / "repos.yml" + source.write_text("repos:\n - repo: o/r\n - repo: o/r2\n") + + with caplog.at_level("INFO"): + cfg = Config() + object.__setattr__(cfg, "repos_file", str(source)) + cfg.__post_init__() + + assert "fleet: 2 repos, 0 with a checkout" in caplog.text + assert "forges:" not in caplog.text + + +def test_a_mixed_fleet_says_which_forge_each_repo_is_on(tmp_path, caplog): + source = tmp_path / "repos.yml" + source.write_text("repos:\n - repo: o/r\n - repo: acme/web\n forge: gitlab\n") + + with caplog.at_level("INFO"): + cfg = Config() + object.__setattr__(cfg, "repos_file", str(source)) + cfg.__post_init__() + + assert "fleet: 2 repos, 0 with a checkout" in caplog.text + assert "forges: 1 on github, 1 on gitlab" in caplog.text diff --git a/collector/tests/test_coverage.py b/collector/tests/test_coverage.py index fd3af5a..c56437f 100644 --- a/collector/tests/test_coverage.py +++ b/collector/tests/test_coverage.py @@ -145,7 +145,7 @@ def __init__(self, artifact_id: int = 42) -> None: self.artifact_id = artifact_id self.downloads: list[str] = [] - def list_repos(self): + def list_repos(self, _fleet=None): return [ { "full_name": "o/r", diff --git a/collector/tests/test_fleet.py b/collector/tests/test_fleet.py index ad8ab3a..2896f7a 100644 --- a/collector/tests/test_fleet.py +++ b/collector/tests/test_fleet.py @@ -20,6 +20,7 @@ import pytest from jq_collector import localgit, repos +from jq_collector import origin as origin_module from jq_collector.config import Config REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] @@ -389,10 +390,13 @@ def write_fleet(tmp_path: pathlib.Path, body: str) -> str: def test_a_checkout_is_named_by_its_origin(tmp_path): path = make_checkout(tmp_path, "cvxgrp", "cvxsimulator") - fleet, paths = repos.load(write_fleet(tmp_path, f" - path: {path}\n")) + fleet, paths, forges = repos.load(write_fleet(tmp_path, f" - path: {path}\n")) assert fleet == ("cvxgrp/cvxsimulator",) assert paths == {"cvxgrp/cvxsimulator": str(path)} + # make_checkout gives the clone a github.com origin, so the forge follows + # from the host without the entry having to say. + assert forges == {"cvxgrp/cvxsimulator": "github"} def test_an_explicit_repo_overrides_the_origin(tmp_path): @@ -403,14 +407,17 @@ def test_an_explicit_repo_overrides_the_origin(tmp_path): assert repos.load(write_fleet(tmp_path, body)) == ( ("cvxpy/cvxpy",), {"cvxpy/cvxpy": str(path)}, + {"cvxpy/cvxpy": "github"}, ) def test_an_entry_may_name_a_repo_with_no_checkout(tmp_path): - fleet, paths = repos.load(write_fleet(tmp_path, " - repo: Jebel-Quant/actions\n")) + fleet, paths, forges = repos.load(write_fleet(tmp_path, " - repo: Jebel-Quant/actions\n")) assert fleet == ("Jebel-Quant/actions",) assert paths == {} + # No checkout means no origin to infer from, so it falls to the default. + assert forges == {"Jebel-Quant/actions": "github"} def test_a_bare_string_entry_is_a_path(tmp_path): @@ -429,7 +436,7 @@ def test_a_named_repo_whose_checkout_is_missing_keeps_its_github_panels(tmp_path body = " - path: ~/nowhere/rhiza\n repo: Jebel-Quant/rhiza\n" with caplog.at_level("WARNING"): - fleet, paths = repos.load(write_fleet(tmp_path, body)) + fleet, paths, _forges = repos.load(write_fleet(tmp_path, body)) assert fleet == ("Jebel-Quant/rhiza",) assert paths == {} @@ -601,9 +608,109 @@ def test_an_origin_that_names_no_owner_is_refused(tmp_path, origin): def test_git_being_unrunnable_is_refused_not_guessed_at(tmp_path, monkeypatch): """Same answer as a missing remote: the collector will not invent a name.""" path = make_checkout(tmp_path, "Jebel-Quant", "rhiza") + # The git call lives in `origin` now, which is what both repos.py and + # localgit.py read a remote through. monkeypatch.setattr( - repos.subprocess, "run", lambda *_a, **_k: (_ for _ in ()).throw(OSError("no git")) + origin_module.subprocess, "run", lambda *_a, **_k: (_ for _ in ()).throw(OSError("no git")) ) with pytest.raises(repos.FleetError, match="origin"): repos.load(write_fleet(tmp_path, f" - path: {path}\n")) + + +# -- which forge each entry belongs to --------------------------------------- +# +# The forge is what decides which API a repo is read through, so getting it +# wrong is not a crash but a repo on the board with every remote panel blank. +# It is inferred from the origin host where there is a checkout, and stated +# outright where there is not. + + +def test_the_forge_is_inferred_from_a_checkouts_origin_host(tmp_path): + """Free, because the origin URL is parsed for the repo's name anyway.""" + path = make_checkout(tmp_path, "acme", "web", origin="git@gitlab.com:acme/web.git") + + _fleet, _paths, forges = repos.load(write_fleet(tmp_path, f" - path: {path}\n")) + + assert forges == {"acme/web": "gitlab"} + + +def test_a_gitlab_subgroup_keeps_its_whole_namespace(tmp_path): + """The old parser kept the last two segments, which named a project that + does not exist and an API path that 404s.""" + path = make_checkout( + tmp_path, "x", "y", origin="https://gitlab.com/acme/platform/infra/web.git" + ) + + fleet, paths, forges = repos.load(write_fleet(tmp_path, f" - path: {path}\n")) + + assert fleet == ("acme/platform/infra/web",) + assert paths == {"acme/platform/infra/web": str(path)} + assert forges == {"acme/platform/infra/web": "gitlab"} + + +def test_an_explicit_forge_wins_over_the_origin(tmp_path): + """For a checkout whose origin is a mirror on the other forge.""" + path = make_checkout(tmp_path, "acme", "web", origin="git@github.com:acme/web.git") + body = f" - path: {path}\n forge: gitlab\n" + + _fleet, _paths, forges = repos.load(write_fleet(tmp_path, body)) + + assert forges == {"acme/web": "gitlab"} + + +def test_an_entry_with_no_checkout_states_its_forge(tmp_path): + """No origin to infer from, so anything but the default has to say.""" + body = " - repo: acme/platform/infra/web\n forge: gitlab\n" + + fleet, paths, forges = repos.load(write_fleet(tmp_path, body)) + + assert fleet == ("acme/platform/infra/web",) + assert paths == {} + assert forges == {"acme/platform/infra/web": "gitlab"} + + +def test_an_explicit_repo_still_takes_its_forge_from_the_origin(tmp_path): + """`repo:` overrides the name - that is what it is for, on a fork - but the + origin's host is still the best evidence of where the repo lives.""" + path = make_checkout(tmp_path, "me", "web", origin="git@gitlab.com:me/web.git") + body = f" - path: {path}\n repo: acme/web\n" + + _fleet, _paths, forges = repos.load(write_fleet(tmp_path, body)) + + assert forges == {"acme/web": "gitlab"} + + +@pytest.mark.parametrize("declared", ["gitbucket", "GitHub Enterprise", "gitea"]) +def test_a_forge_nobody_implements_is_refused(tmp_path, declared): + """Reading an unknown name as GitHub would put the repo on the board with + every remote panel wrong and nothing saying why.""" + body = f" - repo: acme/web\n forge: {declared}\n" + + with pytest.raises(repos.FleetError, match="forge"): + repos.load(write_fleet(tmp_path, body)) + + +@pytest.mark.parametrize("declared", ["gitlab", "GitLab", " GITLAB "]) +def test_a_declared_forge_is_case_and_space_insensitive(tmp_path, declared): + body = f" - repo: acme/web\n forge: '{declared}'\n" + + assert repos.load(write_fleet(tmp_path, body))[2] == {"acme/web": "gitlab"} + + +def test_the_same_path_on_two_forges_is_refused(tmp_path): + """One `repo` label is one repo, and the board's whole label scheme rests on + that. A merged pair would report one repo's CI under the other's name and + read as a working board while doing it.""" + body = " - repo: acme/web\n - repo: acme/web\n forge: gitlab\n" + + with pytest.raises(repos.FleetError, match="on github and on gitlab"): + repos.load(write_fleet(tmp_path, body)) + + +def test_the_duplicate_message_stays_plain_when_the_forge_matches(tmp_path): + """Naming a forge here would suggest the forge was the problem.""" + body = " - repo: acme/web\n - repo: acme/web\n" + + with pytest.raises(repos.FleetError, match="acme/web is listed twice$"): + repos.load(write_fleet(tmp_path, body)) diff --git a/collector/tests/test_github_http.py b/collector/tests/test_github_http.py index 419fdcf..bcef9b4 100644 --- a/collector/tests/test_github_http.py +++ b/collector/tests/test_github_http.py @@ -644,7 +644,7 @@ def branch_sha(self, full_name, *_): raise RuntimeError("this repo is cursed") return "sha" - def list_repos(self): + def list_repos(self, _fleet=None): return [ { "full_name": f"o/{n}", diff --git a/collector/tests/test_gitlab.py b/collector/tests/test_gitlab.py new file mode 100644 index 0000000..5ec716a --- /dev/null +++ b/collector/tests/test_gitlab.py @@ -0,0 +1,702 @@ +"""What the GitLab collector derives from a payload. + +Faked at `_json`/`_text`, so this drives the derivation rather than the HTTP +layer - test_gitlab_http.py does that half. The interesting cases are the ones +where GitLab's answer differs in shape from GitHub's and the board's contract has +to be preserved anyway: a pipeline of jobs standing in for N workflows, a status +vocabulary that spells `failed` where the history says `failure`, and a +nested namespace that must survive being put in a URL. +""" + +from __future__ import annotations + +import pytest +from conftest import job, project + +from jq_collector import gitlab +from jq_collector.config import Config +from jq_collector.forge import normalise_gitlab_status + + +def config(**overrides) -> Config: + cfg = Config() + for key, value in overrides.items(): + object.__setattr__(cfg, key, value) + return cfg + + +# -- the project path, which becomes a URL segment -------------------------- +@pytest.mark.parametrize( + ("full_name", "encoded"), + [ + ("o/r", "o%2Fr"), + # The case the last-two-segments parser used to get wrong. + ("acme/platform/infra/web", "acme%2Fplatform%2Finfra%2Fweb"), + # A dot is legal in a project path and must not be left to be read as a + # path traversal or an extension. + ("o/r.js", "o%2Fr.js"), + ], +) +def test_a_project_path_is_url_encoded_whole(full_name, encoded): + assert gitlab._pid(full_name) == encoded + + +def test_a_nested_namespace_is_asked_for_by_its_whole_path(make_gitlab): + client = make_gitlab( + {"/projects/acme%2Fplatform%2Finfra%2Fweb": project("acme/platform/infra/web")} + ) + + found = client.list_projects(("acme/platform/infra/web",)) + + assert [p["path_with_namespace"] for p in found] == ["acme/platform/infra/web"] + + +def test_a_project_that_cannot_be_read_is_dropped_with_a_warning(make_gitlab, caplog): + """One bad line in repos.yml must not blank the whole board.""" + client = make_gitlab({"/projects/o%2Fgood": project("o/good")}) + + with caplog.at_level("WARNING"): + found = client.list_projects(("o/good", "o/missing")) + + assert [p["path_with_namespace"] for p in found] == ["o/good"] + assert "o/missing is not readable on GitLab" in caplog.text + + +def test_only_this_forges_share_is_asked_for(make_gitlab): + """A GitHub repo in a mixed fleet must not be asked of GitLab, which would + answer 404 and log it as unreadable.""" + client = make_gitlab({"/projects/o%2Fr": project()}) + + client.list_projects(("o/r",)) + + assert client.calls == ["/projects/o%2Fr"] + + +# -- timestamps ------------------------------------------------------------- +@pytest.mark.parametrize( + ("value", "expected_year"), + [("2026-08-31T06:05:36.000Z", 2026), ("2026-08-31T06:05:36Z", 2026)], +) +def test_gitlab_timestamps_with_a_z_and_milliseconds_parse(value, expected_year): + import datetime + + assert datetime.datetime.fromtimestamp(gitlab._ts(value), datetime.UTC).year == expected_year + + +@pytest.mark.parametrize("value", [None, "", "not a date"]) +def test_an_unusable_timestamp_is_zero_not_a_crash(value): + assert gitlab._ts(value) == 0.0 + + +# -- the status vocabulary -------------------------------------------------- +@pytest.mark.parametrize( + ("gitlab_status", "conclusion"), + [ + ("success", "success"), + # GitLab spells it `failed`; the stored history and every dashboard + # value mapping say `failure`. + ("failed", "failure"), + # One l, unlike GitHub's `cancelled`. + ("canceled", "cancelled"), + ("skipped", "skipped"), + # A held deploy gate is not a failure. Counting it as red would make a + # correctly configured pipeline permanently angry. + ("manual", "cancelled"), + # In flight: the board reports the last *completed* state, so a running + # pipeline must not overwrite the verdict of the one before it. + ("running", "stale"), + ("pending", "stale"), + # GitLab has added states before and will again. A new one appearing as + # a fleet-wide red is worse than it appearing as "no verdict yet". + ("some_future_state", "stale"), + ("", "stale"), + ], +) +def test_gitlab_statuses_are_normalised_to_the_boards_vocabulary(gitlab_status, conclusion): + assert normalise_gitlab_status(gitlab_status) == conclusion + + +@pytest.mark.parametrize( + ("status", "checks"), + [ + ("success", "success"), + ("failed", "failure"), + ("running", "pending"), + ("created", "pending"), + ("canceled", "cancelled"), + ("manual", "cancelled"), + (None, "none"), + ], +) +def test_an_mrs_pipeline_status_becomes_the_boards_check_state(status, checks): + """The PR table's vocabulary is GitHub's check-run summary, where in-flight + is `pending` - not the `stale` a *completed* run would normalise to.""" + assert gitlab._checks_state(status) == checks + + +# -- branch protection ------------------------------------------------------ +def test_an_unprotected_branch_is_false_not_unknown(): + """GitHub needs a third state because its endpoint 404s both for an + unprotected branch and for a token without admin. GitLab's needs no special + rights, so a 404 here is unambiguous and `None` would understate it.""" + protected, force, reviews = gitlab._protection(None) + + assert protected is False + assert force is False + assert reviews == 0 + + +def test_a_protected_branch_reports_its_force_push_setting(): + protected, force, reviews = gitlab._protection({"allow_force_push": True}) + + assert protected is True + assert force is True + # MR approval rules are a Premium feature, so there is nothing to report. + assert reviews == 0 + + +# -- template drift --------------------------------------------------------- +def test_drift_is_measured_against_the_github_template_tags(): + """The template lives on GitHub whichever forge pins it, so a GitLab repo's + drift is the same count against the same list.""" + tags = ["v2.0.0", "v1.9.0", "v1.8.0"] + + assert gitlab._behind_count(tags, "v2.0.0") == 0 + assert gitlab._behind_count(tags, "v1.8.0") == 2 + + +@pytest.mark.parametrize("ref", ["", "main", "deadbeef", "v9.9.9"]) +def test_a_ref_that_is_not_a_release_is_unknown_not_zero(ref): + """Zero would read as "up to date", which is the opposite of "we cannot + tell" for a repo pinned to a branch or a sha.""" + assert gitlab._behind_count(["v2.0.0", "v1.9.0"], ref) is None + + +def test_the_template_pointer_is_read_as_yaml_from_the_raw_route(make_gitlab): + client = make_gitlab( + {}, + texts={"/projects/o%2Fr/repository/files/.rhiza%2Ftemplate.yml/raw": "ref: v1.7.1\n"}, + ) + + assert client.template_ref("o/r", "main") == "v1.7.1" + + +def test_a_malformed_pointer_is_data_not_a_crash(make_gitlab, caplog): + """A hand-edited pointer is one repo's problem and must not take the rest of + the refresh down with it.""" + client = make_gitlab( + {}, + texts={"/projects/o%2Fr/repository/files/.rhiza%2Ftemplate.yml/raw": "ref: [unclosed"}, + ) + + with caplog.at_level("WARNING"): + assert client.template_ref("o/r", "main") == "" + + assert "malformed template pointer" in caplog.text + + +def test_a_repo_with_no_pointer_is_simply_unmanaged(make_gitlab): + client = make_gitlab({}, texts={}) + + assert client.template_ref("o/r", "main") == "" + + +# -- merge requests --------------------------------------------------------- +def test_an_mrs_checks_cost_a_follow_up_call(make_gitlab): + """GitLab documents `head_pipeline` on the merge-request response, but only + the *single* MR endpoint returns it - the listing omits the key entirely. + Reading that absence as "no pipeline" reported every GitLab MR as + unchecked, which is why this looks like GitHub's per-PR call.""" + client = make_gitlab( + { + "/projects/o%2Fr/merge_requests": [ + { + "iid": 7, + "title": "Fix the thing", + "author": {"username": "someone"}, + "draft": False, + "created_at": "2026-08-30T10:00:00.000Z", + "updated_at": "2026-08-31T10:00:00.000Z", + "web_url": "https://gitlab.com/o/r/-/merge_requests/7", + # Deliberately present here and ignored: the real listing + # does not carry it, so trusting it would hide the bug. + "head_pipeline": {"status": "success"}, + } + ], + "/projects/o%2Fr/merge_requests/7": {"head_pipeline": {"status": "failed"}}, + } + ) + + total, pulls = client.open_merge_requests("o/r") + + assert total == 1 + assert pulls[0].number == 7, "iid is the number the UI and the URL use, not id" + assert pulls[0].checks == "failure", "the listing's stale copy was trusted" + assert pulls[0].url == "https://gitlab.com/o/r/-/merge_requests/7" + assert client.calls == [ + "/projects/o%2Fr/merge_requests", + "/projects/o%2Fr/merge_requests/7", + ] + + +def test_an_mr_the_api_will_not_describe_has_no_checks(make_gitlab): + """The follow-up 404s - a token that can list but not read one MR.""" + client = make_gitlab( + {"/projects/o%2Fr/merge_requests": [{"iid": 1, "author": {}, "title": "x"}]} + ) + + _total, pulls = client.open_merge_requests("o/r") + + assert pulls[0].checks == "none" + + +def test_an_mr_with_no_pipeline_at_all_has_no_checks(make_gitlab): + """A brand-new MR on a project with no CI, or before the first run.""" + client = make_gitlab( + { + "/projects/o%2Fr/merge_requests": [{"iid": 1, "author": {}, "title": "x"}], + "/projects/o%2Fr/merge_requests/1": {"iid": 1, "head_pipeline": None}, + } + ) + + _total, pulls = client.open_merge_requests("o/r") + + assert pulls[0].checks == "none" + + +def test_an_mr_with_no_iid_is_not_looked_up(make_gitlab): + """A malformed entry must not send a request to `/merge_requests/0`.""" + client = make_gitlab({"/projects/o%2Fr/merge_requests": [{"author": {}, "title": "x"}]}) + + _total, pulls = client.open_merge_requests("o/r") + + assert pulls[0].checks == "none" + assert client.calls == ["/projects/o%2Fr/merge_requests"] + + +def test_a_long_mr_title_is_clipped_like_githubs(make_gitlab): + """One essay of a title must not stretch the table's column past the rest.""" + client = make_gitlab( + {"/projects/o%2Fr/merge_requests": [{"iid": 1, "author": {}, "title": "x" * 300}]} + ) + + _total, pulls = client.open_merge_requests("o/r") + + assert len(pulls[0].title) == 120 + + +def test_an_mr_with_no_author_is_unknown_not_blank(make_gitlab): + client = make_gitlab({"/projects/o%2Fr/merge_requests": [{"iid": 1, "title": "x"}]}) + + _total, pulls = client.open_merge_requests("o/r") + + assert pulls[0].author == "unknown" + + +def test_open_mrs_are_capped_but_the_total_is_not(make_gitlab, cfg): + """The tile must stay right even on a repo with an MR flood.""" + object.__setattr__(cfg, "max_prs_per_repo", 2) + client = make_gitlab( + { + "/projects/o%2Fr/merge_requests": [ + {"iid": n, "author": {}, "title": f"mr {n}"} for n in range(5) + ] + } + ) + + total, pulls = client.open_merge_requests("o/r") + + assert total == 5 + assert len(pulls) == 2 + + +def test_an_unmerged_entry_is_left_out_of_the_merged_list(make_gitlab): + """`merged_at` is what makes a MergedPull immutable; without one there is + nothing to place it on the timeline with.""" + client = make_gitlab( + { + "/projects/o%2Fr/merge_requests": [ + {"iid": 1, "author": {}, "title": "landed", "merged_at": "2026-08-31T06:00:00Z"}, + {"iid": 2, "author": {}, "title": "closed unmerged", "merged_at": None}, + ] + } + ) + + merged = client.recent_merges("o/r", 10) + + assert [m.number for m in merged] == [1] + + +# -- collect(): the whole derivation --------------------------------------- +def _stub(monkeypatch, **methods): + """A GitLab whose methods are canned, installed over the constructor. + + A callable value is used as the method itself, so a test can count calls or + raise; anything else becomes a method returning it. The default argument is + what binds each value at definition time - a bare closure over the loop + variable would give every method the last one. + """ + + class Stub: + def close(self): + pass + + for name, value in methods.items(): + if callable(value): + setattr(Stub, name, value) + else: + setattr(Stub, name, lambda _s, *_a, _v=value, **_k: _v) + monkeypatch.setattr(gitlab, "GitLab", lambda _cfg: Stub()) + return Stub + + +def test_one_job_becomes_one_workflow_row(monkeypatch): + """A GitLab pipeline is one run of many jobs where GitHub has many + independent workflows. The job is what is individually red and individually + worth naming, so it takes the workflow's place on the board.""" + _stub( + monkeypatch, + list_projects=[project()], + branch_sha="sha", + template_ref="", + latest_pipeline={"id": 5, "status": "failed", "coverage": None}, + pipeline_jobs=[ + job("lint", "failed", finished="2026-08-31T06:00:00Z"), + job("test", "success", finished="2026-08-31T06:01:00Z"), + ], + protected_branch=None, + open_merge_requests=(0, []), + recent_merges=[], + ) + + remote, _api, _excluded = gitlab.collect(config(), {}, {}, fleet=("o/r",)) + + repo = remote["o/r"] + assert {w.name for w in repo.workflows} == {"lint", "test"} + # The representative run is the failure, not whichever job finished last - + # the same rule the GitHub collector applies. + assert repo.ci_workflow == "lint" + assert repo.ci_conclusion == "failure" + + +def test_a_green_pipeline_shows_its_most_recent_job(monkeypatch): + _stub( + monkeypatch, + list_projects=[project()], + branch_sha="sha", + template_ref="", + latest_pipeline={"id": 5, "status": "success", "coverage": "87.3"}, + pipeline_jobs=[ + job("build", "success", finished="2026-08-31T06:00:00Z"), + job("test", "success", finished="2026-08-31T06:02:00Z"), + ], + protected_branch=None, + open_merge_requests=(0, []), + recent_merges=[], + ) + + remote, _api, _excluded = gitlab.collect(config(), {}, {}, fleet=("o/r",)) + + assert remote["o/r"].ci_workflow == "test" + assert remote["o/r"].ci_conclusion == "success" + + +def test_coverage_is_a_field_not_a_download(monkeypatch): + """GitLab reports it as a string percentage on the pipeline, so there is no + artifact to list and no zip to fetch.""" + _stub( + monkeypatch, + list_projects=[project()], + branch_sha="sha", + template_ref="", + latest_pipeline={"id": 5, "status": "success", "coverage": "87.34"}, + pipeline_jobs=[], + protected_branch=None, + open_merge_requests=(0, []), + recent_merges=[], + ) + + remote, _api, _excluded = gitlab.collect(config(), {}, {}, fleet=("o/r",)) + + assert remote["o/r"].coverage == pytest.approx(87.34) + # Lines are not reported, and zero says so rather than inventing a count. + assert remote["o/r"].coverage_lines == 0 + assert remote["o/r"].coverage_artifact == 0 + + +@pytest.mark.parametrize("raw", [None, "", "not a number"]) +def test_a_project_with_no_coverage_regex_reports_none(monkeypatch, raw): + """None means "publishes no coverage", which is not the same as zero.""" + _stub( + monkeypatch, + list_projects=[project()], + branch_sha="sha", + template_ref="", + latest_pipeline={"id": 5, "status": "success", "coverage": raw}, + pipeline_jobs=[], + protected_branch=None, + open_merge_requests=(0, []), + recent_merges=[], + ) + + remote, _api, _excluded = gitlab.collect(config(), {}, {}, fleet=("o/r",)) + + assert remote["o/r"].coverage is None + + +def test_a_repo_with_no_pipeline_at_all_has_no_ci(monkeypatch): + _stub( + monkeypatch, + list_projects=[project()], + branch_sha="sha", + template_ref="", + latest_pipeline=None, + pipeline_jobs=[], + protected_branch=None, + open_merge_requests=(0, []), + recent_merges=[], + ) + + remote, _api, _excluded = gitlab.collect(config(), {}, {}, fleet=("o/r",)) + + assert remote["o/r"].workflows == () + assert remote["o/r"].ci_conclusion == "" + + +def test_the_forge_and_url_come_from_the_project(monkeypatch): + """The dashboard reads these instead of pasting the repo label onto a fixed + github.com prefix, which is what makes a link correct on either forge.""" + _stub( + monkeypatch, + list_projects=[project("acme/platform/infra/web")], + branch_sha="sha", + template_ref="", + latest_pipeline=None, + pipeline_jobs=[], + protected_branch=None, + open_merge_requests=(0, []), + recent_merges=[], + ) + + remote, _api, _excluded = gitlab.collect(config(), {}, {}, fleet=("acme/platform/infra/web",)) + + repo = remote["acme/platform/infra/web"] + assert repo.forge == "gitlab" + assert repo.url == "https://gitlab.com/acme/platform/infra/web" + # The owner is the whole nested namespace, not its first segment. + assert repo.owner == "acme/platform/infra" + assert repo.name == "web" + + +def test_dependabot_has_no_counterpart_and_says_so(monkeypatch): + """GitLab's equivalent is Ultimate-tier. `alerts_enabled=False` renders as + unknown rather than as a green "zero open alerts" tile - the distinction + RemoteRepo keeps precisely so an unscanned repo is not mistaken for a clean + one.""" + _stub( + monkeypatch, + list_projects=[project()], + branch_sha="sha", + template_ref="", + latest_pipeline=None, + pipeline_jobs=[], + protected_branch=None, + open_merge_requests=(0, []), + recent_merges=[], + ) + + remote, _api, _excluded = gitlab.collect(config(), {}, {}, fleet=("o/r",)) + + assert remote["o/r"].alerts_enabled is False + assert remote["o/r"].alerts == () + + +def test_open_issues_are_not_double_counted(monkeypatch): + """GitLab's open_issues_count already excludes merge requests, unlike + GitHub's, so there is nothing to subtract.""" + _stub( + monkeypatch, + list_projects=[project(open_issues_count=4)], + branch_sha="sha", + template_ref="", + latest_pipeline=None, + pipeline_jobs=[], + protected_branch=None, + open_merge_requests=(3, []), + recent_merges=[], + ) + + remote, _api, _excluded = gitlab.collect(config(), {}, {}, fleet=("o/r",)) + + assert remote["o/r"].open_issues == 4 + + +def test_an_unchanged_branch_head_skips_the_pointer_read(monkeypatch): + """The steady-state cost of drift correctness is zero extra calls, exactly + as on the GitHub side.""" + reads: list[str] = [] + + def counting_template_ref(_self, full_name, _branch): + reads.append(full_name) + return "v1.7.1" + + _stub( + monkeypatch, + list_projects=[project()], + branch_sha="sha", + template_ref=counting_template_ref, + latest_pipeline=None, + pipeline_jobs=[], + protected_branch=None, + open_merge_requests=(0, []), + recent_merges=[], + ) + + cold, *_ = gitlab.collect(config(), {}, {}, fleet=("o/r",)) + assert reads == ["o/r"] + assert cold["o/r"].rhiza_ref == "v1.7.1" + + warm, *_ = gitlab.collect(config(), {"o/r": ("sha", "v1.7.1")}, {}, fleet=("o/r",)) + assert reads == ["o/r"], "an unchanged head still cost a pointer read" + assert warm["o/r"].rhiza_ref == "v1.7.1" + + +def test_one_failing_repo_does_not_sink_the_refresh(monkeypatch, caplog): + def cursed_sha(_self, full_name, *_a): + if full_name == "o/bad": + raise RuntimeError("this repo is cursed") + return "sha" + + _stub( + monkeypatch, + list_projects=[project("o/good"), project("o/bad")], + branch_sha=cursed_sha, + template_ref="", + latest_pipeline=None, + pipeline_jobs=[], + protected_branch=None, + open_merge_requests=(0, []), + recent_merges=[], + ) + + remote, _api, _excluded = gitlab.collect(config(), {}, {}, fleet=("o/good", "o/bad")) + + assert set(remote) == {"o/good"} + assert "o/bad failed" in caplog.text + + +@pytest.mark.parametrize( + ("overrides", "cfg_overrides"), + [ + ({"archived": True}, {}), + # `internal` means every signed-in user on the instance - not public, + # though it reads like it. + ({"visibility": "internal"}, {"public_only": True}), + ({"visibility": "private"}, {"public_only": True}), + ], +) +def test_a_dropped_repo_is_reported_back_as_excluded(monkeypatch, overrides, cfg_overrides): + """Reported back so the local scan skips its clone too - otherwise a + checkout keeps a dropped repo on the board after the remote half has + correctly stopped reporting it.""" + _stub( + monkeypatch, + list_projects=[project(**overrides)], + branch_sha="sha", + template_ref="", + latest_pipeline=None, + pipeline_jobs=[], + protected_branch=None, + open_merge_requests=(0, []), + recent_merges=[], + ) + + remote, _api, excluded = gitlab.collect(config(**cfg_overrides), {}, {}, fleet=("o/r",)) + + assert excluded == frozenset({"o/r"}) + assert remote == {} + + +def test_an_ignored_repo_is_excluded(monkeypatch): + _stub( + monkeypatch, + list_projects=[project()], + branch_sha="sha", + template_ref="", + latest_pipeline=None, + pipeline_jobs=[], + protected_branch=None, + open_merge_requests=(0, []), + recent_merges=[], + ) + + _remote, _api, excluded = gitlab.collect(config(ignore=("o/r",)), {}, {}, fleet=("o/r",)) + + assert excluded == frozenset({"o/r"}) + + +# -- the branches a healthy fleet never takes ------------------------------- +def test_the_client_can_be_closed(make_gitlab): + """`__main__` closes it in a `finally`, so a bad snapshot cannot leak a + connection pool.""" + client = make_gitlab({}) + + client.close() + + assert client._client.is_closed + + +@pytest.mark.parametrize("entry", ["no-slash", "o/r"]) +def test_an_unusable_or_repeated_fleet_entry_is_skipped(make_gitlab, entry): + """A bare name cannot address a project, and a repeat would cost a second + call for a row that is already there.""" + client = make_gitlab({"/projects/o%2Fr": project()}) + + found = client.list_projects(("o/r", entry)) + + assert [p["path_with_namespace"] for p in found] == ["o/r"] + assert client.calls.count("/projects/o%2Fr") == 1 + + +def test_a_branch_the_api_will_not_describe_has_no_sha(make_gitlab): + """An empty repository, or a default_branch that no longer exists.""" + client = make_gitlab({}) + + assert client.branch_sha("o/r", "main") == "" + + +def test_a_branch_payload_with_no_commit_has_no_sha(make_gitlab): + client = make_gitlab({"/projects/o%2Fr/repository/branches/main": {"name": "main"}}) + + assert client.branch_sha("o/r", "main") == "" + + +def test_a_pipeline_listing_entry_with_no_id_is_no_pipeline(make_gitlab): + """Without an id there is nothing to fetch in full, and a half-read pipeline + would report a CI state nobody can click through to.""" + client = make_gitlab({"/projects/o%2Fr/pipelines": [{"status": "success"}]}) + + assert client.latest_pipeline("o/r", "main") is None + + +def test_the_jobs_of_a_pipeline_are_paginated(make_gitlab): + client = make_gitlab( + {"/projects/o%2Fr/pipelines/77/jobs": [job("test", "success"), job("lint", "failed")]} + ) + + jobs = client.pipeline_jobs("o/r", 77) + + assert [j["name"] for j in jobs] == ["test", "lint"] + assert client.calls == ["/projects/o%2Fr/pipelines/77/jobs"] + + +def test_a_merged_listing_the_api_will_not_give_is_empty(make_gitlab): + client = make_gitlab({}) + + assert client.recent_merges("o/r", 10) == [] + + +def test_an_unknown_mr_pipeline_status_is_no_checks(): + """GitLab has added states before and will again; an unrecognised one must + not render as a red check.""" + assert gitlab._checks_state("some_future_state") == "none" diff --git a/collector/tests/test_gitlab_http.py b/collector/tests/test_gitlab_http.py new file mode 100644 index 0000000..ec70d37 --- /dev/null +++ b/collector/tests/test_gitlab_http.py @@ -0,0 +1,263 @@ +"""The real HTTP layer of the GitLab client, driven through a mock transport. + +test_gitlab.py fakes the client at `_json`/`_text`, which is right for testing +what the collector *derives* from a payload but means the layer producing those +payloads - the auth header, status handling, pagination, URL encoding - never +runs. A MockTransport keeps the client, the request building and the response +handling real, and only replaces the socket. +""" + +from __future__ import annotations + +import httpx +import pytest + +from jq_collector.config import Config +from jq_collector.gitlab import GitLab + + +def client(handler, **cfg_overrides) -> GitLab: + """A GitLab whose transport is `handler`, everything else real.""" + cfg = Config() + for key, value in cfg_overrides.items(): + object.__setattr__(cfg, key, value) + api = GitLab(cfg) + api._client = httpx.Client( + base_url=cfg.gitlab_api, + headers=dict(api._client.headers), + transport=httpx.MockTransport(handler), + follow_redirects=True, + ) + return api + + +def route(table: dict[str, httpx.Response]): + """Dispatch on the raw path, ignoring query parameters. + + Raw, not `url.path`: a project id is a percent-encoded path, and `url.path` + decodes it back to slashes - which is exactly the collapse these tests exist + to catch. Keys therefore carry the `/api/v4` prefix and the `%2F` intact. + """ + + def handler(request: httpx.Request) -> httpx.Response: + path = request.url.raw_path.decode().split("?", 1)[0] + response = table.get(path) + if response is None: # pragma: no cover - a test asked for an unstubbed path + raise AssertionError(f"unstubbed path {path}") + return httpx.Response( + response.status_code, content=response.content, headers=response.headers + ) + + return handler + + +def ok(payload) -> httpx.Response: + return httpx.Response(200, json=payload) + + +# -- the token header ------------------------------------------------------- +def test_a_token_is_sent_as_a_private_token_header(): + """PRIVATE-TOKEN, not Bearer: GITLAB_TOKEN holds a personal or project + access token, and Bearer would be an OAuth one.""" + api = GitLab(_cfg(gitlab_token="sekrit")) + + assert api._client.headers["PRIVATE-TOKEN"] == "sekrit" + assert "Authorization" not in api._client.headers + + +def test_no_token_sends_no_auth_header(): + """Public projects are readable unauthenticated, so an empty token must + produce a valid anonymous request rather than an empty header.""" + api = GitLab(_cfg(gitlab_token="")) + + assert "PRIVATE-TOKEN" not in api._client.headers + + +def _cfg(**overrides) -> Config: + cfg = Config() + for key, value in overrides.items(): + object.__setattr__(cfg, key, value) + return cfg + + +# -- the statuses a real fleet produces ------------------------------------- +@pytest.mark.parametrize("status", [401, 403, 404]) +def test_the_expected_empties_are_none_not_exceptions(status, caplog): + """A project the token cannot see, a feature the tier excludes, a file that + is not there. All states a real fleet contains; failing the refresh over one + would blank the whole board.""" + api = client(route({"/api/v4/projects/o%2Fr": httpx.Response(status)})) + + with caplog.at_level("INFO"): + assert api._json("/projects/o%2Fr") is None + + # Logged, not silent: a repo vanishing from the board because its token lost + # a scope must leave a trace somebody can find. + assert f"-> {status}" in caplog.text + + +@pytest.mark.parametrize("status", [500, 502, 429]) +def test_a_server_error_is_raised_so_the_refresh_is_recorded_as_failed(status): + """Unlike the empties, these are not a state of the fleet - they are the API + being broken, and must show up as a failed refresh rather than as a repo + that quietly vanished.""" + api = client(route({"/api/v4/projects/o%2Fr": httpx.Response(status)})) + + with pytest.raises(httpx.HTTPStatusError): + api._json("/projects/o%2Fr") + + +# -- URL encoding, end to end ---------------------------------------------- +def test_a_nested_namespace_reaches_the_wire_encoded(): + """The encoded path must survive httpx's own URL handling: an unescaped + slash here would address a different endpoint entirely.""" + seen: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request.url.raw_path.decode()) + return httpx.Response(200, json={"path_with_namespace": "acme/platform/infra/web"}) + + api = client(handler) + api.list_projects(("acme/platform/infra/web",)) + + assert seen == ["/api/v4/projects/acme%2Fplatform%2Finfra%2Fweb"] + + +def test_a_branch_name_with_a_slash_is_encoded(): + """`release/2.0` is a legal branch name and a broken URL path.""" + seen: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request.url.raw_path.decode()) + return httpx.Response(200, json={"commit": {"id": "abc123"}}) + + api = client(handler) + assert api.branch_sha("o/r", "release/2.0") == "abc123" + assert seen == ["/api/v4/projects/o%2Fr/repository/branches/release%2F2.0"] + + +# -- pagination ------------------------------------------------------------- +def test_a_full_page_is_followed_and_a_short_one_ends_it(): + pages: list[int] = [] + + def handler(request: httpx.Request) -> httpx.Response: + page = int(request.url.params.get("page", 1)) + pages.append(page) + if page == 1: + return httpx.Response(200, json=[{"iid": n} for n in range(100)]) + return httpx.Response(200, json=[{"iid": 100}]) + + api = client(handler) + items = api._paginate("/projects/o%2Fr/merge_requests") + + assert pages == [1, 2] + assert len(items) == 101 + + +def test_pagination_stops_at_a_sane_ceiling(): + """A runaway feed must not spend the whole refresh. Ten pages of 100 is far + beyond anything a monitored repo produces.""" + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=[{"iid": n} for n in range(100)]) + + api = client(handler) + items = api._paginate("/projects/o%2Fr/merge_requests") + + assert len(items) == 1000 + + +def test_an_empty_first_page_is_no_items_not_an_error(): + api = client(route({"/api/v4/projects/o%2Fr/merge_requests": ok([])})) + + assert api._paginate("/projects/o%2Fr/merge_requests") == [] + + +# -- the raw-file route, which is not JSON --------------------------------- +def test_the_pointer_is_read_as_text_not_parsed_as_json(): + """`repository/files/.../raw` answers with the file itself. Parsing that as + JSON would fail on any ordinary YAML pointer.""" + api = client( + route( + { + "/api/v4/projects/o%2Fr/repository/files/.rhiza%2Ftemplate.yml/raw": httpx.Response( + 200, content=b"ref: v1.7.1\nrepo: Jebel-Quant/rhiza\n" + ) + } + ) + ) + + assert api.template_ref("o/r", "main") == "v1.7.1" + + +def test_a_missing_pointer_is_an_unmanaged_repo(): + api = client( + route( + { + "/api/v4/projects/o%2Fr/repository/files/.rhiza%2Ftemplate.yml/raw": httpx.Response( + 404 + ) + } + ) + ) + + assert api.template_ref("o/r", "main") == "" + + +# -- the pipeline pair ------------------------------------------------------ +def test_the_newest_pipeline_is_fetched_in_full_for_its_coverage(): + """The listing carries a status but not coverage, so the one worth having is + fetched in full - two calls, and only for the newest.""" + api = client( + route( + { + "/api/v4/projects/o%2Fr/pipelines": ok([{"id": 77, "status": "success"}]), + "/api/v4/projects/o%2Fr/pipelines/77": ok( + {"id": 77, "status": "success", "coverage": "91.2"} + ), + } + ) + ) + + pipeline = api.latest_pipeline("o/r", "main") + + assert pipeline["coverage"] == "91.2" + + +def test_a_repo_that_has_never_run_a_pipeline_is_none(): + api = client(route({"/api/v4/projects/o%2Fr/pipelines": ok([])})) + + assert api.latest_pipeline("o/r", "main") is None + + +def test_the_listing_stands_in_when_the_full_fetch_fails(): + """Losing coverage is worth it to keep the status; dropping the pipeline + entirely would report a repo with CI as having none.""" + api = client( + route( + { + "/api/v4/projects/o%2Fr/pipelines": ok([{"id": 77, "status": "failed"}]), + "/api/v4/projects/o%2Fr/pipelines/77": httpx.Response(404), + } + ) + ) + + pipeline = api.latest_pipeline("o/r", "main") + + assert pipeline["status"] == "failed" + + +def test_a_protected_branch_is_read_and_an_unprotected_one_is_none(): + api = client( + route( + { + "/api/v4/projects/o%2Fr/protected_branches/main": ok( + {"name": "main", "allow_force_push": False} + ), + "/api/v4/projects/o%2Fr/protected_branches/dev": httpx.Response(404), + } + ) + ) + + assert api.protected_branch("o/r", "main")["allow_force_push"] is False + assert api.protected_branch("o/r", "dev") is None diff --git a/collector/tests/test_localgit_edges.py b/collector/tests/test_localgit_edges.py index dacba1e..90c7c05 100644 --- a/collector/tests/test_localgit_edges.py +++ b/collector/tests/test_localgit_edges.py @@ -26,7 +26,7 @@ _line_counts, _tags_mtime, _template_ref, - origin_owner_name, + origin_full_name, ) # -- _git: the wrapper every other call goes through ------------------------ @@ -61,25 +61,29 @@ def test_a_nonzero_exit_is_a_none(tmp_path): assert _git(str(tmp_path), "rev-parse", "HEAD") is None -# -- origin_owner_name: every URL shape a remote can carry ------------------ +# -- origin_full_name: reading a real checkout's remote ---------------------- +# +# The URL grammar itself is `origin.parse`, and test_origin.py drives every +# shape it accepts. What is left here is the part that needs a checkout on disk: +# that the remote is actually read, and that the states with no usable answer +# degrade to None rather than raising. @pytest.mark.parametrize( ("url", "expected"), [ - ("git@github.com:o/r.git", ("o", "r")), - ("git@github.com:o/r", ("o", "r")), - ("https://github.com/o/r.git", ("o", "r")), - ("ssh://git@github.com/o/r", ("o", "r")), + ("git@github.com:o/r.git", "o/r"), + ("https://github.com/o/r.git", "o/r"), + # A GitLab subgroup: the whole namespace, not the last two segments. + ("https://gitlab.com/acme/platform/infra/web", "acme/platform/infra/web"), # A path remote, which is what a local clone-of-a-clone has. - ("/srv/mirrors/o/r", ("o", "r")), - ("relative/o/r", ("o", "r")), + ("/srv/mirrors/o/r", "o/r"), ], ) -def test_owner_and_name_are_read_from_any_remote_shape(tmp_path, url, expected): +def test_the_full_name_is_read_from_the_remote(tmp_path, url, expected): path = make_checkout(tmp_path, "x", "y", origin=url) - assert origin_owner_name(str(path)) == expected + assert origin_full_name(str(path)) == expected def test_a_checkout_with_no_origin_has_no_identity(tmp_path): @@ -88,14 +92,14 @@ def test_a_checkout_with_no_origin_has_no_identity(tmp_path): path.mkdir() subprocess.run(["git", "init", "-q", "-b", "main", str(path)], check=True) - assert origin_owner_name(str(path)) is None + assert origin_full_name(str(path)) is None def test_a_remote_too_short_to_name_a_repo_is_refused(tmp_path): - """`owner/name` needs two segments; one cannot be split into a repo id.""" + """`namespace/name` needs two segments; one cannot be split into a repo id.""" path = make_checkout(tmp_path, "x", "y", origin="https://example.com/lonely") - assert origin_owner_name(str(path)) is None + assert origin_full_name(str(path)) is None # -- the template pointer --------------------------------------------------- diff --git a/collector/tests/test_main.py b/collector/tests/test_main.py index c560f92..5b1ea7b 100644 --- a/collector/tests/test_main.py +++ b/collector/tests/test_main.py @@ -85,29 +85,27 @@ def test_a_success_after_failures_keeps_the_error_count(): def test_the_loop_waits_before_its_first_run(): """main() has already seeded both sources, so running immediately would - spend a second full GitHub pass for nothing.""" + spend a second full remote pass for nothing.""" stop = threading.Event() stop.set() ran: list[str] = [] - entry._loop("github", 60, Store(), lambda: ran.append("x"), stop) + entry._loop(60, lambda: ran.append("x"), stop) assert ran == [] def test_the_loop_runs_the_work_and_then_notices_the_stop(): stop = threading.Event() - store = Store() runs: list[str] = [] def work(): runs.append("x") stop.set() - entry._loop("local", 0, store, work, stop) + entry._loop(0, work, stop) assert runs == ["x"] - assert store.health_for("local").last_success > 0 def test_a_stop_during_the_wait_returns_without_working(): @@ -116,11 +114,42 @@ def test_a_stop_during_the_wait_returns_without_working(): ran: list[str] = [] threading.Timer(0.01, stop.set).start() - entry._loop("github", 0.05, Store(), lambda: ran.append("x"), stop) + entry._loop(0.05, lambda: ran.append("x"), stop) assert ran == [] +def test_a_raising_tick_does_not_kill_the_loop(caplog): + """The loop is the only thing that will ever try again, and an exception + reaching a thread's target kills it silently. The per-forge health inside a + tick cannot cover the work between forges - merging and publishing - so the + loop holds the last resort.""" + stop = threading.Event() + calls: list[str] = [] + + def boom(): + calls.append("x") + stop.set() + raise RuntimeError("the snapshot was bad") + + entry._loop(0, boom, stop) + + assert calls == ["x"] + assert "refresh tick failed" in caplog.text + + +def test_the_local_tick_records_its_own_health(monkeypatch): + """Health bookkeeping moved out of `_loop` and into the ticks when the + remote pass grew from one source to one per forge - a single source name + per loop could no longer describe what it had talked to.""" + monkeypatch.setattr(entry.localgit, "scan", lambda *_a, **_k: {}) + store = Store() + + entry._tick_local(Config(), store) + + assert store.health_for("local").last_success > 0 + + # -- the two refreshers, and the caches they thread through --------------- @@ -160,9 +189,9 @@ class FakeAPI: def close(self): type(self).closed = True - def fake_collect(_cfg, ref_cache, coverage_cache): - seen.update(ref_cache=ref_cache, coverage_cache=coverage_cache) - return {"o/r": RemoteRepo(name="r")}, FakeAPI(), "v9", frozenset({"o/gone"}) + def fake_collect(_cfg, ref_cache, coverage_cache, fleet=None): + seen.update(ref_cache=ref_cache, coverage_cache=coverage_cache, fleet=fleet) + return {"o/r": RemoteRepo(name="r")}, FakeAPI(), ["v9", "v8"], frozenset({"o/gone"}) monkeypatch.setattr(entry.gh, "collect", fake_collect) store = Store() @@ -181,7 +210,7 @@ def fake_collect(_cfg, ref_cache, coverage_cache): } ) - entry._refresh_github(Config(), store) + entry._refresh_remote(Config(), store) assert seen["ref_cache"] == {"o/r": ("abc", "v1")} assert seen["coverage_cache"] == {"o/r": (99, (87.3, 472))} @@ -203,15 +232,15 @@ class FakeAPI: def close(self): pass - def fake_collect(_cfg, _ref_cache, coverage_cache): + def fake_collect(_cfg, _ref_cache, coverage_cache, fleet=None): seen["coverage_cache"] = coverage_cache - return {}, FakeAPI(), "", frozenset() + return {}, FakeAPI(), [], frozenset() monkeypatch.setattr(entry.gh, "collect", fake_collect) store = Store() store.update(remote={"o/r": RemoteRepo(name="r", coverage=None, coverage_artifact=7)}) - entry._refresh_github(Config(), store) + entry._refresh_remote(Config(), store) assert seen["coverage_cache"] == {"o/r": (7, None)} @@ -229,15 +258,19 @@ def close(self): monkeypatch.setattr( entry.gh, "collect", - lambda *_a, **_k: ({}, FakeAPI(), "", frozenset()), + lambda *_a, **_k: ({}, FakeAPI(), [], frozenset()), ) store = Store() monkeypatch.setattr(store, "update", lambda **_: (_ for _ in ()).throw(RuntimeError("bad"))) + # Two guarantees in one: the `finally` in the gather closure closes the + # client even though the store update blew up inside it, and the forge's own + # health records the failure rather than swallowing it. with pytest.raises(RuntimeError): - entry._refresh_github(Config(), store) + entry._refresh_remote(Config(), store) assert closed == [True] + assert store.health_for("github").errors == 1 # -- main(): seeding order, then shutdown --------------------------------- @@ -248,7 +281,7 @@ def test_main_seeds_both_sources_before_opening_the_port(monkeypatch): forever as a real observation of a fleet that was never in that state.""" order: list[str] = [] - monkeypatch.setattr(entry, "_refresh_github", lambda *_: order.append("github")) + monkeypatch.setattr(entry, "_refresh_remote", lambda *_: order.append("remote")) monkeypatch.setattr(entry, "_refresh_local", lambda *_: order.append("local")) monkeypatch.setattr(entry, "start_http_server", lambda _port: order.append("serve")) monkeypatch.setattr(entry.REGISTRY, "register", lambda _c: order.append("register")) @@ -258,14 +291,14 @@ def test_main_seeds_both_sources_before_opening_the_port(monkeypatch): entry.main() - assert order[:2] == ["github", "local"], "the port opened before both halves were seeded" + assert order[:2] == ["remote", "local"], "the port opened before both halves were seeded" assert order.index("serve") > order.index("local") def test_main_warns_when_there_is_no_token(monkeypatch, caplog): """60 calls an hour will not refresh this fleet once.""" - monkeypatch.setattr(entry, "_refresh_github", lambda *_: None) - monkeypatch.setattr(entry, "_refresh_local", lambda *_: None) + monkeypatch.setattr(entry, "_refresh_remote", lambda *_: None) + monkeypatch.setattr(entry, "_tick_local", lambda *_: None) monkeypatch.setattr(entry, "start_http_server", lambda _port: None) monkeypatch.setattr(entry.REGISTRY, "register", lambda _c: None) monkeypatch.setattr(entry.signal, "signal", lambda _sig, handler: handler(_sig, None)) @@ -288,8 +321,8 @@ def start(self): started.append(self.name) super().start() - monkeypatch.setattr(entry, "_refresh_github", lambda *_: None) - monkeypatch.setattr(entry, "_refresh_local", lambda *_: None) + monkeypatch.setattr(entry, "_refresh_remote", lambda *_: None) + monkeypatch.setattr(entry, "_tick_local", lambda *_: None) monkeypatch.setattr(entry, "start_http_server", lambda _port: None) monkeypatch.setattr(entry.REGISTRY, "register", lambda _c: None) monkeypatch.setattr(entry.threading, "Thread", RecordingThread) @@ -297,13 +330,13 @@ def start(self): entry.main() - assert sorted(started) == ["github", "local"] + assert sorted(started) == ["local", "remote"] def test_main_logs_that_it_is_shutting_down(monkeypatch, caplog): caplog.set_level(logging.INFO, logger="jq_collector") - monkeypatch.setattr(entry, "_refresh_github", lambda *_: None) - monkeypatch.setattr(entry, "_refresh_local", lambda *_: None) + monkeypatch.setattr(entry, "_refresh_remote", lambda *_: None) + monkeypatch.setattr(entry, "_tick_local", lambda *_: None) monkeypatch.setattr(entry, "start_http_server", lambda _port: None) monkeypatch.setattr(entry.REGISTRY, "register", lambda _c: None) monkeypatch.setattr(entry.signal, "signal", lambda _sig, handler: handler(_sig, None)) @@ -315,8 +348,8 @@ def test_main_logs_that_it_is_shutting_down(monkeypatch, caplog): def test_the_seed_records_health_for_both_sources(monkeypatch): """The Data age tile reads these, and it must not be blank on first scrape.""" - monkeypatch.setattr(entry, "_refresh_github", lambda *_: None) - monkeypatch.setattr(entry, "_refresh_local", lambda *_: None) + monkeypatch.setattr(entry.gh, "collect", lambda *_a, **_k: ({}, _NullAPI(), [], frozenset())) + monkeypatch.setattr(entry.localgit, "scan", lambda *_a, **_k: {}) monkeypatch.setattr(entry, "start_http_server", lambda _port: None) seen: list[Store] = [] monkeypatch.setattr(entry.REGISTRY, "register", lambda c: seen.append(c._store)) @@ -335,8 +368,8 @@ def test_main_holds_the_process_open_until_the_signal_arrives(monkeypatch): daemons and would not hold the process on their own. Registering the handler schedules the signal rather than firing it, so main() actually enters the loop.""" - monkeypatch.setattr(entry, "_refresh_github", lambda *_: None) - monkeypatch.setattr(entry, "_refresh_local", lambda *_: None) + monkeypatch.setattr(entry, "_refresh_remote", lambda *_: None) + monkeypatch.setattr(entry, "_tick_local", lambda *_: None) monkeypatch.setattr(entry, "start_http_server", lambda _port: None) monkeypatch.setattr(entry.REGISTRY, "register", lambda _c: None) monkeypatch.setattr( @@ -363,7 +396,7 @@ def test_running_the_package_as_a_module_calls_main(monkeypatch): # modules, so they still apply to the fresh execution. monkeypatch.delitem(sys.modules, "jq_collector.__main__", raising=False) - monkeypatch.setattr(entry.gh, "collect", lambda *_a, **_k: ({}, _NullAPI(), "", frozenset())) + monkeypatch.setattr(entry.gh, "collect", lambda *_a, **_k: ({}, _NullAPI(), [], frozenset())) monkeypatch.setattr(entry.localgit, "scan", lambda *_a, **_k: {}) monkeypatch.setattr("prometheus_client.start_http_server", lambda *_a, **_k: None) monkeypatch.setattr(entry.REGISTRY, "register", lambda _c: None) @@ -377,3 +410,168 @@ class _NullAPI: def close(self): pass + + +# -- two forges, one snapshot ----------------------------------------------- +# +# One remote pass rather than a loop each, because `Store.update(remote=...)` +# replaces the whole map: a loop per forge would have each thread publish only +# its own share, and the board would show half the fleet flickering against the +# other half. + + +def _cfg_with(forges: dict[str, str], repos: tuple[str, ...]) -> Config: + cfg = Config() + object.__setattr__(cfg, "repos", repos) + object.__setattr__(cfg, "forges", forges) + return cfg + + +def _fake_github(result, tags=("v2.0.0", "v1.9.0"), excluded=frozenset()): + def collect(_cfg, _ref_cache, _coverage_cache, fleet=None): + collect.fleet = fleet + return result, _NullAPI(), list(tags), excluded + + collect.fleet = None + return collect + + +def _fake_gitlab(result, excluded=frozenset()): + def collect(_cfg, _ref_cache, _coverage_cache, fleet=(), tags=None): + collect.fleet = fleet + collect.tags = tags + return result, _NullAPI(), excluded + + collect.fleet = None + collect.tags = None + return collect + + +def test_both_forges_land_in_one_remote_map(monkeypatch): + gh_collect = _fake_github({"o/gh": RemoteRepo(name="gh", forge="github")}) + gl_collect = _fake_gitlab({"acme/platform/web": RemoteRepo(name="web", forge="gitlab")}) + monkeypatch.setattr(entry.gh, "collect", gh_collect) + monkeypatch.setattr(entry.gl, "collect", gl_collect) + store = Store() + + entry._refresh_remote( + _cfg_with({"acme/platform/web": "gitlab"}, ("o/gh", "acme/platform/web")), store + ) + + assert set(store.snapshot().remote) == {"o/gh", "acme/platform/web"} + + +def test_each_forge_is_only_asked_about_its_own_share(monkeypatch): + """Asking GitHub about a GitLab repo answers 404 and logs it as unreadable.""" + gh_collect = _fake_github({}) + gl_collect = _fake_gitlab({}) + monkeypatch.setattr(entry.gh, "collect", gh_collect) + monkeypatch.setattr(entry.gl, "collect", gl_collect) + + entry._refresh_remote(_cfg_with({"acme/web": "gitlab"}, ("o/gh", "acme/web", "o/gh2")), Store()) + + assert gh_collect.fleet == ("o/gh", "o/gh2") + assert gl_collect.fleet == ("acme/web",) + + +def test_the_template_tags_are_fetched_once_and_handed_to_gitlab(monkeypatch): + """The template lives on GitHub whichever forge pins it, so drift is + measured against the same list - and fetching it twice would be a wasted + call per refresh.""" + gh_collect = _fake_github({}, tags=("v2.0.0", "v1.9.0")) + gl_collect = _fake_gitlab({}) + monkeypatch.setattr(entry.gh, "collect", gh_collect) + monkeypatch.setattr(entry.gl, "collect", gl_collect) + store = Store() + + entry._refresh_remote(_cfg_with({"acme/web": "gitlab"}, ("acme/web",)), store) + + assert gl_collect.tags == ["v2.0.0", "v1.9.0"] + assert store.snapshot().latest_template_ref == "v2.0.0" + + +def test_github_is_collected_even_with_no_repos_on_it(monkeypatch): + """It is the source of the template's releases, so a GitLab-only fleet still + needs the one call - otherwise every repo's drift reads as unknown.""" + gh_collect = _fake_github({}) + monkeypatch.setattr(entry.gh, "collect", gh_collect) + monkeypatch.setattr(entry.gl, "collect", _fake_gitlab({})) + + entry._refresh_remote(_cfg_with({"acme/web": "gitlab"}, ("acme/web",)), Store()) + + assert gh_collect.fleet == (), "GitHub was skipped and took the template with it" + + +def test_gitlab_is_not_touched_when_no_repo_lives_there(monkeypatch): + """A GitHub-only deployment must not build a GitLab client at all.""" + called: list[str] = [] + monkeypatch.setattr(entry.gh, "collect", _fake_github({})) + monkeypatch.setattr(entry.gl, "collect", lambda *_a, **_k: called.append("gitlab")) + + entry._refresh_remote(_cfg_with({}, ("o/gh",)), Store()) + + assert called == [] + assert "gitlab" not in Store().snapshot().health + + +def test_a_gitlab_outage_leaves_the_github_repos_standing(monkeypatch): + """The failure must be visible as its own source rather than as a silently + short fleet.""" + monkeypatch.setattr(entry.gh, "collect", _fake_github({"o/gh": RemoteRepo(name="gh")})) + + def exploding(*_a, **_k): + raise RuntimeError("gitlab is down") + + monkeypatch.setattr(entry.gl, "collect", exploding) + store = Store() + + entry._refresh_remote(_cfg_with({"acme/web": "gitlab"}, ("o/gh", "acme/web")), store) + + assert set(store.snapshot().remote) == {"o/gh"} + assert store.health_for("gitlab").errors == 1 + assert store.health_for("gitlab").last_error == "gitlab is down" + assert store.health_for("github").errors == 0 + + +def test_both_forges_contribute_to_the_excluded_set(monkeypatch): + """Reported back so the local scan skips their clones too - a checkout would + otherwise keep a dropped repo on the board.""" + monkeypatch.setattr(entry.gh, "collect", _fake_github({}, excluded=frozenset({"o/archived"}))) + monkeypatch.setattr(entry.gl, "collect", _fake_gitlab({}, excluded=frozenset({"acme/old"}))) + store = Store() + + entry._refresh_remote(_cfg_with({"acme/web": "gitlab"}, ("o/gh", "acme/web")), store) + + assert store.snapshot().excluded == frozenset({"o/archived", "acme/old"}) + + +def test_main_warns_about_a_missing_gitlab_token_only_when_it_is_needed(monkeypatch, caplog): + monkeypatch.setattr(entry, "_refresh_remote", lambda *_: None) + monkeypatch.setattr(entry, "_tick_local", lambda *_: None) + monkeypatch.setattr(entry, "start_http_server", lambda _port: None) + monkeypatch.setattr(entry.REGISTRY, "register", lambda _c: None) + monkeypatch.setattr(entry.signal, "signal", lambda _sig, handler: handler(_sig, None)) + + cfg = _cfg_with({"acme/web": "gitlab"}, ("acme/web",)) + object.__setattr__(cfg, "gitlab_token", "") + monkeypatch.setattr(entry, "Config", lambda: cfg) + + entry.main() + + assert "no GITLAB_TOKEN" in caplog.text + + +def test_a_github_only_fleet_is_not_nagged_about_gitlab(monkeypatch, caplog): + monkeypatch.setattr(entry, "_refresh_remote", lambda *_: None) + monkeypatch.setattr(entry, "_tick_local", lambda *_: None) + monkeypatch.setattr(entry, "start_http_server", lambda _port: None) + monkeypatch.setattr(entry.REGISTRY, "register", lambda _c: None) + monkeypatch.setattr(entry.signal, "signal", lambda _sig, handler: handler(_sig, None)) + + cfg = _cfg_with({}, ("o/gh",)) + object.__setattr__(cfg, "gitlab_token", "") + monkeypatch.setattr(entry, "Config", lambda: cfg) + + entry.main() + + assert "GITLAB_TOKEN" not in caplog.text diff --git a/collector/tests/test_metrics.py b/collector/tests/test_metrics.py index 93ae200..095e64e 100644 --- a/collector/tests/test_metrics.py +++ b/collector/tests/test_metrics.py @@ -354,3 +354,307 @@ def test_a_source_that_has_never_refreshed_reports_no_health(): assert any('source="github"' in ln for ln in lines) assert not any('source="local"' in ln for ln in lines) + + +# -- the forge and the URLs, which live on the identity metrics only ---------- +# +# Deliberately only there. Adding a label to a metric family starts a new series +# and orphans the old one, and the thirty families keyed on `repo` alone are the +# history behind every trend panel and alert rule. The `*_info` metrics are +# `Always 1` join targets whose label sets churn anyway, so they are where a new +# dimension costs nothing. + + +def test_the_forge_and_url_ride_on_the_identity_metric(): + snap = Snapshot( + remote={ + "acme/platform/web": RemoteRepo( + name="web", + owner="acme/platform", + forge="gitlab", + url="https://gitlab.com/acme/platform/web", + visibility="private", + ) + } + ) + + line = next(x for x in expose(snap) if x.startswith("jq_repo_info")) + + assert 'forge="gitlab"' in line + assert 'repo_url="https://gitlab.com/acme/platform/web"' in line + # The owner is the whole nested namespace, not its first segment. + assert 'owner="acme/platform"' in line + + +def test_a_repo_with_no_remote_half_still_gets_an_identity_row(): + """A clone of a repo the forge could not be read for. The row must exist - + the dashboard joins on it - and must not claim a forge it does not know.""" + snap = Snapshot(local={"o/r": LocalRepo(name="r", path="/p")}) + + line = next(x for x in expose(snap) if x.startswith("jq_repo_info")) + + assert 'forge="github"' in line + assert 'repo_url=""' in line + + +def test_no_other_family_gained_a_forge_label(): + """The history-preserving half of that decision, pinned. A `forge` label + leaking onto a `["repo"]` family would silently orphan 180 days of it.""" + snap = Snapshot( + remote={"o/r": RemoteRepo(name="r", owner="o", forge="gitlab", pushed_at=1.0)}, + local={"o/r": LocalRepo(name="r", path="/p", dirty_files=2)}, + ) + + carrying = [x.split("{")[0] for x in expose(snap) if 'forge="' in x] + + assert carrying == ["jq_repo_info"] + + +def test_a_pull_requests_url_is_exposed_rather_than_rebuilt(): + """The dashboard used to paste the repo label onto `https://github.com/` and + append the number. That cannot be right for both forges, and never had to be + - the API hands the URL over.""" + snap = Snapshot( + remote={ + "acme/web": RemoteRepo( + name="web", + owner="acme", + forge="gitlab", + pulls=( + PullRequest( + number=7, + title="Fix it", + author="someone", + draft=False, + created_at=1.0, + updated_at=2.0, + checks="success", + url="https://gitlab.com/acme/web/-/merge_requests/7", + ), + ), + ) + } + ) + + line = next(x for x in expose(snap) if x.startswith("jq_pull_request_info")) + + assert 'url="https://gitlab.com/acme/web/-/merge_requests/7"' in line + + +def test_the_merged_timeline_itself_gains_no_label(): + """That timestamp *is* the merged-PR history the panel reads, so the URL + goes on a sibling family. A label here would have orphaned every existing + series and, until they went stale, shown each merged PR twice.""" + from jq_collector.state import MergedPull + + snap = Snapshot( + remote={ + "o/r": RemoteRepo( + name="r", + owner="o", + merged=( + MergedPull( + number=4, + title="Landed", + author="someone", + merged_at=100.0, + url="https://github.com/o/r/pull/4", + ), + ), + ) + } + ) + + line = next(x for x in expose(snap) if x.startswith("jq_merged_pull_request_timestamp_seconds")) + + assert "url=" not in line + + +def test_a_merged_pull_requests_url_is_exposed_too(): + from jq_collector.state import MergedPull + + snap = Snapshot( + remote={ + "acme/web": RemoteRepo( + name="web", + owner="acme", + merged=( + MergedPull( + number=4, + title="Landed", + author="someone", + merged_at=100.0, + url="https://gitlab.com/acme/web/-/merge_requests/4", + ), + ), + ) + } + ) + + line = next(x for x in expose(snap) if x.startswith("jq_merged_pull_request_info")) + + assert 'url="https://gitlab.com/acme/web/-/merge_requests/4"' in line + + +def test_health_is_exposed_for_whatever_reported(): + """The source list used to be a hardcoded ("github", "local") tuple, which + would have dropped gitlab's health silently.""" + snap = Snapshot( + health={ + "github": SourceHealth(last_success=1.0), + "gitlab": SourceHealth(last_success=2.0, errors=3), + "local": SourceHealth(last_success=3.0), + } + ) + + lines = [x for x in expose(snap) if x.startswith("jq_collector_last_success")] + + assert {'source="github"', 'source="gitlab"', 'source="local"'} == { + x.split("{")[1].split("}")[0] for x in lines + } + + +def test_the_pull_request_listing_url_is_carried_per_forge(): + """No API reports this one, so each collector builds it - GitHub spells it + /pulls and GitLab /-/merge_requests. The dashboard reads the label rather + than learning either convention.""" + snap = Snapshot( + remote={ + "acme/web": RemoteRepo( + name="web", + owner="acme", + forge="gitlab", + url="https://gitlab.com/acme/web", + pulls_url="https://gitlab.com/acme/web/-/merge_requests", + ) + } + ) + + line = next(x for x in expose(snap) if x.startswith("jq_repo_info")) + + assert 'pulls_url="https://gitlab.com/acme/web/-/merge_requests"' in line + + +def test_the_ci_run_url_is_exposed_on_the_info_metric(): + """Linking to the run itself, not to a listing page whose path differs per + forge - and strictly more useful than the /actions link it replaces.""" + snap = Snapshot( + remote={ + "acme/web": RemoteRepo( + name="web", + owner="acme", + ci_conclusion="failure", + ci_workflow="test", + ci_url="https://gitlab.com/acme/web/-/jobs/9", + ) + } + ) + + line = next(x for x in expose(snap) if x.startswith("jq_ci_last_run_info")) + + assert 'url="https://gitlab.com/acme/web/-/jobs/9"' in line + + +def test_each_workflow_carries_its_own_run_url(): + """A separate family from jq_ci_workflow_success, which is a real gauge with + history - a label there would have orphaned all of it.""" + snap = Snapshot( + remote={ + "acme/web": RemoteRepo( + name="web", + owner="acme", + workflows=( + WorkflowRun( + name="lint", + conclusion="failure", + finished_at=10.0, + duration=1.0, + url="https://gitlab.com/acme/web/-/jobs/1", + ), + ), + ) + } + ) + + lines = [x for x in expose(snap) if x.startswith("jq_ci_workflow_info")] + + assert len(lines) == 1 + assert 'workflow="lint"' in lines[0] + assert 'url="https://gitlab.com/acme/web/-/jobs/1"' in lines[0] + + +def test_the_gauge_families_behind_the_history_carry_no_url(): + """The other half of the label decision. A `url` label leaking onto a + `["repo"]` gauge would silently orphan 180 days of its series.""" + snap = Snapshot( + remote={ + "o/r": RemoteRepo( + name="r", + owner="o", + url="https://github.com/o/r", + pulls_url="https://github.com/o/r/pulls", + ci_conclusion="success", + ci_workflow="test", + ci_url="https://github.com/o/r/actions/runs/1", + workflows=( + WorkflowRun( + name="test", + conclusion="success", + finished_at=1.0, + duration=1.0, + url="https://github.com/o/r/actions/runs/1", + ), + ), + ) + } + ) + + carrying = sorted({x.split("{")[0] for x in expose(snap) if 'url="http' in x}) + + assert carrying == [ + "jq_ci_last_run_info", + "jq_ci_workflow_info", + "jq_repo_info", + ], "a url label reached a family that carries history" + + +def test_only_the_join_targets_carry_a_url(): + """The census, pinned. Every family here is an `Always 1` join target whose + label set already churns as PRs open and close; a `url` reaching anything + else would start a new series and orphan its history.""" + from jq_collector.state import MergedPull + + snap = Snapshot( + remote={ + "o/r": RemoteRepo( + name="r", + owner="o", + url="https://github.com/o/r", + pulls_url="https://github.com/o/r/pulls", + ci_conclusion="success", + ci_workflow="test", + ci_url="https://github.com/o/r/actions/runs/1", + workflows=(wf("test", "success"),), + pulls=(pull(7, "success"),), + merged=( + MergedPull( + number=4, + title="Landed", + author="someone", + merged_at=100.0, + url="https://github.com/o/r/pull/4", + ), + ), + ) + } + ) + + families = sorted({x.split("{")[0] for x in expose(snap) if "url=" in x}) + + assert families == [ + "jq_ci_last_run_info", + "jq_ci_workflow_info", + "jq_merged_pull_request_info", + "jq_pull_request_info", + "jq_repo_info", + ] diff --git a/collector/tests/test_origin.py b/collector/tests/test_origin.py new file mode 100644 index 0000000..f4f5fd2 --- /dev/null +++ b/collector/tests/test_origin.py @@ -0,0 +1,121 @@ +"""The origin-URL grammar, which decides both a repo's key and its forge. + +This used to be two copies of one parser, tested only through a checkout on +disk. It is one pure function now, so the shapes can be driven directly - which +matters because getting it wrong is not a crash but a wrong answer: a repo keyed +under a name that does not exist, or read through the wrong API. +""" + +from __future__ import annotations + +import subprocess + +import pytest + +from jq_collector import origin + + +# -- the URL shapes a remote is written in ---------------------------------- +@pytest.mark.parametrize( + ("url", "host", "namespace", "name"), + [ + # scp-like, with and without the .git suffix. + ("git@github.com:o/r.git", "github.com", "o", "r"), + ("git@github.com:o/r", "github.com", "o", "r"), + # https, and ssh:// which is the same grammar with a user in it. + ("https://github.com/o/r.git", "github.com", "o", "r"), + ("ssh://git@github.com/o/r", "github.com", "o", "r"), + # A port must not be mistaken for part of the host. + ("ssh://git@gitlab.com:2222/o/r", "gitlab.com", "o", "r"), + # GitLab namespaces nest, and the whole path is the namespace. Keeping + # the last two segments here would key this repo as `infra/web` and ask + # the API for a project that does not exist. + ( + "https://gitlab.com/acme/platform/infra/web.git", + "gitlab.com", + "acme/platform/infra", + "web", + ), + ("git@gitlab.com:acme/platform/infra/web", "gitlab.com", "acme/platform/infra", "web"), + # Case is not significant in a hostname, and the forge lookup depends on + # it matching. + ("https://GitLab.com/o/r", "gitlab.com", "o", "r"), + # A path remote - a clone of a clone. No host, so the filesystem trail + # is not a namespace and only its tail is meaningful. + ("/srv/mirrors/o/r", "", "o", "r"), + ("relative/o/r", "", "o", "r"), + ("/deeply/nested/mirror/tree/o/r", "", "o", "r"), + # Whitespace, because this comes off `git remote get-url` stdout. + (" https://github.com/o/r\n", "github.com", "o", "r"), + ], +) +def test_every_remote_shape_is_split_into_host_namespace_and_name(url, host, namespace, name): + parsed = origin.parse(url) + + assert parsed is not None + assert (parsed.host, parsed.namespace, parsed.name) == (host, namespace, name) + assert parsed.full_name == f"{namespace}/{name}" + + +@pytest.mark.parametrize("url", ["", " ", "https://example.com/lonely", "lonely", "git@host:x"]) +def test_a_url_that_cannot_name_a_repo_is_none(url): + """One path segment cannot be split into a namespace and a name.""" + assert origin.parse(url) is None + + +# -- which forge a host is -------------------------------------------------- +@pytest.mark.parametrize( + ("host", "forge"), + [ + ("gitlab.com", "gitlab"), + ("GITLAB.COM", "gitlab"), + # A self-hosted GitLab conventionally lives on a gitlab.* hostname. + ("gitlab.internal.acme.com", "gitlab"), + ("github.com", "github"), + # Anything unrecognised stays GitHub, which is what keeps a GitHub + # Enterprise host and every path remote behaving as they did before + # this module existed. + ("github.acme.com", "github"), + ("git.example.com", "github"), + ("", "github"), + ], +) +def test_the_forge_is_inferred_from_the_host(host, forge): + assert origin.forge_for_host(host) == forge + + +def test_the_parsed_origin_carries_its_own_forge(): + """The whole point of keeping the host: inference costs nothing here.""" + assert origin.parse("git@gitlab.com:acme/web").forge == "gitlab" + assert origin.parse("git@github.com:acme/web").forge == "github" + # A path remote has no host to infer from, so it falls to the default. + assert origin.parse("/srv/mirrors/acme/web").forge == "github" + + +# -- reading it off a checkout ---------------------------------------------- +def test_a_directory_that_is_not_a_repo_is_none(tmp_path): + assert origin.read(str(tmp_path)) is None + + +def test_a_git_binary_that_cannot_be_run_is_a_none_not_a_crash(monkeypatch, caplog): + """OSError, e.g. git missing from PATH entirely inside a stripped image.""" + + def explode(*_args, **_kwargs): + raise OSError("no git here") + + monkeypatch.setattr(subprocess, "run", explode) + + assert origin.read("/anywhere") is None + assert "failed" in caplog.text + + +def test_a_git_call_that_times_out_is_a_none_not_a_crash(monkeypatch, caplog): + """A repo on a stalled network mount can hang; the monitor must not.""" + + def stall(*_args, **_kwargs): + raise subprocess.TimeoutExpired(cmd="git", timeout=20) + + monkeypatch.setattr(subprocess, "run", stall) + + assert origin.read("/anywhere") is None + assert "failed" in caplog.text diff --git a/docker-compose.yml b/docker-compose.yml index 43f01cc..294d8a5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,9 @@ services: restart: unless-stopped environment: GITHUB_TOKEN: ${GITHUB_TOKEN:?set GITHUB_TOKEN, e.g. export GITHUB_TOKEN=$(gh auth token)} + # Optional, and only wanted if repos.yml lists a `forge: gitlab` entry. + # Unset is fine: public gitlab.com projects are readable without one. + GITLAB_TOKEN: ${GITLAB_TOKEN:-} JQ_TEMPLATE_REPO: ${JQ_TEMPLATE_REPO:-Jebel-Quant/rhiza} JQ_GITHUB_INTERVAL: ${JQ_GITHUB_INTERVAL:-600} JQ_LOCAL_INTERVAL: ${JQ_LOCAL_INTERVAL:-60} diff --git a/docs/configuration.md b/docs/configuration.md index ee68d30..6d8d1d2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -43,6 +43,41 @@ dead repo on the board — that gap kept `rhiza-brainbug` showing as the fleet's one red repo for a while after it was archived. Set `JQ_INCLUDE_ARCHIVED=true` to opt back in. +## GitHub and GitLab in the same fleet + +An entry is read through GitHub unless it says otherwise: + +```yaml +repos: + - path: ~/repos/jebel-quant/rhiza # github.com, from the origin remote + - path: ~/repos/acme/web # gitlab.com, from the origin remote + - repo: acme/platform/infra/web # no checkout, so it has to say + forge: gitlab +``` + +Where an entry has a `path`, the forge is inferred from the origin remote's +host — `gitlab.com`, or any `gitlab.*` hostname, is GitLab; everything else is +GitHub. Where it has only a `repo:`, there is no origin to look at, so anything +but GitHub must carry `forge: gitlab`. A forge nobody implements is refused at +startup rather than quietly read as GitHub, which would leave the repo on the +board with every remote panel empty and nothing saying why. + +**GitLab namespaces nest, and the whole path is the name.** The project `web` in +the subgroup `acme/platform/infra` is `acme/platform/infra/web`, not +`infra/web` — and that whole path is what appears in the `repo` label and in +every dashboard row. + +**The same `namespace/name` on both forges is refused.** One `repo` label is one +repo, and the board's entire label scheme rests on that; a merged pair would +report one repo's CI under the other's name while still looking like a working +board. Rename one entry with an explicit `repo:` if you genuinely have both. + +Only gitlab.com is supported. Self-hosted GitLab would need the API base URL per +entry rather than one `GITLAB_API`, and that is not built. + +**One panel group cannot be filled for GitLab repos** — see +[the Dependabot note](metrics.md#dependabot-has-no-gitlab-counterpart). + ## Dropping a repo Dropping a repo stops new samples but leaves its **history**, so it still @@ -92,7 +127,10 @@ Passed as `-e NAME=value` on the `docker run`, or in `.env` if you use | Variable | Default | | |---|---|---| -| `GITHUB_TOKEN` | — | Must be able to read every repo in `repos.yml`. `gh auth token` prints a usable one. | +| `GITHUB_TOKEN` | — | Must be able to read every GitHub repo in `repos.yml`. `gh auth token` prints a usable one. | +| `GITLAB_TOKEN` | — | Only needed if `repos.yml` lists a `forge: gitlab` entry. Public projects are readable without one; private ones need `read_api`. | +| `GITLAB_API` | `https://gitlab.com/api/v4` | Only gitlab.com is supported. | +| `JQ_REPO_FORGES` | — | Which forge each repo is read through, as `owner/name=forge` pairs. Filled from `repos.yml`; set it for a deployment with no file to mount. | | `JQ_TEMPLATE_REPO` | `Jebel-Quant/rhiza` | Whose releases define "up to date", as `owner/name`. | | `JQ_IGNORE` | — | Repos to drop without editing `repos.yml`, as bare names or `owner/name`. Applies to both halves. | | `JQ_INCLUDE_ARCHIVED` | `false` | Archived repos are dropped from both halves. | @@ -100,7 +138,7 @@ Passed as `-e NAME=value` on the `docker run`, or in `.env` if you use | `JQ_REPO_PATHS` | — | Where each checkout really is, as `owner/name=path` pairs. Filled from `repos.yml`; set it to override one entry — see [Checkout paths](#checkout-paths). | | `JQ_HOST_ROOT` | `/host` | Where your home directory is mounted, and therefore what `~` in `repos.yml` means. | | `JQ_PROM_ADMIN_API` | `false` | Enables the API [`purge-repo`](#dropping-a-repo) needs. Leave it off. | -| `JQ_GITHUB_INTERVAL` | `300` | Seconds between GitHub refreshes. | +| `JQ_GITHUB_INTERVAL` | `300` | Seconds between remote refreshes. Both forges are collected in one pass, so this paces both. | | `JQ_MEASURE_MAX_AGE` | `86400` | Seconds an unchanged line/commit count may stand before it is retaken. See [Size and cadence](dashboard.md#size-and-cadence). | | `PROM_RETENTION` | `180d` | How much history to keep. | diff --git a/docs/dashboard.md b/docs/dashboard.md index 36e8373..5cfc1fb 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -11,10 +11,11 @@ keywords: grafana dashboard, promql, panel json, alert rules, state timeline, gr Every tile except *Repos monitored* links to a table listing exactly what is behind that number — collapsed under *Drill-down* at the bottom, opened full screen by the click. Each drill-down's row count is the tile's value by -construction, and each repo name links to GitHub. +construction, and each repo name links to that repo on its own forge — the URL +comes from the API that reported it, so a GitLab row opens GitLab. The tiles are laid out in two rows because that is how the data splits. The top -row is what **GitHub** says about the repos — a headline count, the two workload +row is what **the forge** says about the repos — a headline count, the two workload counts (open PRs, open issues), then the three problem counts. The bottom row is what **this machine** says about the clones: not in sync, dirty, off default branch. A green top row with an amber bottom row means the fleet is healthy and diff --git a/docs/index.md b/docs/index.md index c365294..6feaeda 100644 --- a/docs/index.md +++ b/docs/index.md @@ -59,6 +59,7 @@ working-copy panels once the first scan of the mount completes. | `-v "$HOME:/host:ro"` | Your home directory, read-only, so `~/...` in `repos.yml` resolves. Leave it out and the working-copy panels stay empty; everything the GitHub half reports still works. | | `-v jq-fleet-data:/data` | Prometheus history and Grafana's database. Leave it out and both start empty at every run. | | `-e GITHUB_TOKEN=...` | Needs `repo` and `read:org`. Without one GitHub allows 60 calls an hour, which is not a fleet. | +| `-e GITLAB_TOKEN=...` | Only if the fleet has a GitLab repo in it. Needs `read_api`. | !!! warning "Publish the ports to `127.0.0.1` only" diff --git a/docs/metrics.md b/docs/metrics.md index d6ce13e..d668e03 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -10,14 +10,23 @@ keywords: prometheus metrics, github actions, template drift, pull requests, git ``` your clones ──(read-only bind mount, git --no-optional-locks)──┐ - ├─► collector ─► Prometheus ─► Grafana -GitHub REST API ──(repos, runs, PRs, releases)─────────────────┘ :9109 :9090 :3000 + │ +GitHub REST API ──(repos, runs, PRs, releases)─────────────────┼─► collector ─► Prometheus ─► Grafana + │ :9109 :9090 :3000 +GitLab REST API ──(projects, pipelines, MRs)──────────────────-─┘ ``` The collector holds a cached snapshot and refreshes it on two independent -timers — GitHub every 5 minutes, the local clones every 60 seconds — so a scrape -never waits on the API, and an API outage does not blank out the local panels. -Prometheus scrapes every 30s and keeps 180 days. +timers — the forges every 5 minutes, the local clones every 60 seconds — so a +scrape never waits on an API, and an API outage does not blank out the local +panels. Prometheus scrapes every 30s and keeps 180 days. + +Both forges are collected in **one** remote pass rather than a loop each, because +publishing the snapshot replaces the whole remote map: a loop per forge would +have each one publish only its own share, and the board would show half the +fleet flickering against the other half. Health is still recorded per forge, so +`jq_collector_errors{source="gitlab"}` tells you which one is unwell and a GitLab +outage leaves the GitHub repos standing. ## The four subjects @@ -82,13 +91,72 @@ two levels deep, and a clone is kept only if its `origin` belongs to a monitored org or is named explicitly — so an upstream `numpy` checkout, or the repos under `~/repos/tschm`, are skipped without any extra configuration. -### The `repo` label is `owner/name` - -Not the bare name. Two orgs are in scope and a bare name is only unique within -one owner, so the pair is the identity: the dashboard joins table frames on this -label and builds every GitHub link out of it, and both need it unique on its -own. It also makes alert annotations work unmodified — `{{ $labels.repo }}` is -already the full path. +### The `repo` label is the full `namespace/name` + +Not the bare name. Several namespaces are in scope and a bare name is only +unique within one, so the whole path is the identity: the dashboard joins table +frames on this label, so it has to be unique on its own. It also makes alert +annotations work unmodified — `{{ $labels.repo }}` is already the full path. + +On GitLab that path can be more than two segments deep, because namespaces +nest: `acme/platform/infra/web` is one repo, not a repo called `infra/web` in a +group called `acme/platform`. + +### The forge rides on `jq_repo_info`, and nowhere else + +`jq_repo_info` carries `forge`, `repo_url` and `pulls_url` alongside the +identity labels it always had. Every other metric is untouched. + +That is deliberate rather than lazy. Adding a label to a Prometheus metric +family starts a *new* series and orphans the old one, and the thirty families +keyed on `repo` alone are the history behind every trend panel and every alert +rule. Putting the new dimension on the identity metric — already an `Always 1` +join target whose label set churns as PRs open and close — means a fleet that +predates GitLab support keeps all 180 days of it, and GitLab repos simply arrive +as new `repo` values on the existing families. + +It is `repo_url` and not `url` so that a panel can pull it in with `group_left` +alongside a pull request's or a run's own `url` without the two colliding, which +is exactly what the drill-down tables do. + +### Links come from the API, not from string-building + +The dashboard used to build every link by pasting the `repo` label onto +`https://github.com/`. That cannot be right for two forges — GitLab spells the +merge-request list `/-/merge_requests` — and it never had to be, because every +API hands the URL over. So `jq_pull_request_info`, +`jq_merged_pull_request_timestamp_seconds`, `jq_ci_last_run_info` and +`jq_ci_workflow_info` each carry a `url` label with what the forge actually +reported, and the tables read the field. + +`jq_ci_workflow_info` is a separate family rather than a label on +`jq_ci_workflow_success` for the reason above: that one is a real gauge with +history behind it. + +### Dependabot has no GitLab counterpart + +The three Dependabot panels — *Dependabot off*, *Open Dependabot alerts*, and +their drill-down — stay **blank for GitLab repos**. GitLab's equivalent is +dependency scanning and the vulnerability report, which are Ultimate-tier +features, and the REST Vulnerability Findings API is being retired in favour of +GraphQL. There is nothing to read at most tiers. + +This degrades honestly rather than misleadingly. The collector already keeps +"the feature is off" apart from "zero open alerts" — the whole reason +`jq_dependabot_alerts_enabled` exists next to `jq_dependabot_open_alerts` — so a +GitLab repo reports alerts as *disabled* and shows as unknown. It never renders +as a green "no alerts" tile for a repo nobody is scanning. + +Two smaller gaps, in the same spirit: + +- **Required reviews is always 0 on GitLab.** MR approval rules are a Premium + feature, so `jq_branch_required_reviews` has nothing to report and + `jq_branch_protected` carries the signal instead. +- **Coverage has no line count on GitLab.** It comes from the pipeline's own + `coverage` field rather than from a report the collector parses, so + `jq_ci_coverage_lines` is 0 — which says "not measured" rather than inventing + a denominator. GitLab is *cheaper* here, though: no artifact to list and no + zip to download. ## Two things worth knowing about the local numbers diff --git a/grafana/dashboards/fleet.json b/grafana/dashboards/fleet.json index 4d4f9e2..ff8c4a2 100644 --- a/grafana/dashboards/fleet.json +++ b/grafana/dashboards/fleet.json @@ -18,6 +18,24 @@ "graphTooltip": 1, "templating": { "list": [ + { + "name": "forge", + "label": "Forge", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "jq-prometheus" + }, + "query": "label_values(jq_repo_info, forge)", + "refresh": 2, + "includeAll": true, + "allValue": ".*", + "multi": true, + "current": { + "text": "All", + "value": "$__all" + } + }, { "name": "repo", "label": "Repo", @@ -26,7 +44,7 @@ "type": "prometheus", "uid": "jq-prometheus" }, - "query": "label_values(jq_repo_info, repo)", + "query": "label_values(jq_repo_info{forge=~\"$forge\"}, repo)", "refresh": 2, "includeAll": true, "allValue": ".*", @@ -1163,7 +1181,7 @@ "targets": [ { "refId": "A", - "expr": "jq_ci_last_run_info{repo=~\"$repo\"}", + "expr": "jq_ci_last_run_info{repo=~\"$repo\"} * on(repo) group_left(repo_url) topk by (repo) (1, jq_repo_info{repo=~\"$repo\"})", "instant": true, "format": "table" }, @@ -1379,13 +1397,55 @@ "id": "links", "value": [ { - "title": "Open Actions on GitHub", - "url": "https://github.com/${__value.text}/actions", + "title": "Open the repo on its forge", + "url": "${__data.fields.repo_url}", + "targetBlank": true + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Workflow" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "Open this run", + "url": "${__data.fields.url}", "targetBlank": true } ] } ] + }, + { + "matcher": { + "id": "byName", + "options": "repo_url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] } ] }, @@ -1407,7 +1467,7 @@ "targets": [ { "refId": "A", - "expr": "(time() - jq_pull_request_created_timestamp_seconds{repo=~\"$repo\"}) * on(repo, number) group_left(title, author, checks, draft) topk by (repo, number) (1, jq_pull_request_info{repo=~\"$repo\"})", + "expr": "(time() - jq_pull_request_created_timestamp_seconds{repo=~\"$repo\"}) * on(repo, number) group_left(title, author, checks, draft, url) topk by (repo, number) (1, jq_pull_request_info{repo=~\"$repo\"}) * on(repo) group_left(repo_url, pulls_url) topk by (repo) (1, jq_repo_info{repo=~\"$repo\"})", "instant": true, "format": "table" } @@ -1501,8 +1561,8 @@ "id": "links", "value": [ { - "title": "Open this pull request on GitHub", - "url": "https://github.com/${__data.fields.Repo}/pull/${__data.fields[\"#\"]}", + "title": "Open this pull request", + "url": "${__data.fields.url}", "targetBlank": true } ] @@ -1606,8 +1666,8 @@ "id": "links", "value": [ { - "title": "Open this pull request on GitHub", - "url": "https://github.com/${__data.fields.Repo}/pull/${__data.fields[\"#\"]}", + "title": "Open this pull request", + "url": "${__data.fields.url}", "targetBlank": true } ] @@ -1632,13 +1692,49 @@ "id": "links", "value": [ { - "title": "Open the repo's pull requests on GitHub", - "url": "https://github.com/${__value.text}/pulls", + "title": "Open the repo's pull requests", + "url": "${__data.fields.pulls_url}", "targetBlank": true } ] } ] + }, + { + "matcher": { + "id": "byName", + "options": "repo_url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "pulls_url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] } ] }, @@ -1662,7 +1758,7 @@ "targets": [ { "refId": "A", - "expr": "time() - topk(10, jq_merged_pull_request_timestamp_seconds{repo=~\"$repo\"})", + "expr": "time() - topk(10, jq_merged_pull_request_timestamp_seconds{repo=~\"$repo\"}) * on(repo, number) group_left(url) topk by (repo, number) (1, jq_merged_pull_request_info{repo=~\"$repo\"}) * on(repo) group_left(repo_url, pulls_url) topk by (repo) (1, jq_repo_info{repo=~\"$repo\"})", "instant": true, "format": "table" } @@ -1736,8 +1832,8 @@ "id": "links", "value": [ { - "title": "Open on GitHub", - "url": "https://github.com/${__value.text}", + "title": "Open the repo's pull requests", + "url": "${__data.fields.pulls_url}", "targetBlank": true } ] @@ -1759,7 +1855,7 @@ "value": [ { "title": "Open this pull request", - "url": "https://github.com/${__data.fields.Repo}/pull/${__data.fields[\"#\"]}", + "url": "${__data.fields.url}", "targetBlank": true } ] @@ -1785,6 +1881,42 @@ "value": 140 } ] + }, + { + "matcher": { + "id": "byName", + "options": "repo_url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "pulls_url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] } ] } @@ -2339,7 +2471,7 @@ "targets": [ { "refId": "A", - "expr": "(time() - jq_ci_workflow_timestamp_seconds{repo=~\"$repo\"}) and on(repo, workflow) (jq_ci_workflow_success{repo=~\"$repo\"} == 0)", + "expr": "((time() - jq_ci_workflow_timestamp_seconds{repo=~\"$repo\"}) and on(repo, workflow) (jq_ci_workflow_success{repo=~\"$repo\"} == 0)) * on(repo, workflow) group_left(url) topk by (repo, workflow) (1, jq_ci_workflow_info{repo=~\"$repo\"}) * on(repo) group_left(repo_url) topk by (repo) (1, jq_repo_info{repo=~\"$repo\"})", "instant": true, "format": "table" } @@ -2409,8 +2541,8 @@ "id": "links", "value": [ { - "title": "Open Actions on GitHub", - "url": "https://github.com/${__value.text}/actions", + "title": "Open the repo on its forge", + "url": "${__data.fields.repo_url}", "targetBlank": true } ] @@ -2436,6 +2568,48 @@ "value": 160 } ] + }, + { + "matcher": { + "id": "byName", + "options": "Workflow" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "Open this run", + "url": "${__data.fields.url}", + "targetBlank": true + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "repo_url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] } ] } @@ -2458,7 +2632,7 @@ "targets": [ { "refId": "A", - "expr": "(jq_rhiza_releases_behind{repo=~\"$repo\"} > 0) * on(repo) group_left(ref) topk by (repo) (1, jq_rhiza_template_ref_info{repo=~\"$repo\"})", + "expr": "(jq_rhiza_releases_behind{repo=~\"$repo\"} > 0) * on(repo) group_left(ref) topk by (repo) (1, jq_rhiza_template_ref_info{repo=~\"$repo\"}) * on(repo) group_left(repo_url) topk by (repo) (1, jq_repo_info{repo=~\"$repo\"})", "instant": true, "format": "table" } @@ -2529,8 +2703,8 @@ "id": "links", "value": [ { - "title": "Open on GitHub", - "url": "https://github.com/${__value.text}", + "title": "Open the repo on its forge", + "url": "${__data.fields.repo_url}", "targetBlank": true } ] @@ -2552,6 +2726,18 @@ "value": 160 } ] + }, + { + "matcher": { + "id": "byName", + "options": "repo_url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] } ] } @@ -2574,7 +2760,7 @@ "targets": [ { "refId": "A", - "expr": "(jq_local_dirty_files{repo=~\"$repo\"} > 0) * on(repo) group_left(branch) topk by (repo) (1, jq_local_branch_info{repo=~\"$repo\"})", + "expr": "(jq_local_dirty_files{repo=~\"$repo\"} > 0) * on(repo) group_left(branch) topk by (repo) (1, jq_local_branch_info{repo=~\"$repo\"}) * on(repo) group_left(repo_url) topk by (repo) (1, jq_repo_info{repo=~\"$repo\"})", "instant": true, "format": "table" } @@ -2645,8 +2831,8 @@ "id": "links", "value": [ { - "title": "Open on GitHub", - "url": "https://github.com/${__value.text}", + "title": "Open the repo on its forge", + "url": "${__data.fields.repo_url}", "targetBlank": true } ] @@ -2668,6 +2854,18 @@ "value": 140 } ] + }, + { + "matcher": { + "id": "byName", + "options": "repo_url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] } ] } @@ -2690,7 +2888,7 @@ "targets": [ { "refId": "A", - "expr": "(time() - jq_pull_request_created_timestamp_seconds{repo=~\"$repo\"}) * on(repo, number) group_left(title, author, checks) topk by (repo, number) (1, jq_pull_request_info{repo=~\"$repo\", checks=\"failure\"})", + "expr": "(time() - jq_pull_request_created_timestamp_seconds{repo=~\"$repo\"}) * on(repo, number) group_left(title, author, checks, url) topk by (repo, number) (1, jq_pull_request_info{repo=~\"$repo\", checks=\"failure\"}) * on(repo) group_left(repo_url, pulls_url) topk by (repo) (1, jq_repo_info{repo=~\"$repo\"})", "instant": true, "format": "table" } @@ -2767,8 +2965,8 @@ "id": "links", "value": [ { - "title": "Open the repo's pull requests on GitHub", - "url": "https://github.com/${__value.text}/pulls", + "title": "Open the repo's pull requests", + "url": "${__data.fields.pulls_url}", "targetBlank": true } ] @@ -2809,8 +3007,8 @@ "id": "links", "value": [ { - "title": "Open this pull request on GitHub", - "url": "https://github.com/${__data.fields.Repo}/pull/${__data.fields[\"#\"]}", + "title": "Open this pull request", + "url": "${__data.fields.url}", "targetBlank": true } ] @@ -2827,13 +3025,49 @@ "id": "links", "value": [ { - "title": "Open this pull request on GitHub", - "url": "https://github.com/${__data.fields.Repo}/pull/${__data.fields[\"#\"]}", + "title": "Open this pull request", + "url": "${__data.fields.url}", "targetBlank": true } ] } ] + }, + { + "matcher": { + "id": "byName", + "options": "repo_url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "pulls_url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] } ] } @@ -2856,7 +3090,7 @@ "targets": [ { "refId": "A", - "expr": "topk by (repo) (1, jq_local_branch_info{repo=~\"$repo\"}) and on(repo) (jq_local_on_default_branch{repo=~\"$repo\"} == 0)", + "expr": "(topk by (repo) (1, jq_local_branch_info{repo=~\"$repo\"}) and on(repo) (jq_local_on_default_branch{repo=~\"$repo\"} == 0)) * on(repo) group_left(repo_url) topk by (repo) (1, jq_repo_info{repo=~\"$repo\"})", "instant": true, "format": "table" } @@ -2925,13 +3159,25 @@ "id": "links", "value": [ { - "title": "Open on GitHub", - "url": "https://github.com/${__value.text}", + "title": "Open the repo on its forge", + "url": "${__data.fields.repo_url}", "targetBlank": true } ] } ] + }, + { + "matcher": { + "id": "byName", + "options": "repo_url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] } ] } @@ -2954,7 +3200,7 @@ "targets": [ { "refId": "A", - "expr": "(jq_local_fetch_age_seconds{repo=~\"$repo\"} and on(repo) (jq_local_default_branch_synced{repo=~\"$repo\"} == 0)) * on(repo) group_left(branch) topk by (repo) (1, jq_local_branch_info{repo=~\"$repo\"})", + "expr": "(jq_local_fetch_age_seconds{repo=~\"$repo\"} and on(repo) (jq_local_default_branch_synced{repo=~\"$repo\"} == 0)) * on(repo) group_left(branch) topk by (repo) (1, jq_local_branch_info{repo=~\"$repo\"}) * on(repo) group_left(repo_url) topk by (repo) (1, jq_repo_info{repo=~\"$repo\"})", "instant": true, "format": "table" } @@ -3024,8 +3270,8 @@ "id": "links", "value": [ { - "title": "Open on GitHub", - "url": "https://github.com/${__value.text}", + "title": "Open the repo on its forge", + "url": "${__data.fields.repo_url}", "targetBlank": true } ] @@ -3051,6 +3297,18 @@ "value": 150 } ] + }, + { + "matcher": { + "id": "byName", + "options": "repo_url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] } ] } @@ -3073,7 +3331,7 @@ "targets": [ { "refId": "A", - "expr": "jq_open_issues{repo=~\"$repo\"} > 0", + "expr": "(jq_open_issues{repo=~\"$repo\"} > 0) * on(repo) group_left(repo_url) topk by (repo) (1, jq_repo_info{repo=~\"$repo\"})", "instant": true, "format": "table" } @@ -3141,8 +3399,8 @@ "id": "links", "value": [ { - "title": "Open on GitHub", - "url": "https://github.com/${__value.text}", + "title": "Open the repo on its forge", + "url": "${__data.fields.repo_url}", "targetBlank": true } ] @@ -3164,6 +3422,18 @@ "value": 150 } ] + }, + { + "matcher": { + "id": "byName", + "options": "repo_url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] } ] } @@ -3186,7 +3456,7 @@ "targets": [ { "refId": "A", - "expr": "jq_branch_protected{repo=~\"$repo\"} == 0", + "expr": "(jq_branch_protected{repo=~\"$repo\"} == 0) * on(repo) group_left(repo_url) topk by (repo) (1, jq_repo_info{repo=~\"$repo\"})", "instant": true, "format": "table" } @@ -3254,8 +3524,8 @@ "id": "links", "value": [ { - "title": "Open on GitHub", - "url": "https://github.com/${__value.text}", + "title": "Open the repo on its forge", + "url": "${__data.fields.repo_url}", "targetBlank": true } ] @@ -3277,6 +3547,18 @@ "value": 150 } ] + }, + { + "matcher": { + "id": "byName", + "options": "repo_url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] } ] } @@ -3299,7 +3581,7 @@ "targets": [ { "refId": "A", - "expr": "jq_dependabot_open_alerts{repo=~\"$repo\"} > 0", + "expr": "(jq_dependabot_open_alerts{repo=~\"$repo\"} > 0) * on(repo) group_left(repo_url) topk by (repo) (1, jq_repo_info{repo=~\"$repo\"})", "instant": true, "format": "table" } @@ -3369,8 +3651,8 @@ "id": "links", "value": [ { - "title": "Open on GitHub", - "url": "https://github.com/${__value.text}", + "title": "Open the repo on its forge", + "url": "${__data.fields.repo_url}", "targetBlank": true } ] @@ -3392,6 +3674,18 @@ "value": 150 } ] + }, + { + "matcher": { + "id": "byName", + "options": "repo_url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] } ] } @@ -3414,7 +3708,7 @@ "targets": [ { "refId": "A", - "expr": "jq_dependabot_alerts_enabled{repo=~\"$repo\"} == 0", + "expr": "(jq_dependabot_alerts_enabled{repo=~\"$repo\"} == 0) * on(repo) group_left(repo_url) topk by (repo) (1, jq_repo_info{repo=~\"$repo\"})", "instant": true, "format": "table" } @@ -3482,8 +3776,8 @@ "id": "links", "value": [ { - "title": "Open on GitHub", - "url": "https://github.com/${__value.text}", + "title": "Open the repo on its forge", + "url": "${__data.fields.repo_url}", "targetBlank": true } ] @@ -3505,6 +3799,18 @@ "value": 150 } ] + }, + { + "matcher": { + "id": "byName", + "options": "repo_url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] } ] } @@ -3527,7 +3833,7 @@ "targets": [ { "refId": "A", - "expr": "jq_branch_allows_force_push{repo=~\"$repo\"} == 1", + "expr": "(jq_branch_allows_force_push{repo=~\"$repo\"} == 1) * on(repo) group_left(repo_url) topk by (repo) (1, jq_repo_info{repo=~\"$repo\"})", "instant": true, "format": "table" } @@ -3595,8 +3901,8 @@ "id": "links", "value": [ { - "title": "Open on GitHub", - "url": "https://github.com/${__value.text}", + "title": "Open the repo on its forge", + "url": "${__data.fields.repo_url}", "targetBlank": true } ] @@ -3618,6 +3924,18 @@ "value": 150 } ] + }, + { + "matcher": { + "id": "byName", + "options": "repo_url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] } ] } @@ -3795,7 +4113,7 @@ "targets": [ { "refId": "A", - "expr": "jq_rhiza_template_ref_info{repo=~\"$repo\"}", + "expr": "jq_rhiza_template_ref_info{repo=~\"$repo\"} * on(repo) group_left(repo_url) topk by (repo) (1, jq_repo_info{repo=~\"$repo\"})", "instant": true, "format": "table" }, @@ -3895,13 +4213,25 @@ "id": "links", "value": [ { - "title": "Open on GitHub", - "url": "https://github.com/${__value.text}", + "title": "Open the repo on its forge", + "url": "${__data.fields.repo_url}", "targetBlank": true } ] } ] + }, + { + "matcher": { + "id": "byName", + "options": "repo_url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] } ] }, diff --git a/grafana/provisioning/alerting/rules.yml b/grafana/provisioning/alerting/rules.yml index 96afd0d..fb38b30 100644 --- a/grafana/provisioning/alerting/rules.yml +++ b/grafana/provisioning/alerting/rules.yml @@ -20,7 +20,11 @@ groups: fleet: jebel-quant annotations: summary: "{{ $labels.repo }} has been red on its default branch for over a day" - runbook: "https://github.com/{{ $labels.repo }}/actions" + # The dashboard drill-down, not a forge URL. These rules fire off + # metrics labelled with `repo` alone, so there is nothing here to join + # against for the repo's own forge and address - and the drill-down + # panel carries the real per-forge link anyway. + runbook: "/d/jq-fleet/jebel-quant-fleet?viewPanel=101" noDataState: OK execErrState: Error data: @@ -86,7 +90,7 @@ groups: fleet: jebel-quant annotations: summary: "{{ $labels.repo }} has {{ $values.B }} open PR(s) with failing checks" - runbook: "https://github.com/{{ $labels.repo }}/pulls" + runbook: "/d/jq-fleet/jebel-quant-fleet?viewPanel=104" noDataState: OK execErrState: Error data: diff --git a/image/entrypoint.sh b/image/entrypoint.sh index 9efd764..3743985 100755 --- a/image/entrypoint.sh +++ b/image/entrypoint.sh @@ -22,6 +22,14 @@ if [[ -z "${GITHUB_TOKEN:-}" ]]; then say "no GITHUB_TOKEN - GitHub allows 60 unauthenticated calls an hour, which is not enough for a fleet. Pass -e GITHUB_TOKEN=\"\$(gh auth token)\"." fi +# Only if the fleet actually has a GitLab repo in it. A GitHub-only board must +# not start warning about credentials it has no use for, so this greps the file +# rather than just testing the variable. +if [[ -z "${GITLAB_TOKEN:-}" ]] \ + && grep -qE '^[[:space:]]*forge:[[:space:]]*.?gitlab' "${JQ_REPOS_FILE:-/config/repos.yml}"; then + say "repos.yml lists a GitLab repo but no GITLAB_TOKEN is set - only public + projects will be readable. Pass -e GITLAB_TOKEN=\"...\"." +fi # Not fatal: a board with no working copies mounted is a supported way to run # this. Saying so once beats a user wondering why half the panels are empty. if [[ ! -d "${JQ_HOST_ROOT:-/host}" ]]; then diff --git a/repos.example.yml b/repos.example.yml index ac2340b..555adb1 100644 --- a/repos.example.yml +++ b/repos.example.yml @@ -29,3 +29,29 @@ repos: # for it. This is also the form to use for every entry if you would rather # not mount your home directory at all. # - repo: Jebel-Quant/actions + + # A GitLab project. `forge: gitlab` is what sends it to GitLab's API instead + # of GitHub's; without it the entry is read as a GitHub repo and every remote + # panel comes back empty. + # + # It can be left off when the entry has a `path`, because then the checkout's + # origin host says which forge it is. Spelling it out costs nothing and is the + # only option for an entry with no checkout. + # - repo: acme/web + # forge: gitlab + + # GitLab namespaces nest, and the whole path is the project's name - not just + # the last two segments. This is the project `web` in `acme/platform/infra`. + # - repo: acme/platform/infra/web + # forge: gitlab + + # A GitLab checkout, where the forge follows from the origin remote's host. + # - path: ~/repos/acme/web + + # Two notes on a mixed fleet: + # + # - Set GITLAB_TOKEN as well as GITHUB_TOKEN. Public projects are readable + # without one, private ones are not. + # - The Dependabot tiles stay blank for GitLab repos. GitLab's equivalent is + # an Ultimate-tier feature, so there is nothing to read; the board shows + # them as unknown rather than as a green "no alerts".