Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,11 @@ jobs:
- name: Format
run: uv run --frozen ruff format --check jq_collector tests

# Scope and target version live in [tool.mypy], so this line and a local
# `uv run mypy` cannot check different things.
- name: Types
run: uv run --frozen mypy

- name: Tests
run: |
uv run --frozen python -m pytest \
Expand Down
2 changes: 1 addition & 1 deletion collector/jq_collector/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ def _refresh_remote(cfg: Config, store: Store) -> None:
# (artifact id, percent) per repo: an unchanged artifact means the report
# behind it is byte-identical, so there is nothing to gain from pulling the
# zip down again. GitLab reports coverage as a field and ignores this.
coverage_cache = {
coverage_cache: dict[str, tuple[int, tuple[float, int] | None]] = {
name: (repo.coverage_artifact, (repo.coverage, repo.coverage_lines))
if repo.coverage is not None
else (repo.coverage_artifact, None)
Expand Down
14 changes: 8 additions & 6 deletions collector/jq_collector/github.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import logging
import zipfile
from datetime import datetime
from typing import Any
from xml.etree import ElementTree

import httpx
Expand Down Expand Up @@ -81,7 +82,7 @@ def __init__(self, cfg: Config) -> None:
def close(self) -> None:
self._client.close()

def _get(self, path: str, **params: object) -> httpx.Response:
def _get(self, path: str, **params: str | int) -> httpx.Response:
response = self._client.get(path, params=params or None)
self._note_rate_limit(response)
return response
Expand All @@ -99,7 +100,7 @@ def _note_rate_limit(self, response: httpx.Response) -> None:
except ValueError:
pass

def _json(self, path: str, **params: object) -> object | None:
def _json(self, path: str, **params: str | int) -> object | None:
"""GET returning parsed JSON, or None for the expected empty cases.

404 (no such file), 403 (rate limited or forbidden) and 409 (empty
Expand All @@ -124,7 +125,7 @@ def _bytes(self, path: str) -> bytes | None:
response.raise_for_status()
return response.content

def _paginate(self, path: str, **params: object) -> list[dict]:
def _paginate(self, path: str, **params: str | int) -> list[dict]:
items: list[dict] = []
page = 1
while True:
Expand Down Expand Up @@ -309,8 +310,9 @@ def latest_runs(self, full_name: str, branch: str) -> list[dict]:
exclude_pull_requests="true",
per_page=100,
)
newest: dict[object, dict] = {}
for run in (data or {}).get("workflow_runs") or []:
newest: dict[Any, dict] = {}
feed = data.get("workflow_runs") if isinstance(data, dict) else None
for run in feed or []:
wid = run.get("workflow_id")
if active is not None and wid not in active:
continue # workflow deleted or disabled since this run
Expand All @@ -336,7 +338,7 @@ def latest_runs(self, full_name: str, branch: str) -> list[dict]:
# one; the API has no "conclusive only" filter.
per_page=5,
)
runs = (extra or {}).get("workflow_runs") or []
runs = (extra.get("workflow_runs") if isinstance(extra, dict) else None) or []
conclusive = next((r for r in runs if not _inconclusive(r)), None)
if conclusive is not None:
newest[wid] = conclusive
Expand Down
8 changes: 4 additions & 4 deletions collector/jq_collector/gitlab.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ def __init__(self, cfg: Config) -> None:
def close(self) -> None:
self._client.close()

def _json(self, path: str, **params: object) -> object | None:
def _json(self, path: str, **params: str | int) -> object | None:
"""GET returning parsed JSON, or None for the expected empty cases.

The same three states the GitHub client tolerates, for the same reason:
Expand All @@ -109,7 +109,7 @@ def _json(self, path: str, **params: object) -> object | None:
return None
return response.json()

def _get(self, path: str, **params: object) -> httpx.Response | None:
def _get(self, path: str, **params: str | int) -> httpx.Response | None:
"""GET, or None for the statuses a real fleet legitimately produces."""
response = self._client.get(path, params=params or None)
if response.status_code in (401, 403, 404):
Expand All @@ -118,7 +118,7 @@ def _get(self, path: str, **params: object) -> httpx.Response | None:
response.raise_for_status()
return response

def _text(self, path: str, **params: object) -> str | None:
def _text(self, path: str, **params: str | int) -> str | None:
"""GET returning the body as text, for the routes that serve a file.

``repository/files/.../raw`` answers with the file itself, so parsing it
Expand All @@ -127,7 +127,7 @@ def _text(self, path: str, **params: object) -> str | None:
response = self._get(path, **params)
return response.text if response is not None else None

def _paginate(self, path: str, **params: object) -> list[dict]:
def _paginate(self, path: str, **params: str | int) -> list[dict]:
items: list[dict] = []
page = 1
while True:
Expand Down
15 changes: 14 additions & 1 deletion collector/jq_collector/localgit.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import os
import subprocess
import time
from typing import TypedDict

from . import origin
from .config import Config
Expand Down Expand Up @@ -290,13 +291,25 @@ def scan_repo(
)


class _Measured(TypedDict):
"""The ``LocalRepo`` fields ``_measurements`` fills, typed for ``**`` unpacking."""

code_lines: int
test_lines: int
commits_30d: int
commits_since_release: int | None
last_release: str
fingerprint: str
measured_at: float


def _measurements(
path: str,
ref: str,
cfg: Config,
fingerprint: str,
previous: LocalRepo | None,
) -> dict[str, object]:
) -> _Measured:
"""The measured fields, taken fresh or carried over from ``previous``.

Carried over when the clone has not moved *and* the old reading is younger
Expand Down
4 changes: 2 additions & 2 deletions collector/jq_collector/metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@

from .forge import GOOD_CONCLUSIONS as _GOOD_CONCLUSIONS
from .forge import INCONCLUSIVE_CONCLUSIONS as _INCONCLUSIVE_CONCLUSIONS
from .state import Snapshot
from .state import Snapshot, WorkflowRun


def _gauge(name: str, doc: str, labels: list[str] | None = None) -> GaugeMetricFamily:
Expand Down Expand Up @@ -386,7 +386,7 @@ def render(snap: Snapshot):
# contract: duplicate label sets are silently dropped by Prometheus
# ("samples with different value but same timestamp"), which cost 16
# samples a scrape when the invariant was last broken upstream.
unique: dict[str, object] = {}
unique: dict[str, WorkflowRun] = {}
for wf in sorted(remote.workflows, key=lambda w: w.finished_at, reverse=True):
if wf.conclusion in _INCONCLUSIVE_CONCLUSIONS:
continue
Expand Down
9 changes: 8 additions & 1 deletion collector/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ build-backend = "hatchling.build"
packages = ["jq_collector"]

[dependency-groups]
dev = ["pytest>=8.0", "pytest-cov>=5.0", "ruff>=0.16"]
dev = ["pytest>=8.0", "pytest-cov>=5.0", "ruff>=0.16", "mypy>=1.10", "types-PyYAML>=6.0"]

[tool.pytest.ini_options]
testpaths = ["tests"]
Expand All @@ -37,6 +37,13 @@ source = ["jq_collector"]
# CI's test step fails on a drop without a --cov-fail-under of its own.
fail_under = 100

[tool.mypy]
# The package only, checked against the oldest Python it claims to run on. Not
# --strict yet: this is the floor that catches an undefined attribute before a
# refresh does, and it can be tightened once it holds.
files = ["jq_collector"]
python_version = "3.11"

[tool.ruff]
# Kept here rather than passed as a flag, so `check` and `format` cannot
# disagree - a --line-length on one and not the other silently fails CI.
Expand Down
Loading
Loading