From 2d52bfcbf04af63548305168e8e525af0c3c3a3b Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 00:53:05 +0100 Subject: [PATCH 01/10] Select configs among the configs of the latest backstop push's tasks, as the models name them The config selection models are trained with the configurations of the push data, as mozci names them from the task definitions (renamed, e.g. shippable configurations are the ones of their non-shippable equivalents). The candidates were instead obtained by matching the labels of the current tasks with the configs seen in training, which isn't the same: - configs of tasks whose label doesn't have their Treeherder platform (e.g. the Windows ASan ones) and shippable tasks were unknown to the models; - suites and variants named alike were confused (e.g. mochitest-webgpu and the webgpu variant); - tasks which test selection doesn't optimize (e.g. the standalone ones, which only run on backstop pushes) were candidates. The HTTP service now retrieves the test tasks that test selection optimizes from the task graph of the latest backstop push (which runs all of them, with all their manifests), with their configs computed with mozci's get_configuration, like the ones of the push data. Configs are selected among them. Shippable tasks are left out, as they are too costly: the models are trained with the configs of their non-shippable equivalents, which are selected instead. --- .pre-commit-config.yaml | 1 + bugbug/models/testconfigselect.py | 67 +++++----- http_service/bugbug_http/boot.py | 195 +++++++++++++++++++++-------- http_service/bugbug_http/models.py | 26 +++- http_service/pyproject.toml | 1 + http_service/tests/conftest.py | 27 +++- http_service/tests/test_boot.py | 170 +++++++++++++++++++++++++ tests/test_testconfigselect.py | 62 ++++----- uv.lock | 2 + 9 files changed, 422 insertions(+), 129 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6669b42caea..5d9156c1718 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -71,6 +71,7 @@ repos: - types-python-dateutil==2.9.0.20261104 - types-orjson==3.6.2 - types-tabulate==0.10.0.20261006 + - types-PyYAML==6.0.12.20261113 - repo: local hooks: - id: hackbot-deploy-names diff --git a/bugbug/models/testconfigselect.py b/bugbug/models/testconfigselect.py index df99e7f5597..eb9cebe7dc9 100644 --- a/bugbug/models/testconfigselect.py +++ b/bugbug/models/testconfigselect.py @@ -24,13 +24,11 @@ """ import collections -import fnmatch import functools import logging import math import os import random -import re from typing import Any, Collection, Iterable, Iterator, Mapping, NamedTuple, Sequence import numpy as np @@ -312,49 +310,44 @@ def generate_rows( history.add(regression) -def _label_without_chunk(label: str) -> str: - parts = label.split("-") - return "-".join(parts[:-1] if parts[-1].isdigit() else parts) +class LiveTask(NamedTuple): + """A test task of the current task graph, by label without chunk number (see live_candidates).""" + # Its config, as mozci names it (see mozci.task.get_configuration). + config: str + # The manifests run by its chunks. + manifests: Sequence[str] -def live_candidates( - manifests_by_task: Mapping[str, Iterable[str]], known_configs: Iterable[str] -) -> dict[str, list[str]]: - """The configs of the tasks in the current task graph that run each group. - `manifests_by_task` maps the labels of the tasks scheduled on the push's project to the manifests - they run (the decision task's manifests-by-task.json, restricted to its target tasks, as it comes - from the full task graph, which also has tasks of other projects, e.g. shippable variants). The - configs are patterns over task labels without their chunk number (e.g. - test-macosx1500-aarch64/debug-*-swr), as the taskgraph matches them: each task is named after - the most specific `known_configs` pattern it matches, or after its own label without the chunk - number if none matches (e.g. for a new platform). +def live_tasks(tasks: Mapping[str, LiveTask]) -> dict[str, dict[str, list[str]]]: + """The tasks of the current task graph running each group, by config. - As the candidates come from the current task graph, configs that don't exist anymore (e.g. after - a platform migration) are never selected. Groups are named as by the models (see + `tasks` are the test tasks of the push's project among which configs are selected (see + bugbug_http.boot), by label without chunk number. Groups are named as by the models (see test_scheduling.normalize_group). """ - patterns = sorted( - ((len(c), c, re.compile(fnmatch.translate(c))) for c in set(known_configs)), - reverse=True, + result: dict[str, dict[str, set[str]]] = collections.defaultdict( + lambda: collections.defaultdict(set) ) + for label, task in tasks.items(): + for manifest in task.manifests: + result[test_scheduling.normalize_group(manifest)][task.config].add(label) + + return { + group: {config: sorted(labels) for config, labels in configs.items()} + for group, configs in result.items() + } - @functools.cache - def config_of(label: str) -> str: - for _, config, pattern in patterns: - if pattern.match(label): - return config - return label - - result: dict[str, set[str]] = collections.defaultdict(set) - for label, manifests in manifests_by_task.items(): - if not label.startswith("test-"): - continue - config = config_of(_label_without_chunk(label)) - for manifest in manifests: - result[test_scheduling.normalize_group(manifest)].add(config) - - return {group: sorted(configs) for group, configs in result.items()} + +def live_candidates( + tasks_by_group: Mapping[str, Mapping[str, Sequence[str]]], +) -> dict[str, list[str]]: + """The configs of the tasks in the current task graph that run each group (see live_tasks). + + As the candidates come from the current task graph, configs that don't exist anymore (e.g. after + a platform migration) are never selected. + """ + return {group: sorted(configs) for group, configs in tasks_by_group.items()} class GroupCandidates(NamedTuple): diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index a6bc608ed24..e58db5cd99f 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -6,15 +6,17 @@ import collections import concurrent.futures import contextlib -import gzip import json import logging import os import re +from typing import Iterable, Iterator import hglib +import orjson import requests import tenacity +import yaml from tqdm import tqdm from bugbug import build_platforms, repository, test_scheduling, utils @@ -28,6 +30,11 @@ # How often the data about the current tasks, build platforms and commits is retrieved again (in # seconds, see bugbug_http.refresh). REFRESH_INTERVAL = 24 * 60 * 60 +CONFIG_SELECTION_TASKS_PATH = "config_selection_tasks.json" +# The timeouts of the requests for the current tasks (to connect, and between bytes received, in +# seconds), so that a stalled request fails (and is retried) instead of blocking the boot or the +# refresh forever. +REQUEST_TIMEOUT = (30, 120) def _write(path: str, content: str) -> None: @@ -38,35 +45,152 @@ def _write(path: str, content: str) -> None: os.replace(tmp_path, path) -@tenacity.retry( - stop=tenacity.stop_after_attempt(7), - wait=tenacity.wait_exponential(multiplier=1, min=1, max=8), -) -def retrieve_schedulable_tasks() -> None: +def _decision_artifact_url(rev: str, path: str) -> str: + return f"https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.autoland.revision.{rev}.taskgraph.decision/artifacts/public/{path}" + + +def _without_chunk(label: str) -> str: + return re.sub(r"-\d+$", "", label) + + +def _eligible_tasks( + task_graph: dict[str, dict], +) -> Iterator[tuple[str, str, list[str]]]: + """The test tasks of a task graph that test selection optimizes: (label without chunk number, + config as mozci names it, manifests). + + The tasks that test selection doesn't optimize (e.g. the standalone ones, which only run on + backstop pushes) are left out. So are shippable tasks, as they are too costly to select: the + config selection models are trained with the configs of their non-shippable equivalents (see + test_scheduling.rename_task), which are selected instead. + """ + # mozci.data needs to be imported before mozci.task to avoid a circular import. + import mozci.data # noqa: F401 + from mozci.task import get_configuration + + utils.setup_mozci() + + configurations: dict[str, str | None] = {} + for task in task_graph.values(): + manifests = task["attributes"].get("test_manifests") + if ( + not task["label"].startswith("test-") + or not manifests + or "test" not in (task.get("optimization") or {}) + ): + continue + + label = _without_chunk(task["label"]) + if label not in configurations: + try: + configuration = get_configuration(task) + except Exception: + # E.g. a test variant that mozci doesn't know yet. + logger.warning( + "Failed to get the configuration of %s", + task["label"], + exc_info=True, + ) + configuration = None + if ( + configuration is not None + and test_scheduling.rename_task(configuration) != configuration + ): + configuration = None + configurations[label] = configuration + + config = configurations[label] + if config is not None: + yield label, config, manifests + + +def get_config_selection_tasks( + task_graph: dict[str, dict], target_tasks: Iterable[str] +) -> dict[str, dict]: + """The test tasks among which configs are selected, by label without chunk number. + + They are the eligible test tasks (see _eligible_tasks) of `task_graph` (a backstop push's + task-graph.json, which has all of them with all their manifests) which are among the push's + target tasks (`target_tasks`, its target-tasks.json: the task graph also has the tasks they + depend on), with their config and manifests (see testconfigselect.LiveTask). + """ + targets = {_without_chunk(label) for label in target_tasks} + tasks: dict[str, dict] = {} + for label, config, manifests in _eligible_tasks(task_graph): + if label not in targets: + continue + task = tasks.setdefault(label, {"config": config, "manifests": []}) + task["manifests"] = sorted(set(task["manifests"]) | set(manifests)) + return tasks + + +def _get(rev: str, path: str) -> requests.Response: + return requests.get(_decision_artifact_url(rev, path), timeout=REQUEST_TIMEOUT) + + +def update_config_selection_tasks(revs: list[str]) -> None: + """Retrieve the test tasks among which configs are selected. + + The tasks come from the task graph of the latest backstop push, which runs all of them with all + their manifests (the manifests of the test tasks of other pushes are restricted to the ones + selected by bugbug). + + `revs` are the recent autoland pushes, from the oldest. + """ + for rev in reversed(revs): + r = _get(rev, "parameters.yml") + if not r.ok or not yaml.safe_load(r.text).get("backstop"): + continue + r = _get(rev, "task-graph.json") + targets = _get(rev, "target-tasks.json") + if not r.ok or not targets.ok: + break + tasks = get_config_selection_tasks(orjson.loads(r.content), targets.json()) + logger.info( + "Retrieved %d test tasks from the backstop push %s", len(tasks), rev + ) + if tasks: + _write( + CONFIG_SELECTION_TASKS_PATH, + orjson.dumps({"backstop": rev, "tasks": tasks}).decode(), + ) + return + break + + logger.warning("No backstop push with test tasks to retrieve") + + +def get_recent_revs() -> list[str]: + """The latest AUTOLAND_PUSHES autoland pushes, from the oldest.""" # The pushes of the last week (many more than AUTOLAND_PUSHES, also during holidays). r = requests.get( "https://hg.mozilla.org/integration/autoland/json-pushes", params={"version": "2", "tipsonly": "1", "startdate": "1 week ago"}, + timeout=REQUEST_TIMEOUT, ) r.raise_for_status() pushes = sorted(r.json()["pushes"].items(), key=lambda push: int(push[0])) - revs = [push_obj["changesets"][0] for _, push_obj in pushes[-AUTOLAND_PUSHES:]] + return [push_obj["changesets"][0] for _, push_obj in pushes[-AUTOLAND_PUSHES:]] + + +@tenacity.retry( + stop=tenacity.stop_after_attempt(7), + wait=tenacity.wait_exponential(multiplier=1, min=1, max=8), +) +def retrieve_schedulable_tasks() -> None: + revs = get_recent_revs() logger.info("Retrieving known tasks from %s...", revs) - # Store in a file the list of tasks in the latest autoland pushes, and the manifests run by - # their tasks (to select the configs to run manifests on among the current ones). Pushes - # don't all target the same tasks (only some of them target almost all), so we use the union - # of the target tasks of many pushes, which also protects us from broken decision tasks. + # Store in a file the list of tasks in the latest autoland pushes, and the test tasks among which + # configs are selected for the manifests. Pushes don't all target the same tasks (only some of + # them target almost all), so we use the union of the target tasks of many pushes, which also + # protects us from broken decision tasks. known_tasks = set() - targets_num = {} for rev in revs: - r = requests.get( - f"https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.autoland.revision.{rev}.taskgraph.decision/artifacts/public/target-tasks.json" - ) + r = _get(rev, "target-tasks.json") if r.ok: known_tasks.update(r.json()) - targets_num[rev] = len(r.json()) logger.info("Retrieved %d tasks", len(known_tasks)) @@ -74,43 +198,8 @@ def retrieve_schedulable_tasks() -> None: _write("known_tasks", "\n".join(known_tasks)) - # The manifests run by each task, from the push targeting the most tasks (pushes targeting few - # tasks have no test tasks). - manifests_by_task: dict[str, list[str]] = {} - for rev in sorted(targets_num, key=targets_num.__getitem__, reverse=True)[:3]: - r = requests.get( - f"https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.autoland.revision.{rev}.taskgraph.decision/artifacts/public/manifests-by-task.json.gz" - ) - if not r.ok: - continue - try: - content = gzip.decompress(r.content) - except gzip.BadGzipFile: - content = r.content - manifests_by_task = json.loads(content) - if manifests_by_task: - break - - # Only the target tasks, as the full task graph also has tasks that don't run on autoland - # (e.g. shippable variants), and a group given only configs without tasks on the push - # wouldn't run at all. Tasks are compared without their chunk number, which varies between - # pushes. - def without_chunk(label: str) -> str: - return re.sub(r"-\d+$", "", label) - - target_tasks = {without_chunk(task) for task in known_tasks} - manifests_by_task = { - label: manifests - for label, manifests in manifests_by_task.items() - if without_chunk(label) in target_tasks - } - logger.info("Retrieved the manifests of %d tasks", len(manifests_by_task)) - _write( - "manifests_by_task.json", - json.dumps( - {label: sorted(manifests) for label, manifests in manifests_by_task.items()} - ), - ) + # The test tasks among which configs are selected for the manifests. + update_config_selection_tasks(revs) def retrieve_build_platforms() -> None: diff --git a/http_service/bugbug_http/models.py b/http_service/bugbug_http/models.py index 778f82e04cf..dce95e8f33c 100644 --- a/http_service/bugbug_http/models.py +++ b/http_service/bugbug_http/models.py @@ -569,13 +569,31 @@ def get_known_tasks() -> tuple[str, ...]: CONFIG_SELECTION_THRESHOLD = 0.00015 +@lru_cache(maxsize=None) +def _get_config_selection_tasks() -> dict: + """The test tasks among which configs are selected (see boot.update_config_selection_tasks).""" + try: + with open("config_selection_tasks.json", "rb") as f: + return orjson.loads(f.read()) + except FileNotFoundError: + return {"tasks": {}} + + +@lru_cache(maxsize=None) +def get_live_tasks() -> dict[str, dict[str, list[str]]]: + """The test tasks scheduled on autoland running each group, by config.""" + return testconfigselect.live_tasks( + { + label: testconfigselect.LiveTask(task["config"], task["manifests"]) + for label, task in _get_config_selection_tasks()["tasks"].items() + } + ) + + @lru_cache(maxsize=None) def get_live_config_candidates() -> dict[str, list[str]]: """The configs of the tasks scheduled on autoland that run each group.""" - with open("manifests_by_task.json", "rb") as f: - manifests_by_task = orjson.loads(f.read()) - model = MODEL_CACHE.get(CONFIG_SELECTION_MODEL) - return testconfigselect.live_candidates(manifests_by_task, model.known_configs) + return testconfigselect.live_candidates(get_live_tasks()) @lru_cache(maxsize=None) diff --git a/http_service/pyproject.toml b/http_service/pyproject.toml index 502c97e8b36..a4b304f3cbc 100644 --- a/http_service/pyproject.toml +++ b/http_service/pyproject.toml @@ -25,6 +25,7 @@ dependencies = [ "gunicorn>=25.1,<26.3", "kombu>=5.6.2,<5.8.0", "marshmallow>=3.18.0", + "pyyaml>=6.0.3", "rq>=2.7,<2.13", "rq-dashboard>=0.8.6,<0.10.0", "sentry-sdk[flask]>=2.60.0", diff --git a/http_service/tests/conftest.py b/http_service/tests/conftest.py index da196a0ef5e..e9ff85b0899 100644 --- a/http_service/tests/conftest.py +++ b/http_service/tests/conftest.py @@ -328,10 +328,20 @@ def mock_component_taskcluster_artifact() -> None: ] -MOCK_MANIFESTS_BY_TASK = { - "test-linux1804-64/opt-mochitest-1": ["test-group1", "test-group2"], - "test-windows10/debug-mochitest-1": ["test-group1"], - "test-windows10/opt-mochitest-2": ["test-group1", "test-group2"], +# The test tasks among which configs are selected (see boot.get_config_selection_tasks). +MOCK_CONFIG_SELECTION_TASKS = { + "test-linux1804-64/opt-mochitest": { + "config": "test-linux1804-64/opt-*", + "manifests": ["test-group1", "test-group2"], + }, + "test-windows10/debug-mochitest": { + "config": "test-windows10/debug-*", + "manifests": ["test-group1"], + }, + "test-windows10/opt-mochitest": { + "config": "test-windows10/opt-*", + "manifests": ["test-group1", "test-group2"], + }, } @@ -364,8 +374,11 @@ def get(self, model_name): @pytest.fixture(autouse=True) def mock_config_selection_data(monkeypatch: MonkeyPatch, mock_data: None) -> None: - with open("manifests_by_task.json", "w") as f: - json.dump(MOCK_MANIFESTS_BY_TASK, f) + with open("config_selection_tasks.json", "w") as f: + json.dump( + {"backstop": "backstop-rev", "tasks": MOCK_CONFIG_SELECTION_TASKS}, + f, + ) with open("known_tasks", "w") as f: f.write("\n".join(MOCK_KNOWN_TASKS)) @@ -373,6 +386,8 @@ def mock_config_selection_data(monkeypatch: MonkeyPatch, mock_data: None) -> Non if os.path.exists("build_platforms.json"): os.remove("build_platforms.json") bugbug_http.models.get_known_tasks.cache_clear() + bugbug_http.models._get_config_selection_tasks.cache_clear() + bugbug_http.models.get_live_tasks.cache_clear() bugbug_http.models.get_live_config_candidates.cache_clear() bugbug_http.models.get_build_platforms.cache_clear() diff --git a/http_service/tests/test_boot.py b/http_service/tests/test_boot.py index 044c270f870..3867928b0c2 100644 --- a/http_service/tests/test_boot.py +++ b/http_service/tests/test_boot.py @@ -8,6 +8,9 @@ from pathlib import Path import pytest +import requests +import responses +import tenacity from bugbug_http import boot, refresh @@ -84,3 +87,170 @@ def sleep(seconds: float) -> None: with pytest.raises(Stop): refresh.main() assert calls == ["refresh", "refresh"] + + +def _task( + label: str, + platform: str, + manifests: list[str], + optimization: dict | None = None, + runtime: dict | None = None, +) -> dict: + return { + "label": label, + "attributes": {"test_manifests": manifests}, + "optimization": optimization or {"test": ["firefox", "mochitest"]}, + "task": { + "extra": { + "treeherder-platform": platform, + "suite": "mochitest-plain", + "test-setting": {"runtime": runtime or {}}, + } + }, + } + + +TASK_GRAPH = { + "a": _task( + "test-linux2404-64/opt-mochitest-plain-1", "linux2404-64/opt", ["dom/a.toml"] + ), + "b": _task( + "test-linux2404-64/opt-mochitest-plain-2", "linux2404-64/opt", ["dom/b.toml"] + ), + # Shippable tasks are too costly: their non-shippable equivalents are selected instead. + "c": _task( + "test-linux2404-64-shippable/opt-mochitest-plain-1", + "linux2404-64-shippable/opt", + ["dom/a.toml"], + ), + "d": _task( + "test-windows11-64-25h2-asan/opt-mochitest-plain-1", + "windows11-64-25h2/asan", + ["dom/a.toml"], + ), + # The variants come from the test settings. + "h": _task( + "test-linux2404-64/opt-mochitest-plain-xorig-1", + "linux2404-64/opt", + ["dom/a.toml"], + runtime={"xorigin": True}, + ), + # Not optimized by test selection. + "e": _task( + "test-linux2404-64/opt-mochitest-plain-standalone-1", + "linux2404-64/opt", + ["dom/a.toml"], + {"skip-unless-backstop": None}, + ), + # Not a target task of the recent pushes. + "f": _task( + "test-linux2404-64/debug-mochitest-plain-1", + "linux2404-64/debug", + ["dom/a.toml"], + ), + # Not a test task running manifests. + "g": { + "label": "build-linux64/opt", + "attributes": {}, + "optimization": None, + "task": {}, + }, +} + +KNOWN_TASKS = { + "test-linux2404-64/opt-mochitest-plain-1", + "test-linux2404-64/opt-mochitest-plain-2", + "test-linux2404-64-shippable/opt-mochitest-plain-3", + "test-windows11-64-25h2-asan/opt-mochitest-plain-1", + "test-linux2404-64/opt-mochitest-plain-xorig-1", + "test-linux2404-64/opt-mochitest-plain-standalone-1", + "build-linux64/opt", +} + +CONFIG_SELECTION_TASKS = { + # The manifests of all the chunks. + "test-linux2404-64/opt-mochitest-plain": { + "config": "test-linux2404-64/opt-*", + "manifests": ["dom/a.toml", "dom/b.toml"], + }, + "test-windows11-64-25h2-asan/opt-mochitest-plain": { + "config": "test-windows11-64-25h2/asan-*", + "manifests": ["dom/a.toml"], + }, + "test-linux2404-64/opt-mochitest-plain-xorig": { + "config": "test-linux2404-64/opt-*-xorig", + "manifests": ["dom/a.toml"], + }, +} + + +def assert_requests_have_timeouts() -> None: + """Stalled requests can't block the boot or the refresh (mozci's own requests aside).""" + calls = [ + call + for call in responses.calls + if not call.request.url.startswith("https://hg.mozilla.org/mozilla-central/") + ] + assert calls + for call in calls: + assert call.request.req_kwargs["timeout"] == boot.REQUEST_TIMEOUT, ( + call.request.url + ) + + +@pytest.fixture(autouse=True) +def mock_test_variants() -> None: + # mozci retrieves the suffixes of the test variants. + responses.add( + responses.GET, + "https://hg.mozilla.org/mozilla-central/raw-file/default/taskcluster/test_configs/variants.yml", + body="xorigin:\n suffix: xorig\n", + ) + + +def test_get_config_selection_tasks() -> None: + assert ( + boot.get_config_selection_tasks(TASK_GRAPH, KNOWN_TASKS) + == CONFIG_SELECTION_TASKS + ) + + +def test_retrieve_schedulable_tasks( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + # A transient failure is retried. + responses.add( + responses.GET, + "https://hg.mozilla.org/integration/autoland/json-pushes", + body=requests.ConnectionError("connection reset"), + ) + responses.add( + responses.GET, + "https://hg.mozilla.org/integration/autoland/json-pushes", + json={"pushes": {"1": {"changesets": ["rev1"]}, "2": {"changesets": ["rev2"]}}}, + ) + for rev, backstop in (("rev1", True), ("rev2", False)): + responses.add( + responses.GET, + boot._decision_artifact_url(rev, "target-tasks.json"), + json=sorted(KNOWN_TASKS), + ) + responses.add( + responses.GET, + boot._decision_artifact_url(rev, "parameters.yml"), + body=f"backstop: {'true' if backstop else 'false'}\n", + ) + # The tasks come from the latest backstop push. + responses.add( + responses.GET, + boot._decision_artifact_url("rev1", "task-graph.json"), + json=TASK_GRAPH, + ) + boot.retrieve_schedulable_tasks.retry_with(wait=tenacity.wait_none())() + assert_requests_have_timeouts() + + with open("known_tasks") as f: + assert set(f.read().split("\n")) == KNOWN_TASKS + with open("config_selection_tasks.json") as f: + assert json.load(f) == {"backstop": "rev1", "tasks": CONFIG_SELECTION_TASKS} diff --git a/tests/test_testconfigselect.py b/tests/test_testconfigselect.py index 516a008bf8c..7ca4b0b91f4 100644 --- a/tests/test_testconfigselect.py +++ b/tests/test_testconfigselect.py @@ -130,38 +130,42 @@ def test_generate_rows() -> None: ] -def test_live_candidates() -> None: - known_configs = { - "test-linux2404-64/opt-*", - "test-linux2404-64/opt-*-swr", - "test-macosx1015-64-qr/opt-*", - } - manifests_by_task: dict[str, list[str]] = { - "test-linux2404-64/opt-mochitest-browser-chrome-1": [G2], - "test-linux2404-64/opt-mochitest-browser-chrome-swr-2": [G2], - # A platform not in the training data yet: named after its label. - "test-macosx1500-aarch64/opt-mochitest-browser-chrome-3": [G2], - "test-linux2404-64/opt-mochitest-plain-4": [f"{G}:other.toml"], - # Web-platform test groups are relative to the root of their suite. - "test-linux2404-64/opt-web-platform-tests-5": [ - "/css/cssom-view", - "/_mozilla/fetch", - ], - "build-linux64/opt": [], +def test_live_tasks() -> None: + LiveTask = testconfigselect.LiveTask + tasks = { + "test-linux2404-64/opt-mochitest-browser-chrome": LiveTask( + "test-linux2404-64/opt-*", [G2] + ), + # The configs of some tasks aren't named after the platform of their label. + "test-windows11-64-25h2-asan/opt-mochitest-browser-chrome-swr": LiveTask( + "test-windows11-64-25h2/asan-*-swr", [G2] + ), + # Groups are named as by the models. + "test-linux2404-64/opt-web-platform-tests": LiveTask( + "test-linux2404-64/opt-*", ["/css/cssom-view", f"{G}:other.toml"] + ), } - assert testconfigselect.live_candidates(manifests_by_task, known_configs) == { - # The most specific known pattern names each task; the old macOS config isn't a candidate - # as no current task matches it. - G2: [ - "test-linux2404-64/opt-*", - "test-linux2404-64/opt-*-swr", - "test-macosx1500-aarch64/opt-mochitest-browser-chrome", - ], - G: ["test-linux2404-64/opt-*"], - "testing/web-platform/tests/css/cssom-view": ["test-linux2404-64/opt-*"], - "testing/web-platform/mozilla/tests/fetch": ["test-linux2404-64/opt-*"], + tasks_by_group = testconfigselect.live_tasks(tasks) + assert tasks_by_group == { + G2: { + "test-linux2404-64/opt-*": [ + "test-linux2404-64/opt-mochitest-browser-chrome" + ], + "test-windows11-64-25h2/asan-*-swr": [ + "test-windows11-64-25h2-asan/opt-mochitest-browser-chrome-swr" + ], + }, + "testing/web-platform/tests/css/cssom-view": { + "test-linux2404-64/opt-*": ["test-linux2404-64/opt-web-platform-tests"] + }, + G: {"test-linux2404-64/opt-*": ["test-linux2404-64/opt-web-platform-tests"]}, } + assert testconfigselect.live_candidates(tasks_by_group)[G2] == [ + "test-linux2404-64/opt-*", + "test-windows11-64-25h2/asan-*-swr", + ] + def synthetic_regressions() -> list[ConfigSelectRegression]: # G fails on Windows two times out of three, on Linux otherwise; G2 always on Mac. diff --git a/uv.lock b/uv.lock index cf319b9699f..6d67aaa9a40 100644 --- a/uv.lock +++ b/uv.lock @@ -819,6 +819,7 @@ dependencies = [ { name = "gunicorn" }, { name = "kombu" }, { name = "marshmallow" }, + { name = "pyyaml" }, { name = "rq" }, { name = "rq-dashboard" }, { name = "sentry-sdk", extra = ["flask"] }, @@ -842,6 +843,7 @@ requires-dist = [ { name = "gunicorn", specifier = ">=25.1,<26.3" }, { name = "kombu", specifier = ">=5.6.2,<5.8.0" }, { name = "marshmallow", specifier = ">=3.18.0" }, + { name = "pyyaml", specifier = ">=6.0.3" }, { name = "rq", specifier = ">=2.7,<2.13" }, { name = "rq-dashboard", specifier = ">=0.8.6,<0.10.0" }, { name = "sentry-sdk", extras = ["flask"], specifier = ">=2.60.0" }, From dc6de6fc35ba3ac23193913a4665e4f3a7f66f5f Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 00:53:21 +0100 Subject: [PATCH 02/10] Use an older backstop push's tasks if the graph of a newer one isn't available or has none The decision task of the latest backstop push may not have published its graph yet (or ever, if it failed), and its graph may have none of the tasks among which configs are selected. --- http_service/bugbug_http/boot.py | 6 +++--- http_service/tests/test_boot.py | 17 ++++++++++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index e58db5cd99f..4ccc1b27711 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -133,7 +133,8 @@ def update_config_selection_tasks(revs: list[str]) -> None: The tasks come from the task graph of the latest backstop push, which runs all of them with all their manifests (the manifests of the test tasks of other pushes are restricted to the ones - selected by bugbug). + selected by bugbug). Older backstop pushes are used if the graph of a newer one isn't + available or has none of the tasks. `revs` are the recent autoland pushes, from the oldest. """ @@ -144,7 +145,7 @@ def update_config_selection_tasks(revs: list[str]) -> None: r = _get(rev, "task-graph.json") targets = _get(rev, "target-tasks.json") if not r.ok or not targets.ok: - break + continue tasks = get_config_selection_tasks(orjson.loads(r.content), targets.json()) logger.info( "Retrieved %d test tasks from the backstop push %s", len(tasks), rev @@ -155,7 +156,6 @@ def update_config_selection_tasks(revs: list[str]) -> None: orjson.dumps({"backstop": rev, "tasks": tasks}).decode(), ) return - break logger.warning("No backstop push with test tasks to retrieve") diff --git a/http_service/tests/test_boot.py b/http_service/tests/test_boot.py index 3867928b0c2..38bd7528ef2 100644 --- a/http_service/tests/test_boot.py +++ b/http_service/tests/test_boot.py @@ -228,9 +228,15 @@ def test_retrieve_schedulable_tasks( responses.add( responses.GET, "https://hg.mozilla.org/integration/autoland/json-pushes", - json={"pushes": {"1": {"changesets": ["rev1"]}, "2": {"changesets": ["rev2"]}}}, + json={ + "pushes": { + "1": {"changesets": ["rev1"]}, + "2": {"changesets": ["rev2"]}, + "3": {"changesets": ["rev3"]}, + } + }, ) - for rev, backstop in (("rev1", True), ("rev2", False)): + for rev, backstop in (("rev1", True), ("rev2", True), ("rev3", False)): responses.add( responses.GET, boot._decision_artifact_url(rev, "target-tasks.json"), @@ -241,7 +247,12 @@ def test_retrieve_schedulable_tasks( boot._decision_artifact_url(rev, "parameters.yml"), body=f"backstop: {'true' if backstop else 'false'}\n", ) - # The tasks come from the latest backstop push. + # The tasks come from the latest backstop push with some: the graph of rev2 has none. + responses.add( + responses.GET, + boot._decision_artifact_url("rev2", "task-graph.json"), + json={"g": TASK_GRAPH["g"]}, + ) responses.add( responses.GET, boot._decision_artifact_url("rev1", "task-graph.json"), From ec9f768e37ede7f5ab64fec2b59c62a4584c50a1 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 00:11:18 +0100 Subject: [PATCH 03/10] Give the taskgraph the labels of the tasks of the selected configs The taskgraph matched the selected configs with the labels of its tasks, so it couldn't run the configs of tasks whose label doesn't have their Treeherder platform (e.g. the Windows ASan ones), and it also ran the variants of a selected config (e.g. test-linux2404-64/opt-* matching the swr tasks). The replies now give it the labels (without chunk number) of the tasks of the selected configs, and the selected configs themselves in "config_group_configs". --- bugbug/models/testconfigselect.py | 13 ++++++++++++ http_service/bugbug_http/models.py | 15 +++++++++++++- http_service/tests/test_schedule_tests.py | 25 +++++++++++++++++++---- tests/test_testconfigselect.py | 10 +++++++++ 4 files changed, 58 insertions(+), 5 deletions(-) diff --git a/bugbug/models/testconfigselect.py b/bugbug/models/testconfigselect.py index eb9cebe7dc9..a65a723f925 100644 --- a/bugbug/models/testconfigselect.py +++ b/bugbug/models/testconfigselect.py @@ -350,6 +350,19 @@ def live_candidates( return {group: sorted(configs) for group, configs in tasks_by_group.items()} +def task_labels( + tasks_by_group: Mapping[str, Mapping[str, Sequence[str]]], + group: str, + configs: Iterable[str], +) -> list[str]: + """The labels (without chunk number) of the tasks running a group on the given configs. + + The taskgraph matches them with the labels of its tasks. `tasks_by_group` comes from live_tasks. + """ + tasks = tasks_by_group.get(group, {}) + return sorted({label for config in configs for label in tasks.get(config, ())}) + + class GroupCandidates(NamedTuple): group: str candidates: Sequence[str] diff --git a/http_service/bugbug_http/models.py b/http_service/bugbug_http/models.py index dce95e8f33c..2190b609896 100644 --- a/http_service/bugbug_http/models.py +++ b/http_service/bugbug_http/models.py @@ -910,6 +910,7 @@ def _analyze_patch( "tasks": {}, "groups": {}, "config_groups": {}, + "config_group_configs": {}, "reduced_tasks": {}, "reduced_tasks_higher": {}, "known_tasks": get_known_tasks(), @@ -1035,7 +1036,19 @@ def _analyze_patch( data = { "tasks": tasks, "groups": groups, - "config_groups": config_groups, + # The taskgraph matches the labels (without chunk number) of its tasks with the ones of the + # tasks of the selected configs (a group without any runs on all configs, see + # select_configs). + "config_groups": { + group: ( + "*" not in configs + and testconfigselect.task_labels(get_live_tasks(), group, configs) + ) + or ["*"] + for group, configs in config_groups.items() + }, + # The selected configs, as the models (and mozci) name them. + "config_group_configs": config_groups, "reduced_tasks": {t: c for t, c in tasks.items() if t in reduced}, "reduced_tasks_higher": {t: c for t, c in tasks.items() if t in reduced_higher}, "known_tasks": known_tasks, diff --git a/http_service/tests/test_schedule_tests.py b/http_service/tests/test_schedule_tests.py index 05ee8554568..dc0db7a90ea 100644 --- a/http_service/tests/test_schedule_tests.py +++ b/http_service/tests/test_schedule_tests.py @@ -17,7 +17,16 @@ from bugbug.models import testselect from bugbug_http import models -from .conftest import MOCK_KNOWN_TASKS, MockModelCache +from .conftest import MOCK_CONFIG_SELECTION_TASKS, MOCK_KNOWN_TASKS, MockModelCache + + +def task_labels(group: str, configs: list[str]) -> list[str]: + """The labels of the mock tasks running the group on the configs.""" + return sorted( + label + for label, task in MOCK_CONFIG_SELECTION_TASKS.items() + if task["config"] in configs and group in task["manifests"] + ) @pytest.mark.parametrize( @@ -105,7 +114,7 @@ def test_simple_schedule( value = models.redis.get(f"bugbug:job_result:schedule_tests:mozilla-central_{rev}") assert value is not None result = orjson.loads(zstandard.ZstdDecompressor().decompress(value)) - assert len(result) == 7 + assert len(result) == 8 assert result["tasks"] == labels_to_choose assert result["groups"] == groups_to_choose assert result["reduced_tasks"] == reduced_labels @@ -113,9 +122,13 @@ def test_simple_schedule( assert result["known_tasks"] == MOCK_KNOWN_TASKS # The mock models don't have confidence thresholds. assert result["confidence_thresholds"] == {} - assert {k: set(v) for k, v in result["config_groups"].items()} == { + assert {k: set(v) for k, v in result["config_group_configs"].items()} == { k: set(v) for k, v in config_groups.items() } + # The taskgraph is given the labels of the tasks of the selected configs. + assert result["config_groups"] == { + group: task_labels(group, configs) for group, configs in config_groups.items() + } def test_schedule_with_confidence_thresholds( @@ -566,10 +579,14 @@ def select_configs(groups, modified_paths, very_likely_groups, used_configs=()): ({"test-group1": 0.99}, []), ({"test-group2": group2_confidence}, ["test-linux1804-64/opt-*"]), ] - assert result["config_groups"] == { + assert result["config_group_configs"] == { "test-group1": ["test-linux1804-64/opt-*"], "test-group2": ["test-windows10/opt-*"], } + assert result["config_groups"] == { + "test-group1": ["test-linux1804-64/opt-mochitest"], + "test-group2": ["test-windows10/opt-mochitest"], + } def test_select_configs_never_vetoes_groups(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/test_testconfigselect.py b/tests/test_testconfigselect.py index 7ca4b0b91f4..9005e8e4b94 100644 --- a/tests/test_testconfigselect.py +++ b/tests/test_testconfigselect.py @@ -165,6 +165,16 @@ def test_live_tasks() -> None: "test-linux2404-64/opt-*", "test-windows11-64-25h2/asan-*-swr", ] + assert testconfigselect.task_labels( + tasks_by_group, + G2, + ["test-windows11-64-25h2/asan-*-swr", "test-linux2404-64/opt-*"], + ) == [ + "test-linux2404-64/opt-mochitest-browser-chrome", + "test-windows11-64-25h2-asan/opt-mochitest-browser-chrome-swr", + ] + # Configs without tasks running the group (e.g. of another group) have no tasks. + assert testconfigselect.task_labels(tasks_by_group, G, ["test-other/opt-*"]) == [] def synthetic_regressions() -> list[ConfigSelectRegression]: From b8f1065f75371300206e47dcaff615bcf53e5661 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Thu, 19 Nov 2026 23:11:58 +0100 Subject: [PATCH 04/10] Don't compute and store the configs seen recently with the config selection models They were only used to name the tasks of the current task graph, which are now named after their configurations. --- bugbug/models/testconfigdimensions.py | 6 ++---- bugbug/models/testconfigselect.py | 18 +++++++----------- bugbug/test_scheduling.py | 17 ++++------------- http_service/tests/conftest.py | 5 ----- tests/test_test_scheduling.py | 14 ++++---------- tests/test_testconfigdimensions.py | 4 ++-- tests/test_testconfigselect.py | 2 +- 7 files changed, 20 insertions(+), 46 deletions(-) diff --git a/bugbug/models/testconfigdimensions.py b/bugbug/models/testconfigdimensions.py index cfd78037767..8f213eb7860 100644 --- a/bugbug/models/testconfigdimensions.py +++ b/bugbug/models/testconfigdimensions.py @@ -336,10 +336,9 @@ def __init__(self, lemmatization=False): Model.__init__(self, lemmatization) self.training_dbs = [test_scheduling.PUSH_DATA_CONFIG_GROUP_DB] self.history = DimensionHistory() - self.known_configs: set[str] = set() def train(self, importance_cutoff=0.15, limit=None): - regressions, self.known_configs, _ = load_regressions(with_patches=False) + regressions, _ = load_regressions(with_patches=False) split = split_push_index(regressions) self.history = DimensionHistory() @@ -387,7 +386,6 @@ def __init__(self, lemmatization=False): self.entire_dataset_training = True self.history = DimensionHistory() - self.known_configs: set[str] = set() self._data: tuple | None = None self.row_weights: list[float] = [] self.row_push_index: list[int] = [] @@ -419,7 +417,7 @@ def __getstate__(self): def _load(self): if self._data is None: - regressions, self.known_configs, patches = load_regressions() + regressions, patches = load_regressions() self._data = (regressions, patches, split_push_index(regressions)) return self._data diff --git a/bugbug/models/testconfigselect.py b/bugbug/models/testconfigselect.py index a65a723f925..aefa3df0f45 100644 --- a/bugbug/models/testconfigselect.py +++ b/bugbug/models/testconfigselect.py @@ -386,11 +386,9 @@ def _plainest_cheapest(configs: Iterable[str]) -> str: def load_regressions( with_patches: bool = True, -) -> tuple[ - list[test_scheduling.ConfigSelectRegression], set[str], dict[int, PatchInfo] -]: - """The regressions of the push data, the configs seen recently, and the patch of each regression's push.""" - regressions, known_configs = test_scheduling.get_config_select_regressions( +) -> tuple[list[test_scheduling.ConfigSelectRegression], dict[int, PatchInfo]]: + """The regressions of the push data, and the patch of each regression's push.""" + regressions = test_scheduling.get_config_select_regressions( lambda: db.read(test_scheduling.PUSH_DATA_CONFIG_GROUP_DB) ) logger.info( @@ -400,7 +398,7 @@ def load_regressions( ) if not with_patches: - return regressions, known_configs, {} + return regressions, {} revs = {rev: r.push_index for r in regressions for rev in r.revs} files: dict[int, list[str]] = collections.defaultdict(list) @@ -414,7 +412,7 @@ def load_regressions( for push_index, push_files in files.items() } logger.info("Patches found for %d pushes", len(patches)) - return regressions, known_configs, patches + return regressions, patches def split_push_index( @@ -677,10 +675,8 @@ def __init__(self, lemmatization=False): self.calculate_importance = False self.entire_dataset_training = True - # Inference state: the failure history up to the end of the training data, and the configs - # seen recently (to name the tasks of the current task graph, see live_candidates). + # Inference state: the failure history up to the end of the training data. self.history = ConfigHistory() - self.known_configs: set[str] = set() self._regressions: list[test_scheduling.ConfigSelectRegression] | None = None self._patches: dict[int, PatchInfo] = {} @@ -741,7 +737,7 @@ def __getstate__(self): def _load_regressions(self) -> list[test_scheduling.ConfigSelectRegression]: if self._regressions is None: - self._regressions, self.known_configs, self._patches = load_regressions() + self._regressions, self._patches = load_regressions() return self._regressions def _split_push_index(self) -> int: diff --git a/bugbug/test_scheduling.py b/bugbug/test_scheduling.py index 5a30ee30187..9e8f1e55a7b 100644 --- a/bugbug/test_scheduling.py +++ b/bugbug/test_scheduling.py @@ -402,8 +402,7 @@ def _configs_by_group(config_groups: Iterable[Runnable]) -> dict[Group, set[str] def get_config_select_regressions( push_data_factory: Callable[[], Iterable[PushResult]], - recent_pushes: int = 1000, -) -> tuple[list[ConfigSelectRegression], set[str]]: +) -> list[ConfigSelectRegression]: """The regressions of config/group push data, with the configs they failed on and passed on. A config the regressed group ran on without failing passed, both on the culprit push and on the @@ -412,9 +411,6 @@ def get_config_select_regressions( run after the culprit failed and the failure was attributed to a later push (e.g. because the config didn't run on the culprit): with no run in between, the attribution can't tell the pushes apart, so the config didn't verifiably pass. A failure after a pass doesn't matter (as in get_non_run_negatives). - - Also returns the configs that ran in the last `recent_pushes` pushes, used to name the configs of - the tasks in the current task graph (see testconfigselect.live_candidates). """ rev_index: dict[Revision, int] = {} culprits: list[ @@ -427,7 +423,6 @@ def get_config_select_regressions( ] ] = [] failures_at: dict[int, dict[Group, set[str]]] = {} - num_pushes = 0 for index, push in enumerate(push_data_factory()): ( revisions, @@ -436,7 +431,6 @@ def get_config_select_regressions( possible_regressions, likely_regressions, ) = push - num_pushes += 1 for revision in revisions: rev_index[revision] = index if possible_regressions or likely_regressions: @@ -470,16 +464,13 @@ def get_config_select_regressions( # The first push of the window each config ran the group on. later_runs: dict[tuple[int, Group], dict[str, int]] = collections.defaultdict(dict) - recent: set[str] = set() for index, push in enumerate(push_data_factory()): groups_needed = needed.get(index) - if groups_needed is None and index < num_pushes - recent_pushes: + if groups_needed is None: continue for group, configs in _configs_by_group(push[2]).items(): - if index >= num_pushes - recent_pushes: - recent |= configs - for culprit in groups_needed.get(group, ()) if groups_needed else (): + for culprit in groups_needed.get(group, ()): for config in configs: later_runs[(culprit, group)].setdefault(config, index) @@ -505,7 +496,7 @@ def get_config_select_regressions( ) ) - return regressions, recent + return regressions def get_push_data( diff --git a/http_service/tests/conftest.py b/http_service/tests/conftest.py index e9ff85b0899..1a0b6d11752 100644 --- a/http_service/tests/conftest.py +++ b/http_service/tests/conftest.py @@ -348,11 +348,6 @@ def mock_component_taskcluster_artifact() -> None: def mock_config_select_model() -> bugbug.models.testconfigselect.TestConfigSelectModel: """A config selection model predicting 0.5 for a group's first config, 0 for the following ones.""" model = bugbug.models.testconfigselect.TestConfigSelectModel() - model.known_configs = { - "test-linux1804-64/opt-*", - "test-windows10/debug-*", - "test-windows10/opt-*", - } model._predict = lambda test_jobs: np.array( # type: ignore[method-assign] [0.0 if test_job["selected"] else 0.5 for test_job in test_jobs] ) diff --git a/tests/test_test_scheduling.py b/tests/test_test_scheduling.py index 778e50528a2..248db8dbfd9 100644 --- a/tests/test_test_scheduling.py +++ b/tests/test_test_scheduling.py @@ -1711,9 +1711,7 @@ def test_get_config_select_regressions() -> None: (["rev3"], None, [("test-windows11-64-24h2/opt-*", G)], [], []), ] - regressions, recent_configs = test_scheduling.get_config_select_regressions( - lambda: iter(pushes), recent_pushes=2 - ) + regressions = test_scheduling.get_config_select_regressions(lambda: iter(pushes)) culprit, later = regressions assert culprit == test_scheduling.ConfigSelectRegression( @@ -1723,8 +1721,6 @@ def test_get_config_select_regressions() -> None: assert later == test_scheduling.ConfigSelectRegression( 1, (Revision("rev1"),), G, frozenset({ANDROID}), frozenset({ANDROID, MAC}) ) - # The configs that ran in the last two pushes. - assert recent_configs == {LINUX, "test-windows11-64-24h2/opt-*"} def test_get_config_select_regressions_failure_after_later_pass() -> None: @@ -1747,7 +1743,7 @@ def test_get_config_select_regressions_failure_after_later_pass() -> None: (["rev4"], None, [], [], []), ] - regressions, _ = test_scheduling.get_config_select_regressions(lambda: iter(pushes)) + regressions = test_scheduling.get_config_select_regressions(lambda: iter(pushes)) assert regressions[0] == test_scheduling.ConfigSelectRegression( 0, (Revision("rev0"),), G, frozenset({WIN}), frozenset({WIN, LINUX}) @@ -1767,7 +1763,7 @@ def test_get_config_select_regressions_ignored_configs_only() -> None: (["rev1"], None, [(VERIFY, G)], [(VERIFY, G)], []), ] - regressions, _ = test_scheduling.get_config_select_regressions(lambda: iter(pushes)) + regressions = test_scheduling.get_config_select_regressions(lambda: iter(pushes)) assert regressions == [ test_scheduling.ConfigSelectRegression( @@ -1790,9 +1786,7 @@ def test_get_config_select_regressions_window_is_capped(monkeypatch) -> None: (["rev4"], None, [], [], []), ] - (regression,), _ = test_scheduling.get_config_select_regressions( - lambda: iter(pushes) - ) + (regression,) = test_scheduling.get_config_select_regressions(lambda: iter(pushes)) assert regression.observed == frozenset({A, B}) diff --git a/tests/test_testconfigdimensions.py b/tests/test_testconfigdimensions.py index 4c110653f59..ac8b26825ee 100644 --- a/tests/test_testconfigdimensions.py +++ b/tests/test_testconfigdimensions.py @@ -149,7 +149,7 @@ def mock_data(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( testconfigdimensions, "load_regressions", - lambda with_patches=True: (regressions, {LINUX, LINUX_DEBUG, WINDOWS, MAC}, {}), + lambda with_patches=True: (regressions, {}), ) @@ -224,7 +224,7 @@ def test_class_model_class_only_in_test_set(monkeypatch) -> None: monkeypatch.setattr( testconfigdimensions, "load_regressions", - lambda with_patches=True: (regressions, {LINUX, WINDOWS, MAC, android}, {}), + lambda with_patches=True: (regressions, {}), ) model = testconfigdimensions.TestConfigSelectClassModel() classes, class_names = model.get_labels() diff --git a/tests/test_testconfigselect.py b/tests/test_testconfigselect.py index 9005e8e4b94..bb215927721 100644 --- a/tests/test_testconfigselect.py +++ b/tests/test_testconfigselect.py @@ -203,7 +203,7 @@ def trained_model( monkeypatch.setattr( testconfigselect.test_scheduling, "get_config_select_regressions", - lambda push_data_factory: (regressions, {LINUX, LINUX_DEBUG, WINDOWS, MAC}), + lambda push_data_factory: regressions, ) monkeypatch.setattr( testconfigselect.repository, From 1dbb79bdf9eba802c4dfbdf384848f847311c0ff Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 00:12:29 +0100 Subject: [PATCH 05/10] Retrieve the definitions of some tasks from a full task graph without parsing all of it A decision task's full-task-graph.json is hundreds of MB (and much more once parsed), so only the definitions of the given tasks are parsed, where they are in the text. If one isn't found this way (e.g. if the format of the file changed), the whole graph is parsed. --- http_service/bugbug_http/boot.py | 31 +++++++++++++++++++++++++++++++ http_service/tests/test_boot.py | 19 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index 4ccc1b27711..41af79f8831 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -124,6 +124,37 @@ def get_config_selection_tasks( return tasks +def get_full_graph_tasks(content: bytes, labels: set[str]) -> dict[str, dict]: + """The definitions of some tasks from a decision task's full-task-graph.json, by label. + + The full task graph is hundreds of MB, and much more once parsed, so only the definitions of + the given tasks are parsed, where they are in the text. If one isn't found this way (e.g. if + the format of the file changed), the whole graph is parsed. + """ + text = content.decode() + decoder = json.JSONDecoder() + tasks: dict[str, dict] = {} + for label in labels: + key = f"{json.dumps(label)}:{{" + start = text.find(key) + while start != -1: + try: + task, _ = decoder.raw_decode(text, start + len(key) - 1) + except ValueError: + task = None + if isinstance(task, dict) and task.get("label") == label: + tasks[label] = task + break + start = text.find(key, start + 1) + + missing = labels - tasks.keys() + if missing: + del text + graph = orjson.loads(content) + tasks.update({label: graph[label] for label in missing if label in graph}) + return tasks + + def _get(rev: str, path: str) -> requests.Response: return requests.get(_decision_artifact_url(rev, path), timeout=REQUEST_TIMEOUT) diff --git a/http_service/tests/test_boot.py b/http_service/tests/test_boot.py index 38bd7528ef2..cd842d910c2 100644 --- a/http_service/tests/test_boot.py +++ b/http_service/tests/test_boot.py @@ -7,6 +7,7 @@ import os from pathlib import Path +import orjson import pytest import requests import responses @@ -265,3 +266,21 @@ def test_retrieve_schedulable_tasks( assert set(f.read().split("\n")) == KNOWN_TASKS with open("config_selection_tasks.json") as f: assert json.load(f) == {"backstop": "rev1", "tasks": CONFIG_SELECTION_TASKS} + + +def test_get_full_graph_tasks() -> None: + graph = {"a": TASK_GRAPH["a"], "c": TASK_GRAPH["c"], "g": TASK_GRAPH["g"]} + graph = {task["label"]: task for task in graph.values()} + labels = {"test-linux2404-64/opt-mochitest-plain-1", "build-linux64/opt"} + expected = {label: graph[label] for label in labels} + # The definitions are found in the text, as the full task graph is compact JSON. + assert boot.get_full_graph_tasks(orjson.dumps(graph), labels) == expected + # Otherwise, the whole graph is parsed. + assert ( + boot.get_full_graph_tasks(json.dumps(graph, indent=2).encode(), labels) + == expected + ) + # Tasks which aren't in the graph are left out. + assert ( + boot.get_full_graph_tasks(orjson.dumps(graph), {"test-unknown/opt-x-1"}) == {} + ) From d11730a9d82cbeb30738eb91f45d9ff5dcbb94ab Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 00:15:37 +0100 Subject: [PATCH 06/10] Give the taskgraph the labels of new tasks for the groups selected on their configs Tasks can be renamed after the task graph the config selection tasks come from. The tasks whose manifests aren't known, with their configs, are given for all the groups selected on their configs, as the taskgraph only runs a task for a group if the task runs it. --- bugbug/models/testconfigselect.py | 11 +++++- http_service/bugbug_http/models.py | 12 +++++- http_service/tests/conftest.py | 7 +++- http_service/tests/test_schedule_tests.py | 47 +++++++++++++++++++++++ tests/test_testconfigselect.py | 11 ++++++ 5 files changed, 84 insertions(+), 4 deletions(-) diff --git a/bugbug/models/testconfigselect.py b/bugbug/models/testconfigselect.py index aefa3df0f45..5756d6097de 100644 --- a/bugbug/models/testconfigselect.py +++ b/bugbug/models/testconfigselect.py @@ -354,13 +354,22 @@ def task_labels( tasks_by_group: Mapping[str, Mapping[str, Sequence[str]]], group: str, configs: Iterable[str], + new_tasks: Mapping[str, str] | None = None, ) -> list[str]: """The labels (without chunk number) of the tasks running a group on the given configs. The taskgraph matches them with the labels of its tasks. `tasks_by_group` comes from live_tasks. + `new_tasks` are tasks whose manifests aren't known (e.g. renamed since the task graph the others + come from), by label, with their config: they are given for every group selected on their config, + as the taskgraph only runs a task for a group if the task runs it. """ + configs = set(configs) tasks = tasks_by_group.get(group, {}) - return sorted({label for config in configs for label in tasks.get(config, ())}) + labels = {label for config in configs for label in tasks.get(config, ())} + labels |= { + label for label, config in (new_tasks or {}).items() if config in configs + } + return sorted(labels) class GroupCandidates(NamedTuple): diff --git a/http_service/bugbug_http/models.py b/http_service/bugbug_http/models.py index 2190b609896..f97c72e5ea2 100644 --- a/http_service/bugbug_http/models.py +++ b/http_service/bugbug_http/models.py @@ -576,7 +576,7 @@ def _get_config_selection_tasks() -> dict: with open("config_selection_tasks.json", "rb") as f: return orjson.loads(f.read()) except FileNotFoundError: - return {"tasks": {}} + return {"tasks": {}, "new_tasks": {}} @lru_cache(maxsize=None) @@ -590,6 +590,12 @@ def get_live_tasks() -> dict[str, dict[str, list[str]]]: ) +@lru_cache(maxsize=None) +def get_new_tasks() -> dict[str, str]: + """The test tasks of the latest pushes which aren't in get_live_tasks, with their configs.""" + return _get_config_selection_tasks()["new_tasks"] + + @lru_cache(maxsize=None) def get_live_config_candidates() -> dict[str, list[str]]: """The configs of the tasks scheduled on autoland that run each group.""" @@ -1042,7 +1048,9 @@ def _analyze_patch( "config_groups": { group: ( "*" not in configs - and testconfigselect.task_labels(get_live_tasks(), group, configs) + and testconfigselect.task_labels( + get_live_tasks(), group, configs, get_new_tasks() + ) ) or ["*"] for group, configs in config_groups.items() diff --git a/http_service/tests/conftest.py b/http_service/tests/conftest.py index 1a0b6d11752..e348b255ef5 100644 --- a/http_service/tests/conftest.py +++ b/http_service/tests/conftest.py @@ -371,7 +371,11 @@ def get(self, model_name): def mock_config_selection_data(monkeypatch: MonkeyPatch, mock_data: None) -> None: with open("config_selection_tasks.json", "w") as f: json.dump( - {"backstop": "backstop-rev", "tasks": MOCK_CONFIG_SELECTION_TASKS}, + { + "backstop": "backstop-rev", + "tasks": MOCK_CONFIG_SELECTION_TASKS, + "new_tasks": {}, + }, f, ) with open("known_tasks", "w") as f: @@ -383,6 +387,7 @@ def mock_config_selection_data(monkeypatch: MonkeyPatch, mock_data: None) -> Non bugbug_http.models.get_known_tasks.cache_clear() bugbug_http.models._get_config_selection_tasks.cache_clear() bugbug_http.models.get_live_tasks.cache_clear() + bugbug_http.models.get_new_tasks.cache_clear() bugbug_http.models.get_live_config_candidates.cache_clear() bugbug_http.models.get_build_platforms.cache_clear() diff --git a/http_service/tests/test_schedule_tests.py b/http_service/tests/test_schedule_tests.py index dc0db7a90ea..6f9ed7cb8a5 100644 --- a/http_service/tests/test_schedule_tests.py +++ b/http_service/tests/test_schedule_tests.py @@ -589,6 +589,53 @@ def select_configs(groups, modified_paths, very_likely_groups, used_configs=()): } +def test_schedule_config_groups_with_new_tasks( + monkeypatch: pytest.MonkeyPatch, + mock_hgmo: None, + mock_repo: tuple[str, str], + mock_component_taskcluster_artifact: None, + mock_coverage_mapping_artifact: None, + mock_schedule_tests_classify: Callable[[dict[str, float], dict[str, float]], None], +) -> None: + _, remote_repo_dir = mock_repo + with hglib.open(str(remote_repo_dir)) as hg: + rev = hg.log()[0].node.decode("ascii")[:12] + + mock_schedule_tests_classify({}, {"test-group1": 0.99}) + + # A task renamed since the backstop push the other tasks come from. + with open("config_selection_tasks.json", "w") as f: + json.dump( + { + "backstop": "backstop-rev", + "tasks": MOCK_CONFIG_SELECTION_TASKS, + "new_tasks": { + "test-linux1804-64/opt-mochitest-renamed": "test-linux1804-64/opt-*" + }, + }, + f, + ) + monkeypatch.setattr( + models, + "select_configs", + lambda groups, modified_paths, very_likely_groups, used_configs=(): { + group: ["test-linux1804-64/opt-*"] for group in groups + }, + ) + + assert models.schedule_tests("mozilla-central", rev) == "OK" + + value = models.redis.get(f"bugbug:job_result:schedule_tests:mozilla-central_{rev}") + assert value is not None + result = orjson.loads(zstandard.ZstdDecompressor().decompress(value)) + assert result["config_groups"] == { + "test-group1": [ + "test-linux1804-64/opt-mochitest", + "test-linux1804-64/opt-mochitest-renamed", + ] + } + + def test_select_configs_never_vetoes_groups(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(models, "MODEL_CACHE", MockModelCache()) diff --git a/tests/test_testconfigselect.py b/tests/test_testconfigselect.py index bb215927721..58091556b66 100644 --- a/tests/test_testconfigselect.py +++ b/tests/test_testconfigselect.py @@ -175,6 +175,17 @@ def test_live_tasks() -> None: ] # Configs without tasks running the group (e.g. of another group) have no tasks. assert testconfigselect.task_labels(tasks_by_group, G, ["test-other/opt-*"]) == [] + # Tasks whose manifests aren't known (e.g. renamed) are given for all the groups of their config. + new_tasks = { + "test-linux2404-64/opt-mochitest-browser-chrome-renamed": "test-linux2404-64/opt-*", + "test-macosx1500-aarch64/opt-mochitest-browser-chrome": "test-macosx1500-aarch64/opt-*", + } + assert testconfigselect.task_labels( + tasks_by_group, G2, ["test-linux2404-64/opt-*"], new_tasks + ) == [ + "test-linux2404-64/opt-mochitest-browser-chrome", + "test-linux2404-64/opt-mochitest-browser-chrome-renamed", + ] def synthetic_regressions() -> list[ConfigSelectRegression]: From c1635fc8ebaa41e0c3e34b25c8c3ddda86f653de Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 00:16:02 +0100 Subject: [PATCH 07/10] Select configs on the tasks of the pushes after the backstop push too As the taskgraph matches the labels, they must be the ones of the push's tasks: the test tasks of the pushes after the backstop push which weren't in its task graph (e.g. renamed since then) are stored too. Those are found among the target tasks of the pushes (before optimization, which could remove them as they match none of the labels in the replies), with their definitions from the full task graph, which is only retrieved when there are unknown test tasks: the ones which aren't eligible are remembered. --- http_service/bugbug_http/boot.py | 111 +++++++++++++++-- http_service/tests/conftest.py | 1 + http_service/tests/test_boot.py | 141 +++++++++++++++++++++- http_service/tests/test_schedule_tests.py | 1 + 4 files changed, 244 insertions(+), 10 deletions(-) diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index 41af79f8831..693b84ffd70 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -159,7 +159,25 @@ def _get(rev: str, path: str) -> requests.Response: return requests.get(_decision_artifact_url(rev, path), timeout=REQUEST_TIMEOUT) -def update_config_selection_tasks(revs: list[str]) -> None: +def _classify_new_tasks( + state: dict, ignored: set[str], rev: str, labels: set[str], graph: dict[str, dict] +) -> None: + """Classify test tasks of a push after the backstop push. + + `labels` are the tasks to classify (without chunk number), `graph` the definitions available + for them (some can be missing, e.g. from the push's full task graph). The eligible ones become + new tasks (unless they are candidates already), and the others are ignored. + """ + eligible = {label: config for label, config, _ in _eligible_tasks(graph)} + for label in labels: + if label not in eligible: + ignored.add(label) + elif label not in state["tasks"]: + logger.info("New test task %s (%s) in push %s", label, eligible[label], rev) + state["new_tasks"][label] = eligible[label] + + +def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> None: """Retrieve the test tasks among which configs are selected. The tasks come from the task graph of the latest backstop push, which runs all of them with all @@ -167,8 +185,27 @@ def update_config_selection_tasks(revs: list[str]) -> None: selected by bugbug). Older backstop pushes are used if the graph of a newer one isn't available or has none of the tasks. - `revs` are the recent autoland pushes, from the oldest. + The tasks of the pushes after it which aren't known (e.g. because a task was renamed since the + backstop push) are stored too, with their config: the taskgraph matches the labels of its tasks + with the ones of the selected configs, which would otherwise not run on the renamed tasks. They + are found among the target tasks of the pushes (before optimization, which could remove them as + they match none of the labels in the replies), and their definitions in the full task graph + (only when there are some). Their manifests aren't known (only the ones selected by bugbug run on + those pushes), so the replies give them for all the groups the configs are selected for (see + testconfigselect.task_labels). The test tasks which aren't eligible (see _eligible_tasks) are + remembered, so that they aren't looked for again. + + `revs` are the recent autoland pushes, from the oldest. `known_tasks` are their target tasks, + only used to forget the ignored tasks no push targets anymore. """ + try: + with open(CONFIG_SELECTION_TASKS_PATH, "rb") as f: + previous_state = orjson.loads(f.read()) + except FileNotFoundError: + previous_state = None + + # The latest backstop push with tasks. + state: dict | None = None for rev in reversed(revs): r = _get(rev, "parameters.yml") if not r.ok or not yaml.safe_load(r.text).get("backstop"): @@ -177,18 +214,74 @@ def update_config_selection_tasks(revs: list[str]) -> None: targets = _get(rev, "target-tasks.json") if not r.ok or not targets.ok: continue - tasks = get_config_selection_tasks(orjson.loads(r.content), targets.json()) + task_graph = orjson.loads(r.content) + target_tasks = targets.json() + tasks = get_config_selection_tasks(task_graph, target_tasks) logger.info( "Retrieved %d test tasks from the backstop push %s", len(tasks), rev ) if tasks: - _write( - CONFIG_SELECTION_TASKS_PATH, - orjson.dumps({"backstop": rev, "tasks": tasks}).decode(), + # The test tasks which aren't eligible: the ones of the task graph which aren't among + # the tasks, and the target ones which aren't in the task graph (all the tasks that + # test selection optimizes run on backstop pushes, so the others are optimized + # otherwise, e.g. jittest when no JS engine file changed). The ones found before are + # kept, as long as they are among the target tasks of the recent pushes, as some are + # only targeted by some pushes (e.g. test-verify when tests are modified). + recent_targets = {_without_chunk(label) for label in known_tasks} + previously_ignored = ( + set(previous_state["ignored"]) if previous_state is not None else set() + ) + ignored = ( + { + _without_chunk(label) + for label in ( + *(task["label"] for task in task_graph.values()), + *target_tasks, + ) + if label.startswith("test-") + } + | (previously_ignored & recent_targets) + ) - tasks.keys() + state = { + "backstop": rev, + "tasks": tasks, + "new_tasks": {}, + "ignored": sorted(ignored), + } + break + + if state is None: + logger.warning("No backstop push with test tasks to retrieve") + return + + # The tasks of the later pushes. The ones without their target tasks or full task graph (yet, + # e.g. if the decision task is still running, or ever, e.g. for DONTBUILD pushes or failed + # decision tasks) are left for the next update, without blocking the others. + ignored = set(state["ignored"]) + for rev in revs[revs.index(state["backstop"]) + 1 :]: + r = _get(rev, "target-tasks.json") + if not r.ok: + continue + unknown = { + label + for label in r.json() + if label.startswith("test-") + and _without_chunk(label) not in state["tasks"] + and _without_chunk(label) not in state["new_tasks"] + and _without_chunk(label) not in ignored + } + if unknown: + r = _get(rev, "full-task-graph.json") + if not r.ok: + continue + graph = get_full_graph_tasks(r.content, unknown) + del r + _classify_new_tasks( + state, ignored, rev, {_without_chunk(label) for label in unknown}, graph ) - return + state["ignored"] = sorted(ignored) - logger.warning("No backstop push with test tasks to retrieve") + _write(CONFIG_SELECTION_TASKS_PATH, orjson.dumps(state).decode()) def get_recent_revs() -> list[str]: @@ -230,7 +323,7 @@ def retrieve_schedulable_tasks() -> None: _write("known_tasks", "\n".join(known_tasks)) # The test tasks among which configs are selected for the manifests. - update_config_selection_tasks(revs) + update_config_selection_tasks(revs, known_tasks) def retrieve_build_platforms() -> None: diff --git a/http_service/tests/conftest.py b/http_service/tests/conftest.py index e348b255ef5..09cd745c6e9 100644 --- a/http_service/tests/conftest.py +++ b/http_service/tests/conftest.py @@ -375,6 +375,7 @@ def mock_config_selection_data(monkeypatch: MonkeyPatch, mock_data: None) -> Non "backstop": "backstop-rev", "tasks": MOCK_CONFIG_SELECTION_TASKS, "new_tasks": {}, + "ignored": [], }, f, ) diff --git a/http_service/tests/test_boot.py b/http_service/tests/test_boot.py index cd842d910c2..ab74cd41b07 100644 --- a/http_service/tests/test_boot.py +++ b/http_service/tests/test_boot.py @@ -3,6 +3,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. +import contextlib import json import os from pathlib import Path @@ -220,6 +221,9 @@ def test_retrieve_schedulable_tasks( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: monkeypatch.chdir(tmp_path) + # No tasks were retrieved before. + with contextlib.suppress(FileNotFoundError): + os.remove(boot.CONFIG_SELECTION_TASKS_PATH) # A transient failure is retried. responses.add( responses.GET, @@ -265,7 +269,17 @@ def test_retrieve_schedulable_tasks( with open("known_tasks") as f: assert set(f.read().split("\n")) == KNOWN_TASKS with open("config_selection_tasks.json") as f: - assert json.load(f) == {"backstop": "rev1", "tasks": CONFIG_SELECTION_TASKS} + assert json.load(f) == { + "backstop": "rev1", + "tasks": CONFIG_SELECTION_TASKS, + "new_tasks": {}, + # The test tasks of the backstop push which aren't eligible. + "ignored": [ + "test-linux2404-64-shippable/opt-mochitest-plain", + "test-linux2404-64/debug-mochitest-plain", + "test-linux2404-64/opt-mochitest-plain-standalone", + ], + } def test_get_full_graph_tasks() -> None: @@ -284,3 +298,128 @@ def test_get_full_graph_tasks() -> None: assert ( boot.get_full_graph_tasks(orjson.dumps(graph), {"test-unknown/opt-x-1"}) == {} ) + + +def test_update_config_selection_tasks( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + # No tasks were retrieved before. + with contextlib.suppress(FileNotFoundError): + os.remove(boot.CONFIG_SELECTION_TASKS_PATH) + renamed = _task( + "test-linux2404-64/opt-mochitest-plain-renamed-1", + "linux2404-64/opt", + ["dom/a.toml"], + ) + renamed_shippable = _task( + "test-linux2404-64-shippable/opt-mochitest-plain-renamed-1", + "linux2404-64-shippable/opt", + ["dom/a.toml"], + ) + for rev, backstop in (("b1", True), ("p1", False), ("p2", False), ("b2", True)): + responses.add( + responses.GET, + boot._decision_artifact_url(rev, "parameters.yml"), + body=f"backstop: {'true' if backstop else 'false'}\n", + ) + responses.add( + responses.GET, + boot._decision_artifact_url("b1", "task-graph.json"), + json=TASK_GRAPH, + ) + # A target task of the backstop pushes isn't in their task graph: it's optimized otherwise + # than by test selection. + jittest = "test-linux2404-64/opt-jittest-1proc-1" + responses.add( + responses.GET, + boot._decision_artifact_url("b1", "target-tasks.json"), + json=sorted(KNOWN_TASKS) + [jittest], + ) + responses.add( + responses.GET, + boot._decision_artifact_url("b2", "target-tasks.json"), + json=sorted(KNOWN_TASKS) + [jittest, renamed["label"]], + ) + # A push after the backstop push has tasks renamed since then (found among its target tasks, + # as they could have been optimized away). Shippable ones are still ignored. + responses.add( + responses.GET, + boot._decision_artifact_url("p1", "target-tasks.json"), + json=sorted(KNOWN_TASKS) + + [jittest, renamed["label"], renamed_shippable["label"]], + ) + responses.add( + responses.GET, + boot._decision_artifact_url("p1", "full-task-graph.json"), + body=orjson.dumps( + { + task["label"]: task + for task in (*TASK_GRAPH.values(), renamed, renamed_shippable) + } + ), + ) + # The target tasks of the next one aren't there: it doesn't block the others. + responses.add( + responses.GET, + boot._decision_artifact_url("p2", "target-tasks.json"), + status=404, + ) + + def read() -> dict: + with open("config_selection_tasks.json") as f: + return json.load(f) + + ignored = [ + "test-linux2404-64-shippable/opt-mochitest-plain", + "test-linux2404-64/debug-mochitest-plain", + "test-linux2404-64/opt-jittest-1proc", + "test-linux2404-64/opt-mochitest-plain-standalone", + ] + boot.update_config_selection_tasks(["b1", "p1", "p2"], KNOWN_TASKS) + assert read() == { + "backstop": "b1", + "tasks": CONFIG_SELECTION_TASKS, + "new_tasks": { + "test-linux2404-64/opt-mochitest-plain-renamed": "test-linux2404-64/opt-*" + }, + "ignored": sorted( + ignored + ["test-linux2404-64-shippable/opt-mochitest-plain-renamed"] + ), + } + # The full task graph was only needed for p1's renamed tasks: the jittest task was already + # known not to be eligible. + full_graphs = [ + call.request.url + for call in responses.calls + if call.request.url.endswith("full-task-graph.json") + ] + assert full_graphs == [boot._decision_artifact_url("p1", "full-task-graph.json")] + + # A newer backstop push replaces the tasks, which include the renamed one, even if the target + # tasks of the recent pushes are from before the rename (they are only refreshed daily). The + # tasks found not to be eligible before are kept as long as recent pushes target them (the + # shippable renamed one isn't among the tasks or the target tasks of the new backstop push). + responses.add( + responses.GET, + boot._decision_artifact_url("b2", "task-graph.json"), + json={**TASK_GRAPH, "r": renamed}, + ) + boot.update_config_selection_tasks( + ["b1", "p1", "p2", "b2"], KNOWN_TASKS | {renamed_shippable["label"]} + ) + assert read() == { + "backstop": "b2", + "tasks": { + **CONFIG_SELECTION_TASKS, + "test-linux2404-64/opt-mochitest-plain-renamed": { + "config": "test-linux2404-64/opt-*", + "manifests": ["dom/a.toml"], + }, + }, + "new_tasks": {}, + "ignored": sorted( + ignored + ["test-linux2404-64-shippable/opt-mochitest-plain-renamed"] + ), + } + assert_requests_have_timeouts() diff --git a/http_service/tests/test_schedule_tests.py b/http_service/tests/test_schedule_tests.py index 6f9ed7cb8a5..68e4eeaf153 100644 --- a/http_service/tests/test_schedule_tests.py +++ b/http_service/tests/test_schedule_tests.py @@ -612,6 +612,7 @@ def test_schedule_config_groups_with_new_tasks( "new_tasks": { "test-linux1804-64/opt-mochitest-renamed": "test-linux1804-64/opt-*" }, + "ignored": [], }, f, ) From 1f9db69937a72912b1c18d3f21bf319779abde86 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Thu, 19 Nov 2026 23:33:19 +0100 Subject: [PATCH 08/10] Update the config selection tasks every 10 minutes The tasks of the pushes after the backstop push (e.g. renamed since then) are only useful if they are found soon after they appear, so the test tasks among which configs are selected are updated every 10 minutes in the refresh process, rather than once per day: from the latest backstop push as soon as there is a new one, and with the pushes after it which weren't processed yet. The pushes known not to be backstop pushes aren't checked again, and the file is only written when it changes. --- http_service/bugbug_http/boot.py | 62 +++++++++++++++------ http_service/bugbug_http/refresh.py | 18 +++++-- http_service/tests/conftest.py | 2 + http_service/tests/test_boot.py | 66 ++++++++++++++++++++--- http_service/tests/test_schedule_tests.py | 2 + 5 files changed, 123 insertions(+), 27 deletions(-) diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index 693b84ffd70..97d0a328567 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -30,6 +30,9 @@ # How often the data about the current tasks, build platforms and commits is retrieved again (in # seconds, see bugbug_http.refresh). REFRESH_INTERVAL = 24 * 60 * 60 +# How often the test tasks among which configs are selected are updated with the latest pushes (in +# seconds, see bugbug_http.refresh and update_config_selection_tasks). +CONFIG_SELECTION_TASKS_REFRESH_INTERVAL = 10 * 60 CONFIG_SELECTION_TASKS_PATH = "config_selection_tasks.json" # The timeouts of the requests for the current tasks (to connect, and between bytes received, in # seconds), so that a stalled request fails (and is retried) instead of blocking the boot or the @@ -178,7 +181,7 @@ def _classify_new_tasks( def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> None: - """Retrieve the test tasks among which configs are selected. + """Update the test tasks among which configs are selected with the latest pushes. The tasks come from the task graph of the latest backstop push, which runs all of them with all their manifests (the manifests of the test tasks of other pushes are restricted to the ones @@ -195,20 +198,29 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non testconfigselect.task_labels). The test tasks which aren't eligible (see _eligible_tasks) are remembered, so that they aren't looked for again. - `revs` are the recent autoland pushes, from the oldest. `known_tasks` are their target tasks, - only used to forget the ignored tasks no push targets anymore. + `revs` are the recent autoland pushes, from the oldest. `known_tasks` are their target tasks + (possibly from an earlier refresh), only used to forget the ignored tasks no push targets anymore. """ try: with open(CONFIG_SELECTION_TASKS_PATH, "rb") as f: - previous_state = orjson.loads(f.read()) + state = orjson.loads(f.read()) except FileNotFoundError: - previous_state = None + state = None + previous = orjson.dumps(state) - # The latest backstop push with tasks. - state: dict | None = None + # A1: the latest backstop push with tasks. + known_backstop = state["backstop"] if state is not None else None + not_backstops = set(state["not_backstops"]) if state is not None else set() for rev in reversed(revs): + if rev == known_backstop: + break + if rev in not_backstops: + continue r = _get(rev, "parameters.yml") - if not r.ok or not yaml.safe_load(r.text).get("backstop"): + if not r.ok: + continue + if not yaml.safe_load(r.text).get("backstop"): + not_backstops.add(rev) continue r = _get(rev, "task-graph.json") targets = _get(rev, "target-tasks.json") @@ -228,9 +240,7 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non # kept, as long as they are among the target tasks of the recent pushes, as some are # only targeted by some pushes (e.g. test-verify when tests are modified). recent_targets = {_without_chunk(label) for label in known_tasks} - previously_ignored = ( - set(previous_state["ignored"]) if previous_state is not None else set() - ) + previously_ignored = set(state["ignored"]) if state is not None else set() ignored = ( { _without_chunk(label) @@ -247,18 +257,27 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non "tasks": tasks, "new_tasks": {}, "ignored": sorted(ignored), + "processed": [], } break if state is None: logger.warning("No backstop push with test tasks to retrieve") return + state["not_backstops"] = sorted(not_backstops & set(revs)) - # The tasks of the later pushes. The ones without their target tasks or full task graph (yet, - # e.g. if the decision task is still running, or ever, e.g. for DONTBUILD pushes or failed - # decision tasks) are left for the next update, without blocking the others. ignored = set(state["ignored"]) - for rev in revs[revs.index(state["backstop"]) + 1 :]: + + # A2: the tasks of the later pushes. The ones without their target tasks or full task graph + # (yet, e.g. if the decision task is still running, or ever, e.g. for DONTBUILD pushes or failed + # decision tasks) are left for the next refresh, without blocking the others. + later = ( + revs[revs.index(state["backstop"]) + 1 :] if state["backstop"] in revs else revs + ) + processed = set(state["processed"]) + for rev in later: + if rev in processed: + continue r = _get(rev, "target-tasks.json") if not r.ok: continue @@ -279,9 +298,13 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non _classify_new_tasks( state, ignored, rev, {_without_chunk(label) for label in unknown}, graph ) + processed.add(rev) + state["processed"] = sorted(processed & set(revs)) state["ignored"] = sorted(ignored) - _write(CONFIG_SELECTION_TASKS_PATH, orjson.dumps(state).decode()) + serialized = orjson.dumps(state) + if serialized != previous: + _write(CONFIG_SELECTION_TASKS_PATH, serialized.decode()) def get_recent_revs() -> list[str]: @@ -297,6 +320,13 @@ def get_recent_revs() -> list[str]: return [push_obj["changesets"][0] for _, push_obj in pushes[-AUTOLAND_PUSHES:]] +def refresh_config_selection_tasks() -> None: + """Update the test tasks among which configs are selected (see update_config_selection_tasks).""" + with open("known_tasks", "r") as f: + known_tasks = {line.strip() for line in f} + update_config_selection_tasks(get_recent_revs(), known_tasks) + + @tenacity.retry( stop=tenacity.stop_after_attempt(7), wait=tenacity.wait_exponential(multiplier=1, min=1, max=8), diff --git a/http_service/bugbug_http/refresh.py b/http_service/bugbug_http/refresh.py index edeada1f629..e50652591cf 100644 --- a/http_service/bugbug_http/refresh.py +++ b/http_service/bugbug_http/refresh.py @@ -8,7 +8,8 @@ Workers can run for weeks, while the tasks of the task graph and the files built on each platform change (e.g. platform migrations), and new commits land (used by the features of the test selection -models). This runs next to the worker, in the same container and directory: it rewrites the worker's +models), and tasks get renamed (whose labels the replies give for the selected configs, so they are +refreshed more often). This runs next to the worker, in the same container and directory: it rewrites the worker's files (atomically) and databases (LMDB, whose readers see consistent snapshots), and jobs read them when they run. """ @@ -21,6 +22,7 @@ from bugbug_http.sentry import setup_sentry logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) if os.environ.get("SENTRY_DSN"): setup_sentry(dsn=os.environ.get("SENTRY_DSN")) @@ -30,9 +32,19 @@ def main() -> None: # Mining commits uses all the CPUs: don't slow the jobs down. os.nice(10) + # The test tasks among which configs are selected are updated much more often than the rest, + # so that the selected configs run on tasks renamed since the latest backstop push. + next_refresh = time.monotonic() + boot.REFRESH_INTERVAL while True: - time.sleep(boot.REFRESH_INTERVAL) - boot.refresh_data() + time.sleep(boot.CONFIG_SELECTION_TASKS_REFRESH_INTERVAL) + if time.monotonic() >= next_refresh: + boot.refresh_data() + next_refresh = time.monotonic() + boot.REFRESH_INTERVAL + else: + try: + boot.refresh_config_selection_tasks() + except Exception: + logger.exception("Failed to refresh the config selection tasks") if __name__ == "__main__": diff --git a/http_service/tests/conftest.py b/http_service/tests/conftest.py index 09cd745c6e9..7800774787e 100644 --- a/http_service/tests/conftest.py +++ b/http_service/tests/conftest.py @@ -376,6 +376,8 @@ def mock_config_selection_data(monkeypatch: MonkeyPatch, mock_data: None) -> Non "tasks": MOCK_CONFIG_SELECTION_TASKS, "new_tasks": {}, "ignored": [], + "processed": [], + "not_backstops": [], }, f, ) diff --git a/http_service/tests/test_boot.py b/http_service/tests/test_boot.py index ab74cd41b07..4d74d0cdf2d 100644 --- a/http_service/tests/test_boot.py +++ b/http_service/tests/test_boot.py @@ -72,23 +72,37 @@ def test_refresh_commits(monkeypatch: pytest.MonkeyPatch) -> None: def test_refresh_main(monkeypatch: pytest.MonkeyPatch) -> None: calls: list[str] = [] + now = [0.0] class Stop(Exception): pass def sleep(seconds: float) -> None: - assert seconds == boot.REFRESH_INTERVAL - if len(calls) == 2: + assert seconds == boot.CONFIG_SELECTION_TASKS_REFRESH_INTERVAL + if len(calls) == 7: raise Stop() + now[0] += seconds + def refresh_config_selection_tasks() -> None: + calls.append("tasks") + if len(calls) == 1: + raise Exception("pushlog unavailable") + + monkeypatch.setattr(boot, "REFRESH_INTERVAL", 30) + monkeypatch.setattr(boot, "CONFIG_SELECTION_TASKS_REFRESH_INTERVAL", 10) monkeypatch.setattr(refresh.os, "nice", lambda increment: None) monkeypatch.setattr(refresh.time, "sleep", sleep) + monkeypatch.setattr(refresh.time, "monotonic", lambda: now[0]) monkeypatch.setattr(boot, "refresh_data", lambda: calls.append("refresh")) + monkeypatch.setattr( + boot, "refresh_config_selection_tasks", refresh_config_selection_tasks + ) - # The data is refreshed again and again, after waiting. + # The config selection tasks are refreshed often (also after a failure), the rest of the data + # (including them) once per REFRESH_INTERVAL. with pytest.raises(Stop): refresh.main() - assert calls == ["refresh", "refresh"] + assert calls == ["tasks", "tasks", "refresh", "tasks", "tasks", "refresh", "tasks"] def _task( @@ -279,6 +293,8 @@ def test_retrieve_schedulable_tasks( "test-linux2404-64/debug-mochitest-plain", "test-linux2404-64/opt-mochitest-plain-standalone", ], + "processed": ["rev2", "rev3"], + "not_backstops": ["rev3"], } @@ -317,7 +333,13 @@ def test_update_config_selection_tasks( "linux2404-64-shippable/opt", ["dom/a.toml"], ) - for rev, backstop in (("b1", True), ("p1", False), ("p2", False), ("b2", True)): + for rev, backstop in ( + ("b1", True), + ("p1", False), + ("p2", False), + ("p3", False), + ("b2", True), + ): responses.add( responses.GET, boot._decision_artifact_url(rev, "parameters.yml"), @@ -359,12 +381,25 @@ def test_update_config_selection_tasks( } ), ) - # The target tasks of the next one aren't there: it doesn't block the others. + # The target tasks of the next one aren't there yet, then they are (without unknown tasks, so + # its full task graph isn't needed). responses.add( responses.GET, boot._decision_artifact_url("p2", "target-tasks.json"), status=404, ) + responses.add( + responses.GET, + boot._decision_artifact_url("p2", "target-tasks.json"), + json=sorted(KNOWN_TASKS) + [renamed["label"], renamed_shippable["label"]], + ) + + # The push after it is processed meanwhile. + responses.add( + responses.GET, + boot._decision_artifact_url("p3", "target-tasks.json"), + json=sorted(KNOWN_TASKS), + ) def read() -> dict: with open("config_selection_tasks.json") as f: @@ -376,7 +411,7 @@ def read() -> dict: "test-linux2404-64/opt-jittest-1proc", "test-linux2404-64/opt-mochitest-plain-standalone", ] - boot.update_config_selection_tasks(["b1", "p1", "p2"], KNOWN_TASKS) + boot.update_config_selection_tasks(["b1", "p1", "p2", "p3"], KNOWN_TASKS) assert read() == { "backstop": "b1", "tasks": CONFIG_SELECTION_TASKS, @@ -386,7 +421,13 @@ def read() -> dict: "ignored": sorted( ignored + ["test-linux2404-64-shippable/opt-mochitest-plain-renamed"] ), + "processed": ["p1", "p3"], + "not_backstops": ["p1", "p2", "p3"], } + + # p2 is processed once its target tasks are there. + boot.update_config_selection_tasks(["b1", "p1", "p2", "p3"], KNOWN_TASKS) + assert read()["processed"] == ["p1", "p2", "p3"] # The full task graph was only needed for p1's renamed tasks: the jittest task was already # known not to be eligible. full_graphs = [ @@ -406,7 +447,7 @@ def read() -> dict: json={**TASK_GRAPH, "r": renamed}, ) boot.update_config_selection_tasks( - ["b1", "p1", "p2", "b2"], KNOWN_TASKS | {renamed_shippable["label"]} + ["b1", "p1", "p2", "p3", "b2"], KNOWN_TASKS | {renamed_shippable["label"]} ) assert read() == { "backstop": "b2", @@ -421,5 +462,14 @@ def read() -> dict: "ignored": sorted( ignored + ["test-linux2404-64-shippable/opt-mochitest-plain-renamed"] ), + "processed": [], + "not_backstops": ["p1", "p2", "p3"], } + # Pushes already known not to be backstop pushes aren't checked again. + parameters_calls = [ + call.request.url + for call in responses.calls + if call.request.url.endswith("parameters.yml") + ] + assert len(parameters_calls) == len(set(parameters_calls)) assert_requests_have_timeouts() diff --git a/http_service/tests/test_schedule_tests.py b/http_service/tests/test_schedule_tests.py index 68e4eeaf153..8d9c1a71e9f 100644 --- a/http_service/tests/test_schedule_tests.py +++ b/http_service/tests/test_schedule_tests.py @@ -613,6 +613,8 @@ def test_schedule_config_groups_with_new_tasks( "test-linux1804-64/opt-mochitest-renamed": "test-linux1804-64/opt-*" }, "ignored": [], + "processed": [], + "not_backstops": [], }, f, ) From d68cd189e38ff69cbfdeb484108eaf6d725818d9 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Thu, 19 Nov 2026 23:35:44 +0100 Subject: [PATCH 09/10] Compute the configurations of the config selection tasks with the test variants of their push mozci memoizes the test variants of mozilla-central for the lifetime of the process (the refresh runs for weeks), and those don't have the variants added on autoland until they are merged, so the configurations of their tasks couldn't be computed. The HTTP service gives mozci the test variants of each push it reads tasks from (from the push's taskcluster/test_configs/variants.yml), or makes it retrieve the current ones of mozilla-central again if they can't be retrieved. --- http_service/bugbug_http/boot.py | 30 ++++++++++++ http_service/tests/test_boot.py | 80 +++++++++++++++++++++++++++++++- 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index 97d0a328567..53d6a504728 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -56,6 +56,34 @@ def _without_chunk(label: str) -> str: return re.sub(r"-\d+$", "", label) +def _set_test_variants(rev: str) -> None: + """Make mozci name the configurations of the tasks of a push with the push's test variants. + + mozci memoizes the test variants of mozilla-central for the lifetime of the process (the refresh + runs for weeks), and a variant added on autoland isn't on mozilla-central until it's merged. + The variants are the ones of the push's taskcluster/test_configs/variants.yml, parsed like mozci + does; if they can't be retrieved, mozci retrieves the current ones of mozilla-central again. + """ + # mozci.data needs to be imported before mozci.task to avoid a circular import. + import mozci.data # noqa: F401 + from mozci import task as mozci_task + + mozci_task.fetch_test_variant_yaml.clear() + try: + r = requests.get( + f"https://hg.mozilla.org/integration/autoland/raw-file/{rev}/taskcluster/test_configs/variants.yml", + timeout=REQUEST_TIMEOUT, + ) + r.raise_for_status() + variants = yaml.load(r.text, yaml.BaseLoader) + mozci_task.fetch_test_variant_yaml[()] = { + **mozci_task.REMOVED_TEST_VARIANTS, + **{name: variant["suffix"] for name, variant in variants.items()}, + } + except Exception: + logger.warning("Failed to retrieve the test variants of %s", rev, exc_info=True) + + def _eligible_tasks( task_graph: dict[str, dict], ) -> Iterator[tuple[str, str, list[str]]]: @@ -171,6 +199,7 @@ def _classify_new_tasks( for them (some can be missing, e.g. from the push's full task graph). The eligible ones become new tasks (unless they are candidates already), and the others are ignored. """ + _set_test_variants(rev) eligible = {label: config for label, config, _ in _eligible_tasks(graph)} for label in labels: if label not in eligible: @@ -228,6 +257,7 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non continue task_graph = orjson.loads(r.content) target_tasks = targets.json() + _set_test_variants(rev) tasks = get_config_selection_tasks(task_graph, target_tasks) logger.info( "Retrieved %d test tasks from the backstop push %s", len(tasks), rev diff --git a/http_service/tests/test_boot.py b/http_service/tests/test_boot.py index 4d74d0cdf2d..cc802ca163d 100644 --- a/http_service/tests/test_boot.py +++ b/http_service/tests/test_boot.py @@ -6,13 +6,17 @@ import contextlib import json import os +import re from pathlib import Path +# mozci.data needs to be imported before mozci.task to avoid a circular import. +import mozci.data # noqa: F401 import orjson import pytest import requests import responses import tenacity +from mozci import task as mozci_task from bugbug_http import boot, refresh @@ -214,14 +218,22 @@ def assert_requests_have_timeouts() -> None: ) +AUTOLAND_VARIANTS = re.compile( + r"https://hg.mozilla.org/integration/autoland/raw-file/[^/]+/taskcluster/test_configs/variants.yml" +) + + @pytest.fixture(autouse=True) def mock_test_variants() -> None: - # mozci retrieves the suffixes of the test variants. + # mozci memoizes the test variants. + mozci_task.fetch_test_variant_yaml.clear() + # The suffixes of the test variants, of mozilla-central (retrieved by mozci) and of the pushes. responses.add( responses.GET, "https://hg.mozilla.org/mozilla-central/raw-file/default/taskcluster/test_configs/variants.yml", body="xorigin:\n suffix: xorig\n", ) + responses.add(responses.GET, AUTOLAND_VARIANTS, body="xorigin:\n suffix: xorig\n") def test_get_config_selection_tasks() -> None: @@ -473,3 +485,69 @@ def read() -> dict: ] assert len(parameters_calls) == len(set(parameters_calls)) assert_requests_have_timeouts() + + +def test_update_config_selection_tasks_new_variant( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + with contextlib.suppress(FileNotFoundError): + os.remove(boot.CONFIG_SELECTION_TASKS_PATH) + # A test variant added on autoland, which isn't on mozilla-central yet. + new_variant = _task( + "test-linux2404-64/opt-mochitest-plain-newv-1", + "linux2404-64/opt", + ["dom/a.toml"], + runtime={"newvariant": True}, + ) + # A task of the same variant in a later push. + renamed = _task( + "test-linux2404-64/opt-mochitest-plain-renamed-newv-1", + "linux2404-64/opt", + ["dom/a.toml"], + runtime={"newvariant": True}, + ) + graph = {**TASK_GRAPH, "n": new_variant} + for rev, backstop in (("b1", True), ("p1", False)): + responses.add( + responses.GET, + boot._decision_artifact_url(rev, "parameters.yml"), + body=f"backstop: {'true' if backstop else 'false'}\n", + ) + responses.add( + responses.GET, boot._decision_artifact_url("b1", "task-graph.json"), json=graph + ) + targets = sorted(KNOWN_TASKS) + [new_variant["label"]] + responses.add( + responses.GET, + boot._decision_artifact_url("b1", "target-tasks.json"), + json=targets, + ) + responses.add( + responses.GET, + boot._decision_artifact_url("p1", "target-tasks.json"), + json=targets + [renamed["label"]], + ) + responses.add( + responses.GET, + boot._decision_artifact_url("p1", "full-task-graph.json"), + body=orjson.dumps({**graph, renamed["label"]: renamed}), + ) + responses.replace( + responses.GET, + AUTOLAND_VARIANTS, + body="xorigin:\n suffix: xorig\nnewvariant:\n suffix: newv\n", + ) + + boot.update_config_selection_tasks(["b1", "p1"], KNOWN_TASKS) + + with open(boot.CONFIG_SELECTION_TASKS_PATH) as f: + state = json.load(f) + # The configs are named with the push's test variants, on the first update. + assert state["tasks"]["test-linux2404-64/opt-mochitest-plain-newv"] == { + "config": "test-linux2404-64/opt-*-newv", + "manifests": ["dom/a.toml"], + } + assert state["new_tasks"] == { + "test-linux2404-64/opt-mochitest-plain-renamed-newv": "test-linux2404-64/opt-*-newv" + } From 74129e1bab4f967383ac633be524d0defd9fb949 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Thu, 19 Nov 2026 23:37:58 +0100 Subject: [PATCH 10/10] Try again to compute the configurations of the config selection tasks which failed Tasks whose configuration can't be computed (e.g. if retrieving the push's test variants failed) were ignored, or skipped and left behind once their push was processed. They are now kept pending, with the parts of their definitions needed to try again, and resolved by a later update: the ones of the backstop push become candidates (with their manifests), the ones of the later pushes new tasks. Other pushes targeting them don't need their full task graph meanwhile. --- http_service/bugbug_http/boot.py | 131 ++++++++++++++++++---- http_service/tests/conftest.py | 1 + http_service/tests/test_boot.py | 116 +++++++++++++++++++ http_service/tests/test_schedule_tests.py | 1 + 4 files changed, 226 insertions(+), 23 deletions(-) diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index 53d6a504728..3abf0bd0cab 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -84,8 +84,25 @@ def _set_test_variants(rev: str) -> None: logger.warning("Failed to retrieve the test variants of %s", rev, exc_info=True) +def _minimal_task(task: dict) -> dict: + """The parts of a task's definition needed to compute its config and manifests.""" + extra = task["task"]["extra"] + return { + "label": task["label"], + "attributes": {"test_manifests": task["attributes"]["test_manifests"]}, + "optimization": task.get("optimization"), + "task": { + "extra": { + key: extra[key] + for key in ("treeherder-platform", "suite", "test-setting") + if key in extra + } + }, + } + + def _eligible_tasks( - task_graph: dict[str, dict], + task_graph: dict[str, dict], failed: dict[str, list[dict]] | None = None ) -> Iterator[tuple[str, str, list[str]]]: """The test tasks of a task graph that test selection optimizes: (label without chunk number, config as mozci names it, manifests). @@ -93,7 +110,10 @@ def _eligible_tasks( The tasks that test selection doesn't optimize (e.g. the standalone ones, which only run on backstop pushes) are left out. So are shippable tasks, as they are too costly to select: the config selection models are trained with the configs of their non-shippable equivalents (see - test_scheduling.rename_task), which are selected instead. + test_scheduling.rename_task), which are selected instead. The tasks whose config can't be + computed (e.g. with a test variant mozci doesn't know) are added to `failed`, by label without + chunk number, with the parts of their definitions needed to try again (see _minimal_task): they + may be eligible. """ # mozci.data needs to be imported before mozci.task to avoid a circular import. import mozci.data # noqa: F401 @@ -116,13 +136,14 @@ def _eligible_tasks( try: configuration = get_configuration(task) except Exception: - # E.g. a test variant that mozci doesn't know yet. logger.warning( "Failed to get the configuration of %s", task["label"], exc_info=True, ) configuration = None + if failed is not None: + failed[label] = [] if ( configuration is not None and test_scheduling.rename_task(configuration) != configuration @@ -133,21 +154,26 @@ def _eligible_tasks( config = configurations[label] if config is not None: yield label, config, manifests + elif failed is not None and label in failed: + failed[label].append(_minimal_task(task)) def get_config_selection_tasks( - task_graph: dict[str, dict], target_tasks: Iterable[str] + task_graph: dict[str, dict], + target_tasks: Iterable[str], + failed: dict[str, list[dict]] | None = None, ) -> dict[str, dict]: """The test tasks among which configs are selected, by label without chunk number. They are the eligible test tasks (see _eligible_tasks) of `task_graph` (a backstop push's task-graph.json, which has all of them with all their manifests) which are among the push's target tasks (`target_tasks`, its target-tasks.json: the task graph also has the tasks they - depend on), with their config and manifests (see testconfigselect.LiveTask). + depend on), with their config and manifests (see testconfigselect.LiveTask). The labels of the + tasks whose config can't be computed are added to `failed` (see _eligible_tasks). """ targets = {_without_chunk(label) for label in target_tasks} tasks: dict[str, dict] = {} - for label, config, manifests in _eligible_tasks(task_graph): + for label, config, manifests in _eligible_tasks(task_graph, failed): if label not in targets: continue task = tasks.setdefault(label, {"config": config, "manifests": []}) @@ -190,23 +216,65 @@ def _get(rev: str, path: str) -> requests.Response: return requests.get(_decision_artifact_url(rev, path), timeout=REQUEST_TIMEOUT) +def _unresolved(rev: str, backstop: bool, failed: dict[str, list[dict]]) -> dict: + """The tasks of a push whose config couldn't be computed, to try again (see _resolve_unresolved).""" + return { + rev: { + "backstop": backstop, + "tasks": [task for tasks in failed.values() for task in tasks], + } + } + + def _classify_new_tasks( state: dict, ignored: set[str], rev: str, labels: set[str], graph: dict[str, dict] -) -> None: +) -> dict[str, list[dict]]: """Classify test tasks of a push after the backstop push. `labels` are the tasks to classify (without chunk number), `graph` the definitions available for them (some can be missing, e.g. from the push's full task graph). The eligible ones become - new tasks (unless they are candidates already), and the others are ignored. + new tasks (unless they are candidates already), and the others are ignored, except the ones + whose config can't be computed, which are returned (see _eligible_tasks). """ _set_test_variants(rev) - eligible = {label: config for label, config, _ in _eligible_tasks(graph)} - for label in labels: + failed: dict[str, list[dict]] = {} + eligible = {label: config for label, config, _ in _eligible_tasks(graph, failed)} + for label in labels - failed.keys(): if label not in eligible: ignored.add(label) elif label not in state["tasks"]: logger.info("New test task %s (%s) in push %s", label, eligible[label], rev) state["new_tasks"][label] = eligible[label] + return failed + + +def _resolve_unresolved(state: dict, ignored: set[str], revs: set[str]) -> None: + """Try again to compute the configs of the tasks whose config couldn't be computed before. + + The ones of the backstop push become candidates (with their manifests), the ones of the later + pushes new tasks, and the ones which aren't eligible are ignored. Those whose push isn't recent + anymore (other than the backstop push) are forgotten. + """ + for rev, entry in list(state["unresolved"].items()): + if not entry["backstop"] and rev not in revs: + del state["unresolved"][rev] + continue + + graph = {task["label"]: task for task in entry["tasks"]} + labels = {_without_chunk(label) for label in graph} + if entry["backstop"]: + _set_test_variants(rev) + failed: dict[str, list[dict]] = {} + tasks = get_config_selection_tasks(graph, graph, failed) + state["tasks"].update(tasks) + ignored |= labels - tasks.keys() - failed.keys() + else: + failed = _classify_new_tasks(state, ignored, rev, labels, graph) + + if failed: + entry["tasks"] = [task for tasks in failed.values() for task in tasks] + else: + del state["unresolved"][rev] def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> None: @@ -258,7 +326,8 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non task_graph = orjson.loads(r.content) target_tasks = targets.json() _set_test_variants(rev) - tasks = get_config_selection_tasks(task_graph, target_tasks) + failed: dict[str, list[dict]] = {} + tasks = get_config_selection_tasks(task_graph, target_tasks, failed) logger.info( "Retrieved %d test tasks from the backstop push %s", len(tasks), rev ) @@ -268,25 +337,31 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non # test selection optimizes run on backstop pushes, so the others are optimized # otherwise, e.g. jittest when no JS engine file changed). The ones found before are # kept, as long as they are among the target tasks of the recent pushes, as some are - # only targeted by some pushes (e.g. test-verify when tests are modified). + # only targeted by some pushes (e.g. test-verify when tests are modified). The ones + # whose config couldn't be computed may be eligible: they are looked for again later. recent_targets = {_without_chunk(label) for label in known_tasks} previously_ignored = set(state["ignored"]) if state is not None else set() ignored = ( - { - _without_chunk(label) - for label in ( - *(task["label"] for task in task_graph.values()), - *target_tasks, - ) - if label.startswith("test-") - } - | (previously_ignored & recent_targets) - ) - tasks.keys() + ( + { + _without_chunk(label) + for label in ( + *(task["label"] for task in task_graph.values()), + *target_tasks, + ) + if label.startswith("test-") + } + | (previously_ignored & recent_targets) + ) + - tasks.keys() + - failed.keys() + ) state = { "backstop": rev, "tasks": tasks, "new_tasks": {}, "ignored": sorted(ignored), + "unresolved": _unresolved(rev, True, failed) if failed else {}, "processed": [], } break @@ -297,6 +372,12 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non state["not_backstops"] = sorted(not_backstops & set(revs)) ignored = set(state["ignored"]) + _resolve_unresolved(state, ignored, set(revs)) + pending = { + _without_chunk(task["label"]) + for entry in state["unresolved"].values() + for task in entry["tasks"] + } # A2: the tasks of the later pushes. The ones without their target tasks or full task graph # (yet, e.g. if the decision task is still running, or ever, e.g. for DONTBUILD pushes or failed @@ -318,6 +399,7 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non and _without_chunk(label) not in state["tasks"] and _without_chunk(label) not in state["new_tasks"] and _without_chunk(label) not in ignored + and _without_chunk(label) not in pending } if unknown: r = _get(rev, "full-task-graph.json") @@ -325,9 +407,12 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non continue graph = get_full_graph_tasks(r.content, unknown) del r - _classify_new_tasks( + failed = _classify_new_tasks( state, ignored, rev, {_without_chunk(label) for label in unknown}, graph ) + if failed: + state["unresolved"].update(_unresolved(rev, False, failed)) + pending |= failed.keys() processed.add(rev) state["processed"] = sorted(processed & set(revs)) state["ignored"] = sorted(ignored) diff --git a/http_service/tests/conftest.py b/http_service/tests/conftest.py index 7800774787e..b1a4366ca01 100644 --- a/http_service/tests/conftest.py +++ b/http_service/tests/conftest.py @@ -376,6 +376,7 @@ def mock_config_selection_data(monkeypatch: MonkeyPatch, mock_data: None) -> Non "tasks": MOCK_CONFIG_SELECTION_TASKS, "new_tasks": {}, "ignored": [], + "unresolved": {}, "processed": [], "not_backstops": [], }, diff --git a/http_service/tests/test_boot.py b/http_service/tests/test_boot.py index cc802ca163d..31e5a0084a7 100644 --- a/http_service/tests/test_boot.py +++ b/http_service/tests/test_boot.py @@ -305,6 +305,7 @@ def test_retrieve_schedulable_tasks( "test-linux2404-64/debug-mochitest-plain", "test-linux2404-64/opt-mochitest-plain-standalone", ], + "unresolved": {}, "processed": ["rev2", "rev3"], "not_backstops": ["rev3"], } @@ -433,6 +434,7 @@ def read() -> dict: "ignored": sorted( ignored + ["test-linux2404-64-shippable/opt-mochitest-plain-renamed"] ), + "unresolved": {}, "processed": ["p1", "p3"], "not_backstops": ["p1", "p2", "p3"], } @@ -474,6 +476,7 @@ def read() -> dict: "ignored": sorted( ignored + ["test-linux2404-64-shippable/opt-mochitest-plain-renamed"] ), + "unresolved": {}, "processed": [], "not_backstops": ["p1", "p2", "p3"], } @@ -551,3 +554,116 @@ def test_update_config_selection_tasks_new_variant( assert state["new_tasks"] == { "test-linux2404-64/opt-mochitest-plain-renamed-newv": "test-linux2404-64/opt-*-newv" } + + +def test_update_config_selection_tasks_retries_unresolved( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + with contextlib.suppress(FileNotFoundError): + os.remove(boot.CONFIG_SELECTION_TASKS_PATH) + # Test variants added on autoland, which aren't on mozilla-central yet: the latest one isn't in + # the variants of the pushes at first either (e.g. a transient failure). + new_variant = _task( + "test-linux2404-64/opt-mochitest-plain-newv-1", + "linux2404-64/opt", + ["dom/a.toml"], + runtime={"newvariant": True}, + ) + later_variant = [ + _task( + f"test-linux2404-64/opt-mochitest-plain-later-{chunk}", + "linux2404-64/opt", + [f"dom/later{chunk}.toml"], + runtime={"latervariant": True}, + ) + for chunk in (1, 2) + ] + # A task of a variant added after the backstop push, in a later push. + renamed = _task( + "test-linux2404-64/opt-mochitest-plain-renamed-1", + "linux2404-64/opt", + ["dom/a.toml"], + runtime={"latervariant": True}, + ) + graph = { + **TASK_GRAPH, + "n": new_variant, + **{task["label"]: task for task in later_variant}, + } + for rev, backstop in (("b1", True), ("p1", False), ("p2", False)): + responses.add( + responses.GET, + boot._decision_artifact_url(rev, "parameters.yml"), + body=f"backstop: {'true' if backstop else 'false'}\n", + ) + responses.add( + responses.GET, boot._decision_artifact_url("b1", "task-graph.json"), json=graph + ) + targets = sorted(KNOWN_TASKS) + [ + task["label"] for task in (new_variant, *later_variant) + ] + responses.add( + responses.GET, + boot._decision_artifact_url("b1", "target-tasks.json"), + json=targets, + ) + for rev in ("p1", "p2"): + responses.add( + responses.GET, + boot._decision_artifact_url(rev, "target-tasks.json"), + json=targets + [renamed["label"]], + ) + responses.add( + responses.GET, + boot._decision_artifact_url("p1", "full-task-graph.json"), + body=orjson.dumps({**graph, renamed["label"]: renamed}), + ) + # The variants of the pushes don't have the latest one yet (e.g. a transient failure). + responses.replace( + responses.GET, + AUTOLAND_VARIANTS, + body="xorigin:\n suffix: xorig\nnewvariant:\n suffix: newv\n", + ) + + def read() -> dict: + with open(boot.CONFIG_SELECTION_TASKS_PATH) as f: + return json.load(f) + + boot.update_config_selection_tasks(["b1", "p1", "p2"], KNOWN_TASKS) + state = read() + # The configs are named with the push's test variants. + assert state["tasks"]["test-linux2404-64/opt-mochitest-plain-newv"] == { + "config": "test-linux2404-64/opt-*-newv", + "manifests": ["dom/a.toml"], + } + # The tasks whose config can't be computed aren't candidates or new tasks, nor ignored: they are + # pending, and other pushes targeting them don't need their full task graph again. + later = "test-linux2404-64/opt-mochitest-plain-later" + assert later not in state["tasks"] and later not in state["ignored"] + assert "test-linux2404-64/opt-mochitest-plain-renamed" not in state["new_tasks"] + assert sorted(state["unresolved"]) == ["b1", "p1"] + assert state["processed"] == ["p1", "p2"] + full_graphs = [ + call.request.url + for call in responses.calls + if call.request.url.endswith("full-task-graph.json") + ] + assert full_graphs == [boot._decision_artifact_url("p1", "full-task-graph.json")] + + # Once the variants are known, they are resolved on the next update. + responses.replace( + responses.GET, + AUTOLAND_VARIANTS, + body="xorigin:\n suffix: xorig\nnewvariant:\n suffix: newv\nlatervariant:\n suffix: later\n", + ) + boot.update_config_selection_tasks(["b1", "p1", "p2"], KNOWN_TASKS) + state = read() + assert state["tasks"][later] == { + "config": "test-linux2404-64/opt-*-later", + "manifests": ["dom/later1.toml", "dom/later2.toml"], + } + assert state["new_tasks"] == { + "test-linux2404-64/opt-mochitest-plain-renamed": "test-linux2404-64/opt-*-later" + } + assert state["unresolved"] == {} diff --git a/http_service/tests/test_schedule_tests.py b/http_service/tests/test_schedule_tests.py index 8d9c1a71e9f..335fa003a6d 100644 --- a/http_service/tests/test_schedule_tests.py +++ b/http_service/tests/test_schedule_tests.py @@ -613,6 +613,7 @@ def test_schedule_config_groups_with_new_tasks( "test-linux1804-64/opt-mochitest-renamed": "test-linux1804-64/opt-*" }, "ignored": [], + "unresolved": {}, "processed": [], "not_backstops": [], },