diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6669b42cae..5d9156c171 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -71,6 +71,7 @@ repos: - types-python-dateutil==2.9.0.20261104 - types-orjson==3.6.2 - types-tabulate==0.10.0.20261006 + - types-PyYAML==6.0.12.20261113 - repo: local hooks: - id: hackbot-deploy-names diff --git a/bugbug/models/testconfigdimensions.py b/bugbug/models/testconfigdimensions.py index cfd7803776..8f213eb786 100644 --- a/bugbug/models/testconfigdimensions.py +++ b/bugbug/models/testconfigdimensions.py @@ -336,10 +336,9 @@ def __init__(self, lemmatization=False): Model.__init__(self, lemmatization) self.training_dbs = [test_scheduling.PUSH_DATA_CONFIG_GROUP_DB] self.history = DimensionHistory() - self.known_configs: set[str] = set() def train(self, importance_cutoff=0.15, limit=None): - regressions, self.known_configs, _ = load_regressions(with_patches=False) + regressions, _ = load_regressions(with_patches=False) split = split_push_index(regressions) self.history = DimensionHistory() @@ -387,7 +386,6 @@ def __init__(self, lemmatization=False): self.entire_dataset_training = True self.history = DimensionHistory() - self.known_configs: set[str] = set() self._data: tuple | None = None self.row_weights: list[float] = [] self.row_push_index: list[int] = [] @@ -419,7 +417,7 @@ def __getstate__(self): def _load(self): if self._data is None: - regressions, self.known_configs, patches = load_regressions() + regressions, patches = load_regressions() self._data = (regressions, patches, split_push_index(regressions)) return self._data diff --git a/bugbug/models/testconfigselect.py b/bugbug/models/testconfigselect.py index df99e7f559..5756d6097d 100644 --- a/bugbug/models/testconfigselect.py +++ b/bugbug/models/testconfigselect.py @@ -24,13 +24,11 @@ """ import collections -import fnmatch import functools import logging import math import os import random -import re from typing import Any, Collection, Iterable, Iterator, Mapping, NamedTuple, Sequence import numpy as np @@ -312,49 +310,66 @@ def generate_rows( history.add(regression) -def _label_without_chunk(label: str) -> str: - parts = label.split("-") - return "-".join(parts[:-1] if parts[-1].isdigit() else parts) +class LiveTask(NamedTuple): + """A test task of the current task graph, by label without chunk number (see live_candidates).""" + # Its config, as mozci names it (see mozci.task.get_configuration). + config: str + # The manifests run by its chunks. + manifests: Sequence[str] -def live_candidates( - manifests_by_task: Mapping[str, Iterable[str]], known_configs: Iterable[str] -) -> dict[str, list[str]]: - """The configs of the tasks in the current task graph that run each group. - `manifests_by_task` maps the labels of the tasks scheduled on the push's project to the manifests - they run (the decision task's manifests-by-task.json, restricted to its target tasks, as it comes - from the full task graph, which also has tasks of other projects, e.g. shippable variants). The - configs are patterns over task labels without their chunk number (e.g. - test-macosx1500-aarch64/debug-*-swr), as the taskgraph matches them: each task is named after - the most specific `known_configs` pattern it matches, or after its own label without the chunk - number if none matches (e.g. for a new platform). +def live_tasks(tasks: Mapping[str, LiveTask]) -> dict[str, dict[str, list[str]]]: + """The tasks of the current task graph running each group, by config. - As the candidates come from the current task graph, configs that don't exist anymore (e.g. after - a platform migration) are never selected. Groups are named as by the models (see + `tasks` are the test tasks of the push's project among which configs are selected (see + bugbug_http.boot), by label without chunk number. Groups are named as by the models (see test_scheduling.normalize_group). """ - patterns = sorted( - ((len(c), c, re.compile(fnmatch.translate(c))) for c in set(known_configs)), - reverse=True, + result: dict[str, dict[str, set[str]]] = collections.defaultdict( + lambda: collections.defaultdict(set) ) + for label, task in tasks.items(): + for manifest in task.manifests: + result[test_scheduling.normalize_group(manifest)][task.config].add(label) + + return { + group: {config: sorted(labels) for config, labels in configs.items()} + for group, configs in result.items() + } + + +def live_candidates( + tasks_by_group: Mapping[str, Mapping[str, Sequence[str]]], +) -> dict[str, list[str]]: + """The configs of the tasks in the current task graph that run each group (see live_tasks). - @functools.cache - def config_of(label: str) -> str: - for _, config, pattern in patterns: - if pattern.match(label): - return config - return label + As the candidates come from the current task graph, configs that don't exist anymore (e.g. after + a platform migration) are never selected. + """ + return {group: sorted(configs) for group, configs in tasks_by_group.items()} - result: dict[str, set[str]] = collections.defaultdict(set) - for label, manifests in manifests_by_task.items(): - if not label.startswith("test-"): - continue - config = config_of(_label_without_chunk(label)) - for manifest in manifests: - result[test_scheduling.normalize_group(manifest)].add(config) - return {group: sorted(configs) for group, configs in result.items()} +def task_labels( + tasks_by_group: Mapping[str, Mapping[str, Sequence[str]]], + group: str, + configs: Iterable[str], + new_tasks: Mapping[str, str] | None = None, +) -> list[str]: + """The labels (without chunk number) of the tasks running a group on the given configs. + + The taskgraph matches them with the labels of its tasks. `tasks_by_group` comes from live_tasks. + `new_tasks` are tasks whose manifests aren't known (e.g. renamed since the task graph the others + come from), by label, with their config: they are given for every group selected on their config, + as the taskgraph only runs a task for a group if the task runs it. + """ + configs = set(configs) + tasks = tasks_by_group.get(group, {}) + labels = {label for config in configs for label in tasks.get(config, ())} + labels |= { + label for label, config in (new_tasks or {}).items() if config in configs + } + return sorted(labels) class GroupCandidates(NamedTuple): @@ -380,11 +395,9 @@ def _plainest_cheapest(configs: Iterable[str]) -> str: def load_regressions( with_patches: bool = True, -) -> tuple[ - list[test_scheduling.ConfigSelectRegression], set[str], dict[int, PatchInfo] -]: - """The regressions of the push data, the configs seen recently, and the patch of each regression's push.""" - regressions, known_configs = test_scheduling.get_config_select_regressions( +) -> tuple[list[test_scheduling.ConfigSelectRegression], dict[int, PatchInfo]]: + """The regressions of the push data, and the patch of each regression's push.""" + regressions = test_scheduling.get_config_select_regressions( lambda: db.read(test_scheduling.PUSH_DATA_CONFIG_GROUP_DB) ) logger.info( @@ -394,7 +407,7 @@ def load_regressions( ) if not with_patches: - return regressions, known_configs, {} + return regressions, {} revs = {rev: r.push_index for r in regressions for rev in r.revs} files: dict[int, list[str]] = collections.defaultdict(list) @@ -408,7 +421,7 @@ def load_regressions( for push_index, push_files in files.items() } logger.info("Patches found for %d pushes", len(patches)) - return regressions, known_configs, patches + return regressions, patches def split_push_index( @@ -671,10 +684,8 @@ def __init__(self, lemmatization=False): self.calculate_importance = False self.entire_dataset_training = True - # Inference state: the failure history up to the end of the training data, and the configs - # seen recently (to name the tasks of the current task graph, see live_candidates). + # Inference state: the failure history up to the end of the training data. self.history = ConfigHistory() - self.known_configs: set[str] = set() self._regressions: list[test_scheduling.ConfigSelectRegression] | None = None self._patches: dict[int, PatchInfo] = {} @@ -735,7 +746,7 @@ def __getstate__(self): def _load_regressions(self) -> list[test_scheduling.ConfigSelectRegression]: if self._regressions is None: - self._regressions, self.known_configs, self._patches = load_regressions() + self._regressions, self._patches = load_regressions() return self._regressions def _split_push_index(self) -> int: diff --git a/bugbug/test_scheduling.py b/bugbug/test_scheduling.py index 5a30ee3018..9e8f1e55a7 100644 --- a/bugbug/test_scheduling.py +++ b/bugbug/test_scheduling.py @@ -402,8 +402,7 @@ def _configs_by_group(config_groups: Iterable[Runnable]) -> dict[Group, set[str] def get_config_select_regressions( push_data_factory: Callable[[], Iterable[PushResult]], - recent_pushes: int = 1000, -) -> tuple[list[ConfigSelectRegression], set[str]]: +) -> list[ConfigSelectRegression]: """The regressions of config/group push data, with the configs they failed on and passed on. A config the regressed group ran on without failing passed, both on the culprit push and on the @@ -412,9 +411,6 @@ def get_config_select_regressions( run after the culprit failed and the failure was attributed to a later push (e.g. because the config didn't run on the culprit): with no run in between, the attribution can't tell the pushes apart, so the config didn't verifiably pass. A failure after a pass doesn't matter (as in get_non_run_negatives). - - Also returns the configs that ran in the last `recent_pushes` pushes, used to name the configs of - the tasks in the current task graph (see testconfigselect.live_candidates). """ rev_index: dict[Revision, int] = {} culprits: list[ @@ -427,7 +423,6 @@ def get_config_select_regressions( ] ] = [] failures_at: dict[int, dict[Group, set[str]]] = {} - num_pushes = 0 for index, push in enumerate(push_data_factory()): ( revisions, @@ -436,7 +431,6 @@ def get_config_select_regressions( possible_regressions, likely_regressions, ) = push - num_pushes += 1 for revision in revisions: rev_index[revision] = index if possible_regressions or likely_regressions: @@ -470,16 +464,13 @@ def get_config_select_regressions( # The first push of the window each config ran the group on. later_runs: dict[tuple[int, Group], dict[str, int]] = collections.defaultdict(dict) - recent: set[str] = set() for index, push in enumerate(push_data_factory()): groups_needed = needed.get(index) - if groups_needed is None and index < num_pushes - recent_pushes: + if groups_needed is None: continue for group, configs in _configs_by_group(push[2]).items(): - if index >= num_pushes - recent_pushes: - recent |= configs - for culprit in groups_needed.get(group, ()) if groups_needed else (): + for culprit in groups_needed.get(group, ()): for config in configs: later_runs[(culprit, group)].setdefault(config, index) @@ -505,7 +496,7 @@ def get_config_select_regressions( ) ) - return regressions, recent + return regressions def get_push_data( diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index a6bc608ed2..3abf0bd0ca 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -6,15 +6,17 @@ import collections import concurrent.futures import contextlib -import gzip import json import logging import os import re +from typing import Iterable, Iterator import hglib +import orjson import requests import tenacity +import yaml from tqdm import tqdm from bugbug import build_platforms, repository, test_scheduling, utils @@ -28,6 +30,14 @@ # How often the data about the current tasks, build platforms and commits is retrieved again (in # seconds, see bugbug_http.refresh). REFRESH_INTERVAL = 24 * 60 * 60 +# How often the test tasks among which configs are selected are updated with the latest pushes (in +# seconds, see bugbug_http.refresh and update_config_selection_tasks). +CONFIG_SELECTION_TASKS_REFRESH_INTERVAL = 10 * 60 +CONFIG_SELECTION_TASKS_PATH = "config_selection_tasks.json" +# The timeouts of the requests for the current tasks (to connect, and between bytes received, in +# seconds), so that a stalled request fails (and is retried) instead of blocking the boot or the +# refresh forever. +REQUEST_TIMEOUT = (30, 120) def _write(path: str, content: str) -> None: @@ -38,35 +48,418 @@ def _write(path: str, content: str) -> None: os.replace(tmp_path, path) -@tenacity.retry( - stop=tenacity.stop_after_attempt(7), - wait=tenacity.wait_exponential(multiplier=1, min=1, max=8), -) -def retrieve_schedulable_tasks() -> None: +def _decision_artifact_url(rev: str, path: str) -> str: + return f"https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.autoland.revision.{rev}.taskgraph.decision/artifacts/public/{path}" + + +def _without_chunk(label: str) -> str: + return re.sub(r"-\d+$", "", label) + + +def _set_test_variants(rev: str) -> None: + """Make mozci name the configurations of the tasks of a push with the push's test variants. + + mozci memoizes the test variants of mozilla-central for the lifetime of the process (the refresh + runs for weeks), and a variant added on autoland isn't on mozilla-central until it's merged. + The variants are the ones of the push's taskcluster/test_configs/variants.yml, parsed like mozci + does; if they can't be retrieved, mozci retrieves the current ones of mozilla-central again. + """ + # mozci.data needs to be imported before mozci.task to avoid a circular import. + import mozci.data # noqa: F401 + from mozci import task as mozci_task + + mozci_task.fetch_test_variant_yaml.clear() + try: + r = requests.get( + f"https://hg.mozilla.org/integration/autoland/raw-file/{rev}/taskcluster/test_configs/variants.yml", + timeout=REQUEST_TIMEOUT, + ) + r.raise_for_status() + variants = yaml.load(r.text, yaml.BaseLoader) + mozci_task.fetch_test_variant_yaml[()] = { + **mozci_task.REMOVED_TEST_VARIANTS, + **{name: variant["suffix"] for name, variant in variants.items()}, + } + except Exception: + logger.warning("Failed to retrieve the test variants of %s", rev, exc_info=True) + + +def _minimal_task(task: dict) -> dict: + """The parts of a task's definition needed to compute its config and manifests.""" + extra = task["task"]["extra"] + return { + "label": task["label"], + "attributes": {"test_manifests": task["attributes"]["test_manifests"]}, + "optimization": task.get("optimization"), + "task": { + "extra": { + key: extra[key] + for key in ("treeherder-platform", "suite", "test-setting") + if key in extra + } + }, + } + + +def _eligible_tasks( + task_graph: dict[str, dict], failed: dict[str, list[dict]] | None = None +) -> Iterator[tuple[str, str, list[str]]]: + """The test tasks of a task graph that test selection optimizes: (label without chunk number, + config as mozci names it, manifests). + + The tasks that test selection doesn't optimize (e.g. the standalone ones, which only run on + backstop pushes) are left out. So are shippable tasks, as they are too costly to select: the + config selection models are trained with the configs of their non-shippable equivalents (see + test_scheduling.rename_task), which are selected instead. The tasks whose config can't be + computed (e.g. with a test variant mozci doesn't know) are added to `failed`, by label without + chunk number, with the parts of their definitions needed to try again (see _minimal_task): they + may be eligible. + """ + # mozci.data needs to be imported before mozci.task to avoid a circular import. + import mozci.data # noqa: F401 + from mozci.task import get_configuration + + utils.setup_mozci() + + configurations: dict[str, str | None] = {} + for task in task_graph.values(): + manifests = task["attributes"].get("test_manifests") + if ( + not task["label"].startswith("test-") + or not manifests + or "test" not in (task.get("optimization") or {}) + ): + continue + + label = _without_chunk(task["label"]) + if label not in configurations: + try: + configuration = get_configuration(task) + except Exception: + logger.warning( + "Failed to get the configuration of %s", + task["label"], + exc_info=True, + ) + configuration = None + if failed is not None: + failed[label] = [] + if ( + configuration is not None + and test_scheduling.rename_task(configuration) != configuration + ): + configuration = None + configurations[label] = configuration + + config = configurations[label] + if config is not None: + yield label, config, manifests + elif failed is not None and label in failed: + failed[label].append(_minimal_task(task)) + + +def get_config_selection_tasks( + task_graph: dict[str, dict], + target_tasks: Iterable[str], + failed: dict[str, list[dict]] | None = None, +) -> dict[str, dict]: + """The test tasks among which configs are selected, by label without chunk number. + + They are the eligible test tasks (see _eligible_tasks) of `task_graph` (a backstop push's + task-graph.json, which has all of them with all their manifests) which are among the push's + target tasks (`target_tasks`, its target-tasks.json: the task graph also has the tasks they + depend on), with their config and manifests (see testconfigselect.LiveTask). The labels of the + tasks whose config can't be computed are added to `failed` (see _eligible_tasks). + """ + targets = {_without_chunk(label) for label in target_tasks} + tasks: dict[str, dict] = {} + for label, config, manifests in _eligible_tasks(task_graph, failed): + if label not in targets: + continue + task = tasks.setdefault(label, {"config": config, "manifests": []}) + task["manifests"] = sorted(set(task["manifests"]) | set(manifests)) + return tasks + + +def get_full_graph_tasks(content: bytes, labels: set[str]) -> dict[str, dict]: + """The definitions of some tasks from a decision task's full-task-graph.json, by label. + + The full task graph is hundreds of MB, and much more once parsed, so only the definitions of + the given tasks are parsed, where they are in the text. If one isn't found this way (e.g. if + the format of the file changed), the whole graph is parsed. + """ + text = content.decode() + decoder = json.JSONDecoder() + tasks: dict[str, dict] = {} + for label in labels: + key = f"{json.dumps(label)}:{{" + start = text.find(key) + while start != -1: + try: + task, _ = decoder.raw_decode(text, start + len(key) - 1) + except ValueError: + task = None + if isinstance(task, dict) and task.get("label") == label: + tasks[label] = task + break + start = text.find(key, start + 1) + + missing = labels - tasks.keys() + if missing: + del text + graph = orjson.loads(content) + tasks.update({label: graph[label] for label in missing if label in graph}) + return tasks + + +def _get(rev: str, path: str) -> requests.Response: + return requests.get(_decision_artifact_url(rev, path), timeout=REQUEST_TIMEOUT) + + +def _unresolved(rev: str, backstop: bool, failed: dict[str, list[dict]]) -> dict: + """The tasks of a push whose config couldn't be computed, to try again (see _resolve_unresolved).""" + return { + rev: { + "backstop": backstop, + "tasks": [task for tasks in failed.values() for task in tasks], + } + } + + +def _classify_new_tasks( + state: dict, ignored: set[str], rev: str, labels: set[str], graph: dict[str, dict] +) -> dict[str, list[dict]]: + """Classify test tasks of a push after the backstop push. + + `labels` are the tasks to classify (without chunk number), `graph` the definitions available + for them (some can be missing, e.g. from the push's full task graph). The eligible ones become + new tasks (unless they are candidates already), and the others are ignored, except the ones + whose config can't be computed, which are returned (see _eligible_tasks). + """ + _set_test_variants(rev) + failed: dict[str, list[dict]] = {} + eligible = {label: config for label, config, _ in _eligible_tasks(graph, failed)} + for label in labels - failed.keys(): + if label not in eligible: + ignored.add(label) + elif label not in state["tasks"]: + logger.info("New test task %s (%s) in push %s", label, eligible[label], rev) + state["new_tasks"][label] = eligible[label] + return failed + + +def _resolve_unresolved(state: dict, ignored: set[str], revs: set[str]) -> None: + """Try again to compute the configs of the tasks whose config couldn't be computed before. + + The ones of the backstop push become candidates (with their manifests), the ones of the later + pushes new tasks, and the ones which aren't eligible are ignored. Those whose push isn't recent + anymore (other than the backstop push) are forgotten. + """ + for rev, entry in list(state["unresolved"].items()): + if not entry["backstop"] and rev not in revs: + del state["unresolved"][rev] + continue + + graph = {task["label"]: task for task in entry["tasks"]} + labels = {_without_chunk(label) for label in graph} + if entry["backstop"]: + _set_test_variants(rev) + failed: dict[str, list[dict]] = {} + tasks = get_config_selection_tasks(graph, graph, failed) + state["tasks"].update(tasks) + ignored |= labels - tasks.keys() - failed.keys() + else: + failed = _classify_new_tasks(state, ignored, rev, labels, graph) + + if failed: + entry["tasks"] = [task for tasks in failed.values() for task in tasks] + else: + del state["unresolved"][rev] + + +def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> None: + """Update the test tasks among which configs are selected with the latest pushes. + + The tasks come from the task graph of the latest backstop push, which runs all of them with all + their manifests (the manifests of the test tasks of other pushes are restricted to the ones + selected by bugbug). Older backstop pushes are used if the graph of a newer one isn't + available or has none of the tasks. + + The tasks of the pushes after it which aren't known (e.g. because a task was renamed since the + backstop push) are stored too, with their config: the taskgraph matches the labels of its tasks + with the ones of the selected configs, which would otherwise not run on the renamed tasks. They + are found among the target tasks of the pushes (before optimization, which could remove them as + they match none of the labels in the replies), and their definitions in the full task graph + (only when there are some). Their manifests aren't known (only the ones selected by bugbug run on + those pushes), so the replies give them for all the groups the configs are selected for (see + testconfigselect.task_labels). The test tasks which aren't eligible (see _eligible_tasks) are + remembered, so that they aren't looked for again. + + `revs` are the recent autoland pushes, from the oldest. `known_tasks` are their target tasks + (possibly from an earlier refresh), only used to forget the ignored tasks no push targets anymore. + """ + try: + with open(CONFIG_SELECTION_TASKS_PATH, "rb") as f: + state = orjson.loads(f.read()) + except FileNotFoundError: + state = None + previous = orjson.dumps(state) + + # A1: the latest backstop push with tasks. + known_backstop = state["backstop"] if state is not None else None + not_backstops = set(state["not_backstops"]) if state is not None else set() + for rev in reversed(revs): + if rev == known_backstop: + break + if rev in not_backstops: + continue + r = _get(rev, "parameters.yml") + if not r.ok: + continue + if not yaml.safe_load(r.text).get("backstop"): + not_backstops.add(rev) + continue + r = _get(rev, "task-graph.json") + targets = _get(rev, "target-tasks.json") + if not r.ok or not targets.ok: + continue + task_graph = orjson.loads(r.content) + target_tasks = targets.json() + _set_test_variants(rev) + failed: dict[str, list[dict]] = {} + tasks = get_config_selection_tasks(task_graph, target_tasks, failed) + logger.info( + "Retrieved %d test tasks from the backstop push %s", len(tasks), rev + ) + if tasks: + # The test tasks which aren't eligible: the ones of the task graph which aren't among + # the tasks, and the target ones which aren't in the task graph (all the tasks that + # test selection optimizes run on backstop pushes, so the others are optimized + # otherwise, e.g. jittest when no JS engine file changed). The ones found before are + # kept, as long as they are among the target tasks of the recent pushes, as some are + # only targeted by some pushes (e.g. test-verify when tests are modified). The ones + # whose config couldn't be computed may be eligible: they are looked for again later. + recent_targets = {_without_chunk(label) for label in known_tasks} + previously_ignored = set(state["ignored"]) if state is not None else set() + ignored = ( + ( + { + _without_chunk(label) + for label in ( + *(task["label"] for task in task_graph.values()), + *target_tasks, + ) + if label.startswith("test-") + } + | (previously_ignored & recent_targets) + ) + - tasks.keys() + - failed.keys() + ) + state = { + "backstop": rev, + "tasks": tasks, + "new_tasks": {}, + "ignored": sorted(ignored), + "unresolved": _unresolved(rev, True, failed) if failed else {}, + "processed": [], + } + break + + if state is None: + logger.warning("No backstop push with test tasks to retrieve") + return + state["not_backstops"] = sorted(not_backstops & set(revs)) + + ignored = set(state["ignored"]) + _resolve_unresolved(state, ignored, set(revs)) + pending = { + _without_chunk(task["label"]) + for entry in state["unresolved"].values() + for task in entry["tasks"] + } + + # A2: the tasks of the later pushes. The ones without their target tasks or full task graph + # (yet, e.g. if the decision task is still running, or ever, e.g. for DONTBUILD pushes or failed + # decision tasks) are left for the next refresh, without blocking the others. + later = ( + revs[revs.index(state["backstop"]) + 1 :] if state["backstop"] in revs else revs + ) + processed = set(state["processed"]) + for rev in later: + if rev in processed: + continue + r = _get(rev, "target-tasks.json") + if not r.ok: + continue + unknown = { + label + for label in r.json() + if label.startswith("test-") + and _without_chunk(label) not in state["tasks"] + and _without_chunk(label) not in state["new_tasks"] + and _without_chunk(label) not in ignored + and _without_chunk(label) not in pending + } + if unknown: + r = _get(rev, "full-task-graph.json") + if not r.ok: + continue + graph = get_full_graph_tasks(r.content, unknown) + del r + failed = _classify_new_tasks( + state, ignored, rev, {_without_chunk(label) for label in unknown}, graph + ) + if failed: + state["unresolved"].update(_unresolved(rev, False, failed)) + pending |= failed.keys() + processed.add(rev) + state["processed"] = sorted(processed & set(revs)) + state["ignored"] = sorted(ignored) + + serialized = orjson.dumps(state) + if serialized != previous: + _write(CONFIG_SELECTION_TASKS_PATH, serialized.decode()) + + +def get_recent_revs() -> list[str]: + """The latest AUTOLAND_PUSHES autoland pushes, from the oldest.""" # The pushes of the last week (many more than AUTOLAND_PUSHES, also during holidays). r = requests.get( "https://hg.mozilla.org/integration/autoland/json-pushes", params={"version": "2", "tipsonly": "1", "startdate": "1 week ago"}, + timeout=REQUEST_TIMEOUT, ) r.raise_for_status() pushes = sorted(r.json()["pushes"].items(), key=lambda push: int(push[0])) - revs = [push_obj["changesets"][0] for _, push_obj in pushes[-AUTOLAND_PUSHES:]] + return [push_obj["changesets"][0] for _, push_obj in pushes[-AUTOLAND_PUSHES:]] + + +def refresh_config_selection_tasks() -> None: + """Update the test tasks among which configs are selected (see update_config_selection_tasks).""" + with open("known_tasks", "r") as f: + known_tasks = {line.strip() for line in f} + update_config_selection_tasks(get_recent_revs(), known_tasks) + + +@tenacity.retry( + stop=tenacity.stop_after_attempt(7), + wait=tenacity.wait_exponential(multiplier=1, min=1, max=8), +) +def retrieve_schedulable_tasks() -> None: + revs = get_recent_revs() logger.info("Retrieving known tasks from %s...", revs) - # Store in a file the list of tasks in the latest autoland pushes, and the manifests run by - # their tasks (to select the configs to run manifests on among the current ones). Pushes - # don't all target the same tasks (only some of them target almost all), so we use the union - # of the target tasks of many pushes, which also protects us from broken decision tasks. + # Store in a file the list of tasks in the latest autoland pushes, and the test tasks among which + # configs are selected for the manifests. Pushes don't all target the same tasks (only some of + # them target almost all), so we use the union of the target tasks of many pushes, which also + # protects us from broken decision tasks. known_tasks = set() - targets_num = {} for rev in revs: - r = requests.get( - f"https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.autoland.revision.{rev}.taskgraph.decision/artifacts/public/target-tasks.json" - ) + r = _get(rev, "target-tasks.json") if r.ok: known_tasks.update(r.json()) - targets_num[rev] = len(r.json()) logger.info("Retrieved %d tasks", len(known_tasks)) @@ -74,43 +467,8 @@ def retrieve_schedulable_tasks() -> None: _write("known_tasks", "\n".join(known_tasks)) - # The manifests run by each task, from the push targeting the most tasks (pushes targeting few - # tasks have no test tasks). - manifests_by_task: dict[str, list[str]] = {} - for rev in sorted(targets_num, key=targets_num.__getitem__, reverse=True)[:3]: - r = requests.get( - f"https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.autoland.revision.{rev}.taskgraph.decision/artifacts/public/manifests-by-task.json.gz" - ) - if not r.ok: - continue - try: - content = gzip.decompress(r.content) - except gzip.BadGzipFile: - content = r.content - manifests_by_task = json.loads(content) - if manifests_by_task: - break - - # Only the target tasks, as the full task graph also has tasks that don't run on autoland - # (e.g. shippable variants), and a group given only configs without tasks on the push - # wouldn't run at all. Tasks are compared without their chunk number, which varies between - # pushes. - def without_chunk(label: str) -> str: - return re.sub(r"-\d+$", "", label) - - target_tasks = {without_chunk(task) for task in known_tasks} - manifests_by_task = { - label: manifests - for label, manifests in manifests_by_task.items() - if without_chunk(label) in target_tasks - } - logger.info("Retrieved the manifests of %d tasks", len(manifests_by_task)) - _write( - "manifests_by_task.json", - json.dumps( - {label: sorted(manifests) for label, manifests in manifests_by_task.items()} - ), - ) + # The test tasks among which configs are selected for the manifests. + update_config_selection_tasks(revs, known_tasks) def retrieve_build_platforms() -> None: diff --git a/http_service/bugbug_http/models.py b/http_service/bugbug_http/models.py index 778f82e04c..f97c72e5ea 100644 --- a/http_service/bugbug_http/models.py +++ b/http_service/bugbug_http/models.py @@ -569,13 +569,37 @@ def get_known_tasks() -> tuple[str, ...]: CONFIG_SELECTION_THRESHOLD = 0.00015 +@lru_cache(maxsize=None) +def _get_config_selection_tasks() -> dict: + """The test tasks among which configs are selected (see boot.update_config_selection_tasks).""" + try: + with open("config_selection_tasks.json", "rb") as f: + return orjson.loads(f.read()) + except FileNotFoundError: + return {"tasks": {}, "new_tasks": {}} + + +@lru_cache(maxsize=None) +def get_live_tasks() -> dict[str, dict[str, list[str]]]: + """The test tasks scheduled on autoland running each group, by config.""" + return testconfigselect.live_tasks( + { + label: testconfigselect.LiveTask(task["config"], task["manifests"]) + for label, task in _get_config_selection_tasks()["tasks"].items() + } + ) + + +@lru_cache(maxsize=None) +def get_new_tasks() -> dict[str, str]: + """The test tasks of the latest pushes which aren't in get_live_tasks, with their configs.""" + return _get_config_selection_tasks()["new_tasks"] + + @lru_cache(maxsize=None) def get_live_config_candidates() -> dict[str, list[str]]: """The configs of the tasks scheduled on autoland that run each group.""" - with open("manifests_by_task.json", "rb") as f: - manifests_by_task = orjson.loads(f.read()) - model = MODEL_CACHE.get(CONFIG_SELECTION_MODEL) - return testconfigselect.live_candidates(manifests_by_task, model.known_configs) + return testconfigselect.live_candidates(get_live_tasks()) @lru_cache(maxsize=None) @@ -892,6 +916,7 @@ def _analyze_patch( "tasks": {}, "groups": {}, "config_groups": {}, + "config_group_configs": {}, "reduced_tasks": {}, "reduced_tasks_higher": {}, "known_tasks": get_known_tasks(), @@ -1017,7 +1042,21 @@ def _analyze_patch( data = { "tasks": tasks, "groups": groups, - "config_groups": config_groups, + # The taskgraph matches the labels (without chunk number) of its tasks with the ones of the + # tasks of the selected configs (a group without any runs on all configs, see + # select_configs). + "config_groups": { + group: ( + "*" not in configs + and testconfigselect.task_labels( + get_live_tasks(), group, configs, get_new_tasks() + ) + ) + or ["*"] + for group, configs in config_groups.items() + }, + # The selected configs, as the models (and mozci) name them. + "config_group_configs": config_groups, "reduced_tasks": {t: c for t, c in tasks.items() if t in reduced}, "reduced_tasks_higher": {t: c for t, c in tasks.items() if t in reduced_higher}, "known_tasks": known_tasks, diff --git a/http_service/bugbug_http/refresh.py b/http_service/bugbug_http/refresh.py index edeada1f62..e50652591c 100644 --- a/http_service/bugbug_http/refresh.py +++ b/http_service/bugbug_http/refresh.py @@ -8,7 +8,8 @@ Workers can run for weeks, while the tasks of the task graph and the files built on each platform change (e.g. platform migrations), and new commits land (used by the features of the test selection -models). This runs next to the worker, in the same container and directory: it rewrites the worker's +models), and tasks get renamed (whose labels the replies give for the selected configs, so they are +refreshed more often). This runs next to the worker, in the same container and directory: it rewrites the worker's files (atomically) and databases (LMDB, whose readers see consistent snapshots), and jobs read them when they run. """ @@ -21,6 +22,7 @@ from bugbug_http.sentry import setup_sentry logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) if os.environ.get("SENTRY_DSN"): setup_sentry(dsn=os.environ.get("SENTRY_DSN")) @@ -30,9 +32,19 @@ def main() -> None: # Mining commits uses all the CPUs: don't slow the jobs down. os.nice(10) + # The test tasks among which configs are selected are updated much more often than the rest, + # so that the selected configs run on tasks renamed since the latest backstop push. + next_refresh = time.monotonic() + boot.REFRESH_INTERVAL while True: - time.sleep(boot.REFRESH_INTERVAL) - boot.refresh_data() + time.sleep(boot.CONFIG_SELECTION_TASKS_REFRESH_INTERVAL) + if time.monotonic() >= next_refresh: + boot.refresh_data() + next_refresh = time.monotonic() + boot.REFRESH_INTERVAL + else: + try: + boot.refresh_config_selection_tasks() + except Exception: + logger.exception("Failed to refresh the config selection tasks") if __name__ == "__main__": diff --git a/http_service/pyproject.toml b/http_service/pyproject.toml index 502c97e8b3..a4b304f3cb 100644 --- a/http_service/pyproject.toml +++ b/http_service/pyproject.toml @@ -25,6 +25,7 @@ dependencies = [ "gunicorn>=25.1,<26.3", "kombu>=5.6.2,<5.8.0", "marshmallow>=3.18.0", + "pyyaml>=6.0.3", "rq>=2.7,<2.13", "rq-dashboard>=0.8.6,<0.10.0", "sentry-sdk[flask]>=2.60.0", diff --git a/http_service/tests/conftest.py b/http_service/tests/conftest.py index da196a0ef5..b1a4366ca0 100644 --- a/http_service/tests/conftest.py +++ b/http_service/tests/conftest.py @@ -328,21 +328,26 @@ def mock_component_taskcluster_artifact() -> None: ] -MOCK_MANIFESTS_BY_TASK = { - "test-linux1804-64/opt-mochitest-1": ["test-group1", "test-group2"], - "test-windows10/debug-mochitest-1": ["test-group1"], - "test-windows10/opt-mochitest-2": ["test-group1", "test-group2"], +# The test tasks among which configs are selected (see boot.get_config_selection_tasks). +MOCK_CONFIG_SELECTION_TASKS = { + "test-linux1804-64/opt-mochitest": { + "config": "test-linux1804-64/opt-*", + "manifests": ["test-group1", "test-group2"], + }, + "test-windows10/debug-mochitest": { + "config": "test-windows10/debug-*", + "manifests": ["test-group1"], + }, + "test-windows10/opt-mochitest": { + "config": "test-windows10/opt-*", + "manifests": ["test-group1", "test-group2"], + }, } def mock_config_select_model() -> bugbug.models.testconfigselect.TestConfigSelectModel: """A config selection model predicting 0.5 for a group's first config, 0 for the following ones.""" model = bugbug.models.testconfigselect.TestConfigSelectModel() - model.known_configs = { - "test-linux1804-64/opt-*", - "test-windows10/debug-*", - "test-windows10/opt-*", - } model._predict = lambda test_jobs: np.array( # type: ignore[method-assign] [0.0 if test_job["selected"] else 0.5 for test_job in test_jobs] ) @@ -364,8 +369,19 @@ def get(self, model_name): @pytest.fixture(autouse=True) def mock_config_selection_data(monkeypatch: MonkeyPatch, mock_data: None) -> None: - with open("manifests_by_task.json", "w") as f: - json.dump(MOCK_MANIFESTS_BY_TASK, f) + with open("config_selection_tasks.json", "w") as f: + json.dump( + { + "backstop": "backstop-rev", + "tasks": MOCK_CONFIG_SELECTION_TASKS, + "new_tasks": {}, + "ignored": [], + "unresolved": {}, + "processed": [], + "not_backstops": [], + }, + f, + ) with open("known_tasks", "w") as f: f.write("\n".join(MOCK_KNOWN_TASKS)) @@ -373,6 +389,9 @@ def mock_config_selection_data(monkeypatch: MonkeyPatch, mock_data: None) -> Non if os.path.exists("build_platforms.json"): os.remove("build_platforms.json") bugbug_http.models.get_known_tasks.cache_clear() + bugbug_http.models._get_config_selection_tasks.cache_clear() + bugbug_http.models.get_live_tasks.cache_clear() + bugbug_http.models.get_new_tasks.cache_clear() bugbug_http.models.get_live_config_candidates.cache_clear() bugbug_http.models.get_build_platforms.cache_clear() diff --git a/http_service/tests/test_boot.py b/http_service/tests/test_boot.py index 044c270f87..31e5a0084a 100644 --- a/http_service/tests/test_boot.py +++ b/http_service/tests/test_boot.py @@ -3,11 +3,20 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. +import contextlib import json import os +import re from pathlib import Path +# mozci.data needs to be imported before mozci.task to avoid a circular import. +import mozci.data # noqa: F401 +import orjson import pytest +import requests +import responses +import tenacity +from mozci import task as mozci_task from bugbug_http import boot, refresh @@ -67,20 +76,594 @@ def test_refresh_commits(monkeypatch: pytest.MonkeyPatch) -> None: def test_refresh_main(monkeypatch: pytest.MonkeyPatch) -> None: calls: list[str] = [] + now = [0.0] class Stop(Exception): pass def sleep(seconds: float) -> None: - assert seconds == boot.REFRESH_INTERVAL - if len(calls) == 2: + assert seconds == boot.CONFIG_SELECTION_TASKS_REFRESH_INTERVAL + if len(calls) == 7: raise Stop() + now[0] += seconds + def refresh_config_selection_tasks() -> None: + calls.append("tasks") + if len(calls) == 1: + raise Exception("pushlog unavailable") + + monkeypatch.setattr(boot, "REFRESH_INTERVAL", 30) + monkeypatch.setattr(boot, "CONFIG_SELECTION_TASKS_REFRESH_INTERVAL", 10) monkeypatch.setattr(refresh.os, "nice", lambda increment: None) monkeypatch.setattr(refresh.time, "sleep", sleep) + monkeypatch.setattr(refresh.time, "monotonic", lambda: now[0]) monkeypatch.setattr(boot, "refresh_data", lambda: calls.append("refresh")) + monkeypatch.setattr( + boot, "refresh_config_selection_tasks", refresh_config_selection_tasks + ) - # The data is refreshed again and again, after waiting. + # The config selection tasks are refreshed often (also after a failure), the rest of the data + # (including them) once per REFRESH_INTERVAL. with pytest.raises(Stop): refresh.main() - assert calls == ["refresh", "refresh"] + assert calls == ["tasks", "tasks", "refresh", "tasks", "tasks", "refresh", "tasks"] + + +def _task( + label: str, + platform: str, + manifests: list[str], + optimization: dict | None = None, + runtime: dict | None = None, +) -> dict: + return { + "label": label, + "attributes": {"test_manifests": manifests}, + "optimization": optimization or {"test": ["firefox", "mochitest"]}, + "task": { + "extra": { + "treeherder-platform": platform, + "suite": "mochitest-plain", + "test-setting": {"runtime": runtime or {}}, + } + }, + } + + +TASK_GRAPH = { + "a": _task( + "test-linux2404-64/opt-mochitest-plain-1", "linux2404-64/opt", ["dom/a.toml"] + ), + "b": _task( + "test-linux2404-64/opt-mochitest-plain-2", "linux2404-64/opt", ["dom/b.toml"] + ), + # Shippable tasks are too costly: their non-shippable equivalents are selected instead. + "c": _task( + "test-linux2404-64-shippable/opt-mochitest-plain-1", + "linux2404-64-shippable/opt", + ["dom/a.toml"], + ), + "d": _task( + "test-windows11-64-25h2-asan/opt-mochitest-plain-1", + "windows11-64-25h2/asan", + ["dom/a.toml"], + ), + # The variants come from the test settings. + "h": _task( + "test-linux2404-64/opt-mochitest-plain-xorig-1", + "linux2404-64/opt", + ["dom/a.toml"], + runtime={"xorigin": True}, + ), + # Not optimized by test selection. + "e": _task( + "test-linux2404-64/opt-mochitest-plain-standalone-1", + "linux2404-64/opt", + ["dom/a.toml"], + {"skip-unless-backstop": None}, + ), + # Not a target task of the recent pushes. + "f": _task( + "test-linux2404-64/debug-mochitest-plain-1", + "linux2404-64/debug", + ["dom/a.toml"], + ), + # Not a test task running manifests. + "g": { + "label": "build-linux64/opt", + "attributes": {}, + "optimization": None, + "task": {}, + }, +} + +KNOWN_TASKS = { + "test-linux2404-64/opt-mochitest-plain-1", + "test-linux2404-64/opt-mochitest-plain-2", + "test-linux2404-64-shippable/opt-mochitest-plain-3", + "test-windows11-64-25h2-asan/opt-mochitest-plain-1", + "test-linux2404-64/opt-mochitest-plain-xorig-1", + "test-linux2404-64/opt-mochitest-plain-standalone-1", + "build-linux64/opt", +} + +CONFIG_SELECTION_TASKS = { + # The manifests of all the chunks. + "test-linux2404-64/opt-mochitest-plain": { + "config": "test-linux2404-64/opt-*", + "manifests": ["dom/a.toml", "dom/b.toml"], + }, + "test-windows11-64-25h2-asan/opt-mochitest-plain": { + "config": "test-windows11-64-25h2/asan-*", + "manifests": ["dom/a.toml"], + }, + "test-linux2404-64/opt-mochitest-plain-xorig": { + "config": "test-linux2404-64/opt-*-xorig", + "manifests": ["dom/a.toml"], + }, +} + + +def assert_requests_have_timeouts() -> None: + """Stalled requests can't block the boot or the refresh (mozci's own requests aside).""" + calls = [ + call + for call in responses.calls + if not call.request.url.startswith("https://hg.mozilla.org/mozilla-central/") + ] + assert calls + for call in calls: + assert call.request.req_kwargs["timeout"] == boot.REQUEST_TIMEOUT, ( + call.request.url + ) + + +AUTOLAND_VARIANTS = re.compile( + r"https://hg.mozilla.org/integration/autoland/raw-file/[^/]+/taskcluster/test_configs/variants.yml" +) + + +@pytest.fixture(autouse=True) +def mock_test_variants() -> None: + # mozci memoizes the test variants. + mozci_task.fetch_test_variant_yaml.clear() + # The suffixes of the test variants, of mozilla-central (retrieved by mozci) and of the pushes. + responses.add( + responses.GET, + "https://hg.mozilla.org/mozilla-central/raw-file/default/taskcluster/test_configs/variants.yml", + body="xorigin:\n suffix: xorig\n", + ) + responses.add(responses.GET, AUTOLAND_VARIANTS, body="xorigin:\n suffix: xorig\n") + + +def test_get_config_selection_tasks() -> None: + assert ( + boot.get_config_selection_tasks(TASK_GRAPH, KNOWN_TASKS) + == CONFIG_SELECTION_TASKS + ) + + +def test_retrieve_schedulable_tasks( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + # No tasks were retrieved before. + with contextlib.suppress(FileNotFoundError): + os.remove(boot.CONFIG_SELECTION_TASKS_PATH) + # A transient failure is retried. + responses.add( + responses.GET, + "https://hg.mozilla.org/integration/autoland/json-pushes", + body=requests.ConnectionError("connection reset"), + ) + responses.add( + responses.GET, + "https://hg.mozilla.org/integration/autoland/json-pushes", + json={ + "pushes": { + "1": {"changesets": ["rev1"]}, + "2": {"changesets": ["rev2"]}, + "3": {"changesets": ["rev3"]}, + } + }, + ) + for rev, backstop in (("rev1", True), ("rev2", True), ("rev3", False)): + responses.add( + responses.GET, + boot._decision_artifact_url(rev, "target-tasks.json"), + json=sorted(KNOWN_TASKS), + ) + responses.add( + responses.GET, + boot._decision_artifact_url(rev, "parameters.yml"), + body=f"backstop: {'true' if backstop else 'false'}\n", + ) + # The tasks come from the latest backstop push with some: the graph of rev2 has none. + responses.add( + responses.GET, + boot._decision_artifact_url("rev2", "task-graph.json"), + json={"g": TASK_GRAPH["g"]}, + ) + responses.add( + responses.GET, + boot._decision_artifact_url("rev1", "task-graph.json"), + json=TASK_GRAPH, + ) + boot.retrieve_schedulable_tasks.retry_with(wait=tenacity.wait_none())() + assert_requests_have_timeouts() + + with open("known_tasks") as f: + assert set(f.read().split("\n")) == KNOWN_TASKS + with open("config_selection_tasks.json") as f: + assert json.load(f) == { + "backstop": "rev1", + "tasks": CONFIG_SELECTION_TASKS, + "new_tasks": {}, + # The test tasks of the backstop push which aren't eligible. + "ignored": [ + "test-linux2404-64-shippable/opt-mochitest-plain", + "test-linux2404-64/debug-mochitest-plain", + "test-linux2404-64/opt-mochitest-plain-standalone", + ], + "unresolved": {}, + "processed": ["rev2", "rev3"], + "not_backstops": ["rev3"], + } + + +def test_get_full_graph_tasks() -> None: + graph = {"a": TASK_GRAPH["a"], "c": TASK_GRAPH["c"], "g": TASK_GRAPH["g"]} + graph = {task["label"]: task for task in graph.values()} + labels = {"test-linux2404-64/opt-mochitest-plain-1", "build-linux64/opt"} + expected = {label: graph[label] for label in labels} + # The definitions are found in the text, as the full task graph is compact JSON. + assert boot.get_full_graph_tasks(orjson.dumps(graph), labels) == expected + # Otherwise, the whole graph is parsed. + assert ( + boot.get_full_graph_tasks(json.dumps(graph, indent=2).encode(), labels) + == expected + ) + # Tasks which aren't in the graph are left out. + assert ( + boot.get_full_graph_tasks(orjson.dumps(graph), {"test-unknown/opt-x-1"}) == {} + ) + + +def test_update_config_selection_tasks( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + # No tasks were retrieved before. + with contextlib.suppress(FileNotFoundError): + os.remove(boot.CONFIG_SELECTION_TASKS_PATH) + renamed = _task( + "test-linux2404-64/opt-mochitest-plain-renamed-1", + "linux2404-64/opt", + ["dom/a.toml"], + ) + renamed_shippable = _task( + "test-linux2404-64-shippable/opt-mochitest-plain-renamed-1", + "linux2404-64-shippable/opt", + ["dom/a.toml"], + ) + for rev, backstop in ( + ("b1", True), + ("p1", False), + ("p2", False), + ("p3", False), + ("b2", True), + ): + responses.add( + responses.GET, + boot._decision_artifact_url(rev, "parameters.yml"), + body=f"backstop: {'true' if backstop else 'false'}\n", + ) + responses.add( + responses.GET, + boot._decision_artifact_url("b1", "task-graph.json"), + json=TASK_GRAPH, + ) + # A target task of the backstop pushes isn't in their task graph: it's optimized otherwise + # than by test selection. + jittest = "test-linux2404-64/opt-jittest-1proc-1" + responses.add( + responses.GET, + boot._decision_artifact_url("b1", "target-tasks.json"), + json=sorted(KNOWN_TASKS) + [jittest], + ) + responses.add( + responses.GET, + boot._decision_artifact_url("b2", "target-tasks.json"), + json=sorted(KNOWN_TASKS) + [jittest, renamed["label"]], + ) + # A push after the backstop push has tasks renamed since then (found among its target tasks, + # as they could have been optimized away). Shippable ones are still ignored. + responses.add( + responses.GET, + boot._decision_artifact_url("p1", "target-tasks.json"), + json=sorted(KNOWN_TASKS) + + [jittest, renamed["label"], renamed_shippable["label"]], + ) + responses.add( + responses.GET, + boot._decision_artifact_url("p1", "full-task-graph.json"), + body=orjson.dumps( + { + task["label"]: task + for task in (*TASK_GRAPH.values(), renamed, renamed_shippable) + } + ), + ) + # The target tasks of the next one aren't there yet, then they are (without unknown tasks, so + # its full task graph isn't needed). + responses.add( + responses.GET, + boot._decision_artifact_url("p2", "target-tasks.json"), + status=404, + ) + responses.add( + responses.GET, + boot._decision_artifact_url("p2", "target-tasks.json"), + json=sorted(KNOWN_TASKS) + [renamed["label"], renamed_shippable["label"]], + ) + + # The push after it is processed meanwhile. + responses.add( + responses.GET, + boot._decision_artifact_url("p3", "target-tasks.json"), + json=sorted(KNOWN_TASKS), + ) + + def read() -> dict: + with open("config_selection_tasks.json") as f: + return json.load(f) + + ignored = [ + "test-linux2404-64-shippable/opt-mochitest-plain", + "test-linux2404-64/debug-mochitest-plain", + "test-linux2404-64/opt-jittest-1proc", + "test-linux2404-64/opt-mochitest-plain-standalone", + ] + boot.update_config_selection_tasks(["b1", "p1", "p2", "p3"], KNOWN_TASKS) + assert read() == { + "backstop": "b1", + "tasks": CONFIG_SELECTION_TASKS, + "new_tasks": { + "test-linux2404-64/opt-mochitest-plain-renamed": "test-linux2404-64/opt-*" + }, + "ignored": sorted( + ignored + ["test-linux2404-64-shippable/opt-mochitest-plain-renamed"] + ), + "unresolved": {}, + "processed": ["p1", "p3"], + "not_backstops": ["p1", "p2", "p3"], + } + + # p2 is processed once its target tasks are there. + boot.update_config_selection_tasks(["b1", "p1", "p2", "p3"], KNOWN_TASKS) + assert read()["processed"] == ["p1", "p2", "p3"] + # The full task graph was only needed for p1's renamed tasks: the jittest task was already + # known not to be eligible. + full_graphs = [ + call.request.url + for call in responses.calls + if call.request.url.endswith("full-task-graph.json") + ] + assert full_graphs == [boot._decision_artifact_url("p1", "full-task-graph.json")] + + # A newer backstop push replaces the tasks, which include the renamed one, even if the target + # tasks of the recent pushes are from before the rename (they are only refreshed daily). The + # tasks found not to be eligible before are kept as long as recent pushes target them (the + # shippable renamed one isn't among the tasks or the target tasks of the new backstop push). + responses.add( + responses.GET, + boot._decision_artifact_url("b2", "task-graph.json"), + json={**TASK_GRAPH, "r": renamed}, + ) + boot.update_config_selection_tasks( + ["b1", "p1", "p2", "p3", "b2"], KNOWN_TASKS | {renamed_shippable["label"]} + ) + assert read() == { + "backstop": "b2", + "tasks": { + **CONFIG_SELECTION_TASKS, + "test-linux2404-64/opt-mochitest-plain-renamed": { + "config": "test-linux2404-64/opt-*", + "manifests": ["dom/a.toml"], + }, + }, + "new_tasks": {}, + "ignored": sorted( + ignored + ["test-linux2404-64-shippable/opt-mochitest-plain-renamed"] + ), + "unresolved": {}, + "processed": [], + "not_backstops": ["p1", "p2", "p3"], + } + # Pushes already known not to be backstop pushes aren't checked again. + parameters_calls = [ + call.request.url + for call in responses.calls + if call.request.url.endswith("parameters.yml") + ] + assert len(parameters_calls) == len(set(parameters_calls)) + assert_requests_have_timeouts() + + +def test_update_config_selection_tasks_new_variant( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + with contextlib.suppress(FileNotFoundError): + os.remove(boot.CONFIG_SELECTION_TASKS_PATH) + # A test variant added on autoland, which isn't on mozilla-central yet. + new_variant = _task( + "test-linux2404-64/opt-mochitest-plain-newv-1", + "linux2404-64/opt", + ["dom/a.toml"], + runtime={"newvariant": True}, + ) + # A task of the same variant in a later push. + renamed = _task( + "test-linux2404-64/opt-mochitest-plain-renamed-newv-1", + "linux2404-64/opt", + ["dom/a.toml"], + runtime={"newvariant": True}, + ) + graph = {**TASK_GRAPH, "n": new_variant} + for rev, backstop in (("b1", True), ("p1", False)): + responses.add( + responses.GET, + boot._decision_artifact_url(rev, "parameters.yml"), + body=f"backstop: {'true' if backstop else 'false'}\n", + ) + responses.add( + responses.GET, boot._decision_artifact_url("b1", "task-graph.json"), json=graph + ) + targets = sorted(KNOWN_TASKS) + [new_variant["label"]] + responses.add( + responses.GET, + boot._decision_artifact_url("b1", "target-tasks.json"), + json=targets, + ) + responses.add( + responses.GET, + boot._decision_artifact_url("p1", "target-tasks.json"), + json=targets + [renamed["label"]], + ) + responses.add( + responses.GET, + boot._decision_artifact_url("p1", "full-task-graph.json"), + body=orjson.dumps({**graph, renamed["label"]: renamed}), + ) + responses.replace( + responses.GET, + AUTOLAND_VARIANTS, + body="xorigin:\n suffix: xorig\nnewvariant:\n suffix: newv\n", + ) + + boot.update_config_selection_tasks(["b1", "p1"], KNOWN_TASKS) + + with open(boot.CONFIG_SELECTION_TASKS_PATH) as f: + state = json.load(f) + # The configs are named with the push's test variants, on the first update. + assert state["tasks"]["test-linux2404-64/opt-mochitest-plain-newv"] == { + "config": "test-linux2404-64/opt-*-newv", + "manifests": ["dom/a.toml"], + } + assert state["new_tasks"] == { + "test-linux2404-64/opt-mochitest-plain-renamed-newv": "test-linux2404-64/opt-*-newv" + } + + +def test_update_config_selection_tasks_retries_unresolved( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + with contextlib.suppress(FileNotFoundError): + os.remove(boot.CONFIG_SELECTION_TASKS_PATH) + # Test variants added on autoland, which aren't on mozilla-central yet: the latest one isn't in + # the variants of the pushes at first either (e.g. a transient failure). + new_variant = _task( + "test-linux2404-64/opt-mochitest-plain-newv-1", + "linux2404-64/opt", + ["dom/a.toml"], + runtime={"newvariant": True}, + ) + later_variant = [ + _task( + f"test-linux2404-64/opt-mochitest-plain-later-{chunk}", + "linux2404-64/opt", + [f"dom/later{chunk}.toml"], + runtime={"latervariant": True}, + ) + for chunk in (1, 2) + ] + # A task of a variant added after the backstop push, in a later push. + renamed = _task( + "test-linux2404-64/opt-mochitest-plain-renamed-1", + "linux2404-64/opt", + ["dom/a.toml"], + runtime={"latervariant": True}, + ) + graph = { + **TASK_GRAPH, + "n": new_variant, + **{task["label"]: task for task in later_variant}, + } + for rev, backstop in (("b1", True), ("p1", False), ("p2", False)): + responses.add( + responses.GET, + boot._decision_artifact_url(rev, "parameters.yml"), + body=f"backstop: {'true' if backstop else 'false'}\n", + ) + responses.add( + responses.GET, boot._decision_artifact_url("b1", "task-graph.json"), json=graph + ) + targets = sorted(KNOWN_TASKS) + [ + task["label"] for task in (new_variant, *later_variant) + ] + responses.add( + responses.GET, + boot._decision_artifact_url("b1", "target-tasks.json"), + json=targets, + ) + for rev in ("p1", "p2"): + responses.add( + responses.GET, + boot._decision_artifact_url(rev, "target-tasks.json"), + json=targets + [renamed["label"]], + ) + responses.add( + responses.GET, + boot._decision_artifact_url("p1", "full-task-graph.json"), + body=orjson.dumps({**graph, renamed["label"]: renamed}), + ) + # The variants of the pushes don't have the latest one yet (e.g. a transient failure). + responses.replace( + responses.GET, + AUTOLAND_VARIANTS, + body="xorigin:\n suffix: xorig\nnewvariant:\n suffix: newv\n", + ) + + def read() -> dict: + with open(boot.CONFIG_SELECTION_TASKS_PATH) as f: + return json.load(f) + + boot.update_config_selection_tasks(["b1", "p1", "p2"], KNOWN_TASKS) + state = read() + # The configs are named with the push's test variants. + assert state["tasks"]["test-linux2404-64/opt-mochitest-plain-newv"] == { + "config": "test-linux2404-64/opt-*-newv", + "manifests": ["dom/a.toml"], + } + # The tasks whose config can't be computed aren't candidates or new tasks, nor ignored: they are + # pending, and other pushes targeting them don't need their full task graph again. + later = "test-linux2404-64/opt-mochitest-plain-later" + assert later not in state["tasks"] and later not in state["ignored"] + assert "test-linux2404-64/opt-mochitest-plain-renamed" not in state["new_tasks"] + assert sorted(state["unresolved"]) == ["b1", "p1"] + assert state["processed"] == ["p1", "p2"] + full_graphs = [ + call.request.url + for call in responses.calls + if call.request.url.endswith("full-task-graph.json") + ] + assert full_graphs == [boot._decision_artifact_url("p1", "full-task-graph.json")] + + # Once the variants are known, they are resolved on the next update. + responses.replace( + responses.GET, + AUTOLAND_VARIANTS, + body="xorigin:\n suffix: xorig\nnewvariant:\n suffix: newv\nlatervariant:\n suffix: later\n", + ) + boot.update_config_selection_tasks(["b1", "p1", "p2"], KNOWN_TASKS) + state = read() + assert state["tasks"][later] == { + "config": "test-linux2404-64/opt-*-later", + "manifests": ["dom/later1.toml", "dom/later2.toml"], + } + assert state["new_tasks"] == { + "test-linux2404-64/opt-mochitest-plain-renamed": "test-linux2404-64/opt-*-later" + } + assert state["unresolved"] == {} diff --git a/http_service/tests/test_schedule_tests.py b/http_service/tests/test_schedule_tests.py index 05ee855456..335fa003a6 100644 --- a/http_service/tests/test_schedule_tests.py +++ b/http_service/tests/test_schedule_tests.py @@ -17,7 +17,16 @@ from bugbug.models import testselect from bugbug_http import models -from .conftest import MOCK_KNOWN_TASKS, MockModelCache +from .conftest import MOCK_CONFIG_SELECTION_TASKS, MOCK_KNOWN_TASKS, MockModelCache + + +def task_labels(group: str, configs: list[str]) -> list[str]: + """The labels of the mock tasks running the group on the configs.""" + return sorted( + label + for label, task in MOCK_CONFIG_SELECTION_TASKS.items() + if task["config"] in configs and group in task["manifests"] + ) @pytest.mark.parametrize( @@ -105,7 +114,7 @@ def test_simple_schedule( value = models.redis.get(f"bugbug:job_result:schedule_tests:mozilla-central_{rev}") assert value is not None result = orjson.loads(zstandard.ZstdDecompressor().decompress(value)) - assert len(result) == 7 + assert len(result) == 8 assert result["tasks"] == labels_to_choose assert result["groups"] == groups_to_choose assert result["reduced_tasks"] == reduced_labels @@ -113,9 +122,13 @@ def test_simple_schedule( assert result["known_tasks"] == MOCK_KNOWN_TASKS # The mock models don't have confidence thresholds. assert result["confidence_thresholds"] == {} - assert {k: set(v) for k, v in result["config_groups"].items()} == { + assert {k: set(v) for k, v in result["config_group_configs"].items()} == { k: set(v) for k, v in config_groups.items() } + # The taskgraph is given the labels of the tasks of the selected configs. + assert result["config_groups"] == { + group: task_labels(group, configs) for group, configs in config_groups.items() + } def test_schedule_with_confidence_thresholds( @@ -566,10 +579,65 @@ def select_configs(groups, modified_paths, very_likely_groups, used_configs=()): ({"test-group1": 0.99}, []), ({"test-group2": group2_confidence}, ["test-linux1804-64/opt-*"]), ] - assert result["config_groups"] == { + assert result["config_group_configs"] == { "test-group1": ["test-linux1804-64/opt-*"], "test-group2": ["test-windows10/opt-*"], } + assert result["config_groups"] == { + "test-group1": ["test-linux1804-64/opt-mochitest"], + "test-group2": ["test-windows10/opt-mochitest"], + } + + +def test_schedule_config_groups_with_new_tasks( + monkeypatch: pytest.MonkeyPatch, + mock_hgmo: None, + mock_repo: tuple[str, str], + mock_component_taskcluster_artifact: None, + mock_coverage_mapping_artifact: None, + mock_schedule_tests_classify: Callable[[dict[str, float], dict[str, float]], None], +) -> None: + _, remote_repo_dir = mock_repo + with hglib.open(str(remote_repo_dir)) as hg: + rev = hg.log()[0].node.decode("ascii")[:12] + + mock_schedule_tests_classify({}, {"test-group1": 0.99}) + + # A task renamed since the backstop push the other tasks come from. + with open("config_selection_tasks.json", "w") as f: + json.dump( + { + "backstop": "backstop-rev", + "tasks": MOCK_CONFIG_SELECTION_TASKS, + "new_tasks": { + "test-linux1804-64/opt-mochitest-renamed": "test-linux1804-64/opt-*" + }, + "ignored": [], + "unresolved": {}, + "processed": [], + "not_backstops": [], + }, + f, + ) + monkeypatch.setattr( + models, + "select_configs", + lambda groups, modified_paths, very_likely_groups, used_configs=(): { + group: ["test-linux1804-64/opt-*"] for group in groups + }, + ) + + assert models.schedule_tests("mozilla-central", rev) == "OK" + + value = models.redis.get(f"bugbug:job_result:schedule_tests:mozilla-central_{rev}") + assert value is not None + result = orjson.loads(zstandard.ZstdDecompressor().decompress(value)) + assert result["config_groups"] == { + "test-group1": [ + "test-linux1804-64/opt-mochitest", + "test-linux1804-64/opt-mochitest-renamed", + ] + } def test_select_configs_never_vetoes_groups(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/test_test_scheduling.py b/tests/test_test_scheduling.py index 778e50528a..248db8dbfd 100644 --- a/tests/test_test_scheduling.py +++ b/tests/test_test_scheduling.py @@ -1711,9 +1711,7 @@ def test_get_config_select_regressions() -> None: (["rev3"], None, [("test-windows11-64-24h2/opt-*", G)], [], []), ] - regressions, recent_configs = test_scheduling.get_config_select_regressions( - lambda: iter(pushes), recent_pushes=2 - ) + regressions = test_scheduling.get_config_select_regressions(lambda: iter(pushes)) culprit, later = regressions assert culprit == test_scheduling.ConfigSelectRegression( @@ -1723,8 +1721,6 @@ def test_get_config_select_regressions() -> None: assert later == test_scheduling.ConfigSelectRegression( 1, (Revision("rev1"),), G, frozenset({ANDROID}), frozenset({ANDROID, MAC}) ) - # The configs that ran in the last two pushes. - assert recent_configs == {LINUX, "test-windows11-64-24h2/opt-*"} def test_get_config_select_regressions_failure_after_later_pass() -> None: @@ -1747,7 +1743,7 @@ def test_get_config_select_regressions_failure_after_later_pass() -> None: (["rev4"], None, [], [], []), ] - regressions, _ = test_scheduling.get_config_select_regressions(lambda: iter(pushes)) + regressions = test_scheduling.get_config_select_regressions(lambda: iter(pushes)) assert regressions[0] == test_scheduling.ConfigSelectRegression( 0, (Revision("rev0"),), G, frozenset({WIN}), frozenset({WIN, LINUX}) @@ -1767,7 +1763,7 @@ def test_get_config_select_regressions_ignored_configs_only() -> None: (["rev1"], None, [(VERIFY, G)], [(VERIFY, G)], []), ] - regressions, _ = test_scheduling.get_config_select_regressions(lambda: iter(pushes)) + regressions = test_scheduling.get_config_select_regressions(lambda: iter(pushes)) assert regressions == [ test_scheduling.ConfigSelectRegression( @@ -1790,9 +1786,7 @@ def test_get_config_select_regressions_window_is_capped(monkeypatch) -> None: (["rev4"], None, [], [], []), ] - (regression,), _ = test_scheduling.get_config_select_regressions( - lambda: iter(pushes) - ) + (regression,) = test_scheduling.get_config_select_regressions(lambda: iter(pushes)) assert regression.observed == frozenset({A, B}) diff --git a/tests/test_testconfigdimensions.py b/tests/test_testconfigdimensions.py index 4c110653f5..ac8b26825e 100644 --- a/tests/test_testconfigdimensions.py +++ b/tests/test_testconfigdimensions.py @@ -149,7 +149,7 @@ def mock_data(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( testconfigdimensions, "load_regressions", - lambda with_patches=True: (regressions, {LINUX, LINUX_DEBUG, WINDOWS, MAC}, {}), + lambda with_patches=True: (regressions, {}), ) @@ -224,7 +224,7 @@ def test_class_model_class_only_in_test_set(monkeypatch) -> None: monkeypatch.setattr( testconfigdimensions, "load_regressions", - lambda with_patches=True: (regressions, {LINUX, WINDOWS, MAC, android}, {}), + lambda with_patches=True: (regressions, {}), ) model = testconfigdimensions.TestConfigSelectClassModel() classes, class_names = model.get_labels() diff --git a/tests/test_testconfigselect.py b/tests/test_testconfigselect.py index 516a008bf8..58091556b6 100644 --- a/tests/test_testconfigselect.py +++ b/tests/test_testconfigselect.py @@ -130,37 +130,62 @@ def test_generate_rows() -> None: ] -def test_live_candidates() -> None: - known_configs = { - "test-linux2404-64/opt-*", - "test-linux2404-64/opt-*-swr", - "test-macosx1015-64-qr/opt-*", +def test_live_tasks() -> None: + LiveTask = testconfigselect.LiveTask + tasks = { + "test-linux2404-64/opt-mochitest-browser-chrome": LiveTask( + "test-linux2404-64/opt-*", [G2] + ), + # The configs of some tasks aren't named after the platform of their label. + "test-windows11-64-25h2-asan/opt-mochitest-browser-chrome-swr": LiveTask( + "test-windows11-64-25h2/asan-*-swr", [G2] + ), + # Groups are named as by the models. + "test-linux2404-64/opt-web-platform-tests": LiveTask( + "test-linux2404-64/opt-*", ["/css/cssom-view", f"{G}:other.toml"] + ), } - manifests_by_task: dict[str, list[str]] = { - "test-linux2404-64/opt-mochitest-browser-chrome-1": [G2], - "test-linux2404-64/opt-mochitest-browser-chrome-swr-2": [G2], - # A platform not in the training data yet: named after its label. - "test-macosx1500-aarch64/opt-mochitest-browser-chrome-3": [G2], - "test-linux2404-64/opt-mochitest-plain-4": [f"{G}:other.toml"], - # Web-platform test groups are relative to the root of their suite. - "test-linux2404-64/opt-web-platform-tests-5": [ - "/css/cssom-view", - "/_mozilla/fetch", - ], - "build-linux64/opt": [], + tasks_by_group = testconfigselect.live_tasks(tasks) + assert tasks_by_group == { + G2: { + "test-linux2404-64/opt-*": [ + "test-linux2404-64/opt-mochitest-browser-chrome" + ], + "test-windows11-64-25h2/asan-*-swr": [ + "test-windows11-64-25h2-asan/opt-mochitest-browser-chrome-swr" + ], + }, + "testing/web-platform/tests/css/cssom-view": { + "test-linux2404-64/opt-*": ["test-linux2404-64/opt-web-platform-tests"] + }, + G: {"test-linux2404-64/opt-*": ["test-linux2404-64/opt-web-platform-tests"]}, } - assert testconfigselect.live_candidates(manifests_by_task, known_configs) == { - # The most specific known pattern names each task; the old macOS config isn't a candidate - # as no current task matches it. - G2: [ - "test-linux2404-64/opt-*", - "test-linux2404-64/opt-*-swr", - "test-macosx1500-aarch64/opt-mochitest-browser-chrome", - ], - G: ["test-linux2404-64/opt-*"], - "testing/web-platform/tests/css/cssom-view": ["test-linux2404-64/opt-*"], - "testing/web-platform/mozilla/tests/fetch": ["test-linux2404-64/opt-*"], + + assert testconfigselect.live_candidates(tasks_by_group)[G2] == [ + "test-linux2404-64/opt-*", + "test-windows11-64-25h2/asan-*-swr", + ] + assert testconfigselect.task_labels( + tasks_by_group, + G2, + ["test-windows11-64-25h2/asan-*-swr", "test-linux2404-64/opt-*"], + ) == [ + "test-linux2404-64/opt-mochitest-browser-chrome", + "test-windows11-64-25h2-asan/opt-mochitest-browser-chrome-swr", + ] + # Configs without tasks running the group (e.g. of another group) have no tasks. + assert testconfigselect.task_labels(tasks_by_group, G, ["test-other/opt-*"]) == [] + # Tasks whose manifests aren't known (e.g. renamed) are given for all the groups of their config. + new_tasks = { + "test-linux2404-64/opt-mochitest-browser-chrome-renamed": "test-linux2404-64/opt-*", + "test-macosx1500-aarch64/opt-mochitest-browser-chrome": "test-macosx1500-aarch64/opt-*", } + assert testconfigselect.task_labels( + tasks_by_group, G2, ["test-linux2404-64/opt-*"], new_tasks + ) == [ + "test-linux2404-64/opt-mochitest-browser-chrome", + "test-linux2404-64/opt-mochitest-browser-chrome-renamed", + ] def synthetic_regressions() -> list[ConfigSelectRegression]: @@ -189,7 +214,7 @@ def trained_model( monkeypatch.setattr( testconfigselect.test_scheduling, "get_config_select_regressions", - lambda push_data_factory: (regressions, {LINUX, LINUX_DEBUG, WINDOWS, MAC}), + lambda push_data_factory: regressions, ) monkeypatch.setattr( testconfigselect.repository, diff --git a/uv.lock b/uv.lock index cf319b9699..6d67aaa9a4 100644 --- a/uv.lock +++ b/uv.lock @@ -819,6 +819,7 @@ dependencies = [ { name = "gunicorn" }, { name = "kombu" }, { name = "marshmallow" }, + { name = "pyyaml" }, { name = "rq" }, { name = "rq-dashboard" }, { name = "sentry-sdk", extra = ["flask"] }, @@ -842,6 +843,7 @@ requires-dist = [ { name = "gunicorn", specifier = ">=25.1,<26.3" }, { name = "kombu", specifier = ">=5.6.2,<5.8.0" }, { name = "marshmallow", specifier = ">=3.18.0" }, + { name = "pyyaml", specifier = ">=6.0.3" }, { name = "rq", specifier = ">=2.7,<2.13" }, { name = "rq-dashboard", specifier = ">=0.8.6,<0.10.0" }, { name = "sentry-sdk", extras = ["flask"], specifier = ">=2.60.0" },