Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/zarr-metadata/changes/4635.bugfix.md
Original file line number Diff line number Diff line change
@@ -1 +1 @@
Strategies in `zarr_metadata.testing.strategies` that draw codec pipelines, such as `array_documents` and `pipelines`, now draw the same examples on every derandomized Hypothesis run. A codec was sampled from options that hold strategies built during the draw, which Hypothesis labels by memory address, so the examples its mutator produced varied from one process to the next.
Strategies in `zarr_metadata.testing.strategies` that draw codec pipelines, such as `array_documents` and `pipelines`, now draw the same examples on every derandomized Hypothesis run. A codec was sampled from options that hold strategies, which Hypothesis labels by their memory address, so the examples its mutator produced varied from one process to the next.
16 changes: 10 additions & 6 deletions packages/zarr-metadata/src/zarr_metadata/testing/_draw.py
Original file line number Diff line number Diff line change
Expand Up @@ -278,12 +278,16 @@ def pipelines_for(
def _codec(
draw: st.DrawFn, options: Sequence[tuple[K, Configurations]]
) -> tuple[K, ZarrV3DeclarationJSON]:
# An index, not the options: Hypothesis labels a `sampled_from` by
# hashing its elements, and each option holds a strategy built for this
# draw, whose hash is its address. Labels that follow addresses send
# Hypothesis's mutator down different paths from one run to the next, so
# a derandomized run would not draw the same examples twice.
definition, configurations = options[draw(st.sampled_from(range(len(options))))]
# A name, not an option: Hypothesis labels a `sampled_from` by hashing its
# elements, and an option holds a strategy, which hashes by its `id`. A
# label that follows where a strategy sits in memory sends Hypothesis's
# mutator down other paths from one run to the next, so a derandomized
# run would not draw the same examples twice. Names also keep one set of
# options apart from another, as an index into either would not.
named = {
definition.name: (definition, configurations) for definition, configurations in options
}
definition, configurations = named[draw(st.sampled_from(list(named)))]
return definition, draw(any_spelling(definition, configurations))


Expand Down
99 changes: 67 additions & 32 deletions packages/zarr-metadata/tests/testing/_shares.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,61 +4,55 @@
as it takes, which says nothing of whether a test, at the 100 examples it
runs by default, ever meets it. Runs are derandomized, so a share is a fixed
number for a given strategy. A change to the strategy draws other examples,
as another seed would, so a bound sits where the share stays across seeds:
this is no place to bound what is drawn a few percent of the time.
as another seed would, and so does a change to the line in `_draw` that
Hypothesis derandomizes from, which reseeds every share at once. So a bound
sits where the share stays across seeds: this is no place to bound what is
drawn a few percent of the time.

A bound is set from the shares of 100 seeds, each run as `examples_of`
runs, but with `derandomize=False` under `hypothesis.seed`. It lies past
the most extreme share on its side by a quarter of that share's distance
from the median, rounded outward to a hundredth.
A bound is reset with `bounds(across_seeds(strategy, statistic))`, which
runs 100 seeds and applies the rule `bounds` states.
"""

from __future__ import annotations

from typing import TYPE_CHECKING, TypeVar
import functools
import math
from typing import TYPE_CHECKING, Any, TypeVar, cast

from hypothesis import HealthCheck, given, settings
from hypothesis import HealthCheck, given, seed, settings

if TYPE_CHECKING:
from collections.abc import Callable, Sequence
from collections.abc import Callable, Iterable, Sequence

from hypothesis.strategies import SearchStrategy

T = TypeVar("T")

_DERANDOMIZED = settings(
_SETTINGS = settings(
database=None,
derandomize=True,
deadline=None,
suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow],
)


def examples_of(strategy: SearchStrategy[T], *, examples: int = 300) -> list[T]:
"""The examples a derandomized run of `strategy` draws, checked to be the same in a second run.
def examples_of(strategy: SearchStrategy[T], *, examples: int = 300) -> tuple[T, ...]:
"""The examples a derandomized run of `strategy` draws, drawn once per strategy and checked against a second run.

Hypothesis's mutator copies spans between the parts of an example that
share a label, and a label can follow an object's address: a
`sampled_from` of elements that hold a strategy built for the draw is
labeled by that strategy's `id`. Such a strategy draws other examples
from run to run, and from machine to machine, however derandomized. A
second run in the same process lays objects out at other addresses, so
it catches this.
`sampled_from` of elements that hold a strategy is labeled by that
strategy's `id`. Such a strategy draws other examples from run to run,
and from machine to machine, however derandomized. A second run in the
same process often lays objects out at other addresses, so it often
catches this, but it is a tripwire, not a proof.
"""
first = _draw(strategy, examples)
second = _draw(strategy, examples)
if [repr(value) for value in first] != [repr(value) for value in second]:
raise AssertionError(
f"{strategy!r} drew other examples on a second derandomized run. "
"A strategy built during a draw is likely in a `sampled_from`, which "
"labels it by its address; sample an index into the options instead."
)
return first
return cast("tuple[T, ...]", _checked(strategy, examples))


def share(values: Sequence[T], counts: Callable[[T], bool]) -> float:
def share(values: Iterable[T], counts: Callable[[T], bool]) -> float:
"""The share of `values` that `counts` holds of."""
return sum(counts(value) for value in values) / len(values)
tally = [counts(value) for value in values]
return sum(tally) / len(tally)


def share_of(
Expand All @@ -68,10 +62,51 @@ def share_of(
return share(examples_of(strategy, examples=examples), counts)


def _draw(strategy: SearchStrategy[T], examples: int) -> list[T]:
def across_seeds(
strategy: SearchStrategy[T],
statistic: Callable[[Sequence[T]], float],
*,
seeds: Iterable[int] = range(100),
examples: int = 300,
) -> list[float]:
"""`statistic` of the examples of `strategy` under each of `seeds`, as a change to the strategy would move it."""
return [statistic(_draw(strategy, examples, chosen)) for chosen in seeds]


def bounds(statistics: Sequence[float]) -> tuple[float, float]:
"""A low and a high bound for a statistic measured across seeds.

Each lies past the most extreme measure on its side by a quarter of that
measure's distance from the median, rounded outward to a hundredth.
"""
ordered = sorted(statistics)
median = ordered[len(ordered) // 2]
low = ordered[0] - (median - ordered[0]) / 4
high = ordered[-1] + (ordered[-1] - median) / 4
return math.floor(round(low * 100, 6)) / 100, math.ceil(round(high * 100, 6)) / 100


@functools.cache
def _checked(strategy: SearchStrategy[Any], examples: int) -> tuple[Any, ...]:
first = _draw(strategy, examples)
second = _draw(strategy, examples)
if [repr(value) for value in first] != [repr(value) for value in second]:
raise AssertionError(
f"{strategy!r} drew other examples on a second derandomized run. "
"A strategy is likely in a `sampled_from`, which labels it by its "
"address; sample a name or an index instead."
)
return first


def _draw(strategy: SearchStrategy[T], examples: int, chosen: int | None = None) -> tuple[T, ...]:
"""Derandomized examples of `strategy`, or seeded with `chosen`."""
drawn: list[T] = []
# A lambda, not `drawn.append`: Hypothesis cannot wrap a builtin method,
# and derandomizes from the test function's source, so this line is the seed.
run = given(strategy)(lambda value: drawn.append(value))
settings(_DERANDOMIZED, max_examples=examples)(run)()
return drawn
if chosen is None:
settings(_SETTINGS, max_examples=examples, derandomize=True)(run)()
else:
settings(_SETTINGS, max_examples=examples, derandomize=False)(seed(chosen)(run))()
return tuple(drawn)
33 changes: 19 additions & 14 deletions packages/zarr-metadata/tests/testing/test_documents.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,14 @@

from __future__ import annotations

import functools
import json
from typing import TYPE_CHECKING, Any, cast

import pytest
from hypothesis import find, given, settings
from hypothesis import strategies as st

from tests.testing._shares import examples_of, share
from tests.testing._shares import share_of
from zarr_metadata.definition import AcceptedDeclaration, Definition, resolve
from zarr_metadata.definition._definition import canonicalize
from zarr_metadata.model import (
Expand Down Expand Up @@ -340,29 +339,35 @@ def test_a_data_type_no_codec_takes_is_not_drawn(document: Mapping[str, Any]) ->
assert document["data_type"] == "int8"


_ARRAY_DOCUMENTS = array_documents()
"""One strategy for every share, so `examples_of` draws its examples once."""

_DOCUMENT_SHARES: dict[str, tuple[Callable[[Mapping[str, Any]], bool], float, float]] = {
"no axes": (lambda d: d["shape"] == (), 0.04, 0.34),
"more axes than five": (lambda d: len(d["shape"]) > 5, 0.15, 1),
"no axes": (lambda d: d["shape"] == (), 0.07, 0.31),
"more axes than five": (lambda d: len(d["shape"]) > 5, 0.16, 1),
"a zero-length axis": (lambda d: 0 in d["shape"], 0.44, 1),
"an axis past 32 bits": (lambda d: any(n >= 2**32 for n in d["shape"]), 0.03, 1),
"the first data type": (lambda d: _name(d["data_type"]) == "bool", 0, 0.18),
"an axis past 32 bits": (lambda d: any(n >= 2**32 for n in d["shape"]), 0.02, 1),
}
"""What a share of array documents holds, as a low and a high bound: what a test running only a few hundred examples meets.

Each bound is set as `_shares` says, from 100 seeds.
Each bound is set as `_shares` says.
"""


@functools.cache
def _array_document_examples() -> list[Mapping[str, Any]]:
"""Drawn once for every target, as each counts the same examples."""
return examples_of(array_documents())


@pytest.mark.parametrize("target", _DOCUMENT_SHARES)
def test_array_documents_share(target: str) -> None:
counts, low, high = _DOCUMENT_SHARES[target]
assert low <= share(_array_document_examples(), counts) <= high
assert low <= share_of(_ARRAY_DOCUMENTS, counts) <= high


def test_array_documents_do_not_favor_the_first_data_type() -> None:
"""`bool`, first of the data types by name, is drawn no more often than the rest would allow.

Over 1000 examples: over 300, its share moves too much across seeds to
tell a fair draw from one that draws it a fifth of the time.
"""
share = share_of(_ARRAY_DOCUMENTS, lambda d: _name(d["data_type"]) == "bool", examples=1000)
assert share <= 0.13


def _empty_listings(document: Mapping[str, Any]) -> list[Any]:
Expand Down
26 changes: 26 additions & 0 deletions packages/zarr-metadata/tests/testing/test_shares.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
"""The helpers that measure and bound how often a strategy draws something."""

from __future__ import annotations

import itertools

import pytest
from hypothesis import strategies as st

from tests.testing._shares import bounds, examples_of

_COUNTER = itertools.count()


def test_bounds() -> None:
"""Past each extreme by a quarter of its distance from the median, rounded outward."""
assert bounds([0.1, 0.2, 0.3]) == (0.07, 0.33)
assert bounds([0.3, 0.1, 0.2]) == (0.07, 0.33)
assert bounds([0.5, 0.5, 0.5]) == (0.5, 0.5)
assert bounds([0.0, 0.4, 0.4, 1.0]) == (-0.1, 1.15)


def test_a_strategy_that_draws_otherwise_on_a_second_run_is_an_assertion_error() -> None:
drifting = st.builds(lambda: next(_COUNTER))
with pytest.raises(AssertionError, match="second derandomized run"):
examples_of(drifting, examples=5)
19 changes: 14 additions & 5 deletions packages/zarr-metadata/tests/testing/test_strategies.py
Original file line number Diff line number Diff line change
Expand Up @@ -1068,7 +1068,9 @@ def _float_class(value: object) -> str | None:
)


@pytest.mark.parametrize(("kind", "at_least"), [("normal", 0.2), ("NaN", 0.26), ("infinity", 0.01)])
@pytest.mark.parametrize(
("kind", "at_least"), [("normal", 0.22), ("NaN", 0.25), ("infinity", 0.01)]
)
def test_hex_fill_values_are_not_only_small_numbers(kind: str, at_least: float) -> None:
"""Bits drawn as an integer alone are nearly all subnormal."""
assert share_of(_HEX_FILL_VALUES, lambda value: _float_class(value) == kind) >= at_least
Expand Down Expand Up @@ -1108,7 +1110,7 @@ def _shard_inner_lengths(draw: st.DrawFn) -> tuple[int, ...]:

def test_shards_of_a_rectilinear_grid_are_often_divided() -> None:
"""Edges drawn apart rarely share a factor, which leaves a shard only inner chunks of length 1."""
assert share_of(_shard_inner_lengths(), lambda inner: any(n > 1 for n in inner)) >= 0.39
assert share_of(_shard_inner_lengths(), lambda inner: any(n > 1 for n in inner)) >= 0.36


def _edge_factor(configuration: Mapping[str, Any]) -> int:
Expand Down Expand Up @@ -1156,7 +1158,7 @@ def test_a_long_dimension_keeps_its_edge_factor() -> None:
edges = chunk_grid_configurations(RECTILINEAR_CHUNK_GRID, (2**16 + 1,)).filter(
lambda configuration: not isinstance(configuration["chunk_shapes"][0], int)
)
assert share_of(edges, lambda configuration: _edge_factor(configuration) > 1) >= 0.32
assert share_of(edges, lambda configuration: _edge_factor(configuration) > 1) >= 0.36


def test_divisors_of_a_very_long_shard_are_found_quickly() -> None:
Expand All @@ -1178,5 +1180,12 @@ def _field_types(declaration: Any) -> list[str]: # noqa: ANN401


def test_struct_fields_do_not_favor_the_first_data_type() -> None:
"""A type drawn after a field's name would be `bool` a fifth of the time."""
assert share_of(declarations(STRUCT_DATA_TYPE), lambda d: _field_types(d)[0] == "bool") <= 0.16
"""A type drawn after a field's name would be `bool` a fifth of the time.

Over 1000 examples: over 300, the share of `bool` moves too much across
seeds to tell a fair draw from one that favors it.
"""
first_is_bool = share_of(
declarations(STRUCT_DATA_TYPE), lambda d: _field_types(d)[0] == "bool", examples=1000
)
assert first_is_bool <= 0.13
Loading