diff --git a/packages/zarr-metadata/changes/4635.bugfix.md b/packages/zarr-metadata/changes/4635.bugfix.md index d92c9b5193..791809568b 100644 --- a/packages/zarr-metadata/changes/4635.bugfix.md +++ b/packages/zarr-metadata/changes/4635.bugfix.md @@ -1 +1 @@ -Strategies in `zarr_metadata.testing.strategies` that draw codec pipelines, such as `array_documents` and `pipelines`, now draw the same examples on every derandomized Hypothesis run. A codec was sampled from options that hold strategies built during the draw, which Hypothesis labels by memory address, so the examples its mutator produced varied from one process to the next. +Strategies in `zarr_metadata.testing.strategies` that draw codec pipelines, such as `array_documents` and `pipelines`, now draw the same examples on every derandomized Hypothesis run. A codec was sampled from options that hold strategies, which Hypothesis labels by their memory address, so the examples its mutator produced varied from one process to the next. diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py b/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py index e63d4afe1c..2c42d4f1b3 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py @@ -278,12 +278,16 @@ def pipelines_for( def _codec( draw: st.DrawFn, options: Sequence[tuple[K, Configurations]] ) -> tuple[K, ZarrV3DeclarationJSON]: - # An index, not the options: Hypothesis labels a `sampled_from` by - # hashing its elements, and each option holds a strategy built for this - # draw, whose hash is its address. Labels that follow addresses send - # Hypothesis's mutator down different paths from one run to the next, so - # a derandomized run would not draw the same examples twice. - definition, configurations = options[draw(st.sampled_from(range(len(options))))] + # A name, not an option: Hypothesis labels a `sampled_from` by hashing its + # elements, and an option holds a strategy, which hashes by its `id`. A + # label that follows where a strategy sits in memory sends Hypothesis's + # mutator down other paths from one run to the next, so a derandomized + # run would not draw the same examples twice. Names also keep one set of + # options apart from another, as an index into either would not. + named = { + definition.name: (definition, configurations) for definition, configurations in options + } + definition, configurations = named[draw(st.sampled_from(list(named)))] return definition, draw(any_spelling(definition, configurations)) diff --git a/packages/zarr-metadata/tests/testing/_shares.py b/packages/zarr-metadata/tests/testing/_shares.py index 3d70bf11aa..b69ee73a80 100644 --- a/packages/zarr-metadata/tests/testing/_shares.py +++ b/packages/zarr-metadata/tests/testing/_shares.py @@ -4,61 +4,55 @@ as it takes, which says nothing of whether a test, at the 100 examples it runs by default, ever meets it. Runs are derandomized, so a share is a fixed number for a given strategy. A change to the strategy draws other examples, -as another seed would, so a bound sits where the share stays across seeds: -this is no place to bound what is drawn a few percent of the time. +as another seed would, and so does a change to the line in `_draw` that +Hypothesis derandomizes from, which reseeds every share at once. So a bound +sits where the share stays across seeds: this is no place to bound what is +drawn a few percent of the time. -A bound is set from the shares of 100 seeds, each run as `examples_of` -runs, but with `derandomize=False` under `hypothesis.seed`. It lies past -the most extreme share on its side by a quarter of that share's distance -from the median, rounded outward to a hundredth. +A bound is reset with `bounds(across_seeds(strategy, statistic))`, which +runs 100 seeds and applies the rule `bounds` states. """ from __future__ import annotations -from typing import TYPE_CHECKING, TypeVar +import functools +import math +from typing import TYPE_CHECKING, Any, TypeVar, cast -from hypothesis import HealthCheck, given, settings +from hypothesis import HealthCheck, given, seed, settings if TYPE_CHECKING: - from collections.abc import Callable, Sequence + from collections.abc import Callable, Iterable, Sequence from hypothesis.strategies import SearchStrategy T = TypeVar("T") -_DERANDOMIZED = settings( +_SETTINGS = settings( database=None, - derandomize=True, deadline=None, suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow], ) -def examples_of(strategy: SearchStrategy[T], *, examples: int = 300) -> list[T]: - """The examples a derandomized run of `strategy` draws, checked to be the same in a second run. +def examples_of(strategy: SearchStrategy[T], *, examples: int = 300) -> tuple[T, ...]: + """The examples a derandomized run of `strategy` draws, drawn once per strategy and checked against a second run. Hypothesis's mutator copies spans between the parts of an example that share a label, and a label can follow an object's address: a - `sampled_from` of elements that hold a strategy built for the draw is - labeled by that strategy's `id`. Such a strategy draws other examples - from run to run, and from machine to machine, however derandomized. A - second run in the same process lays objects out at other addresses, so - it catches this. + `sampled_from` of elements that hold a strategy is labeled by that + strategy's `id`. Such a strategy draws other examples from run to run, + and from machine to machine, however derandomized. A second run in the + same process often lays objects out at other addresses, so it often + catches this, but it is a tripwire, not a proof. """ - first = _draw(strategy, examples) - second = _draw(strategy, examples) - if [repr(value) for value in first] != [repr(value) for value in second]: - raise AssertionError( - f"{strategy!r} drew other examples on a second derandomized run. " - "A strategy built during a draw is likely in a `sampled_from`, which " - "labels it by its address; sample an index into the options instead." - ) - return first + return cast("tuple[T, ...]", _checked(strategy, examples)) -def share(values: Sequence[T], counts: Callable[[T], bool]) -> float: +def share(values: Iterable[T], counts: Callable[[T], bool]) -> float: """The share of `values` that `counts` holds of.""" - return sum(counts(value) for value in values) / len(values) + tally = [counts(value) for value in values] + return sum(tally) / len(tally) def share_of( @@ -68,10 +62,51 @@ def share_of( return share(examples_of(strategy, examples=examples), counts) -def _draw(strategy: SearchStrategy[T], examples: int) -> list[T]: +def across_seeds( + strategy: SearchStrategy[T], + statistic: Callable[[Sequence[T]], float], + *, + seeds: Iterable[int] = range(100), + examples: int = 300, +) -> list[float]: + """`statistic` of the examples of `strategy` under each of `seeds`, as a change to the strategy would move it.""" + return [statistic(_draw(strategy, examples, chosen)) for chosen in seeds] + + +def bounds(statistics: Sequence[float]) -> tuple[float, float]: + """A low and a high bound for a statistic measured across seeds. + + Each lies past the most extreme measure on its side by a quarter of that + measure's distance from the median, rounded outward to a hundredth. + """ + ordered = sorted(statistics) + median = ordered[len(ordered) // 2] + low = ordered[0] - (median - ordered[0]) / 4 + high = ordered[-1] + (ordered[-1] - median) / 4 + return math.floor(round(low * 100, 6)) / 100, math.ceil(round(high * 100, 6)) / 100 + + +@functools.cache +def _checked(strategy: SearchStrategy[Any], examples: int) -> tuple[Any, ...]: + first = _draw(strategy, examples) + second = _draw(strategy, examples) + if [repr(value) for value in first] != [repr(value) for value in second]: + raise AssertionError( + f"{strategy!r} drew other examples on a second derandomized run. " + "A strategy is likely in a `sampled_from`, which labels it by its " + "address; sample a name or an index instead." + ) + return first + + +def _draw(strategy: SearchStrategy[T], examples: int, chosen: int | None = None) -> tuple[T, ...]: + """Derandomized examples of `strategy`, or seeded with `chosen`.""" drawn: list[T] = [] # A lambda, not `drawn.append`: Hypothesis cannot wrap a builtin method, # and derandomizes from the test function's source, so this line is the seed. run = given(strategy)(lambda value: drawn.append(value)) - settings(_DERANDOMIZED, max_examples=examples)(run)() - return drawn + if chosen is None: + settings(_SETTINGS, max_examples=examples, derandomize=True)(run)() + else: + settings(_SETTINGS, max_examples=examples, derandomize=False)(seed(chosen)(run))() + return tuple(drawn) diff --git a/packages/zarr-metadata/tests/testing/test_documents.py b/packages/zarr-metadata/tests/testing/test_documents.py index 53cd047724..41bd8eda65 100644 --- a/packages/zarr-metadata/tests/testing/test_documents.py +++ b/packages/zarr-metadata/tests/testing/test_documents.py @@ -8,7 +8,6 @@ from __future__ import annotations -import functools import json from typing import TYPE_CHECKING, Any, cast @@ -16,7 +15,7 @@ from hypothesis import find, given, settings from hypothesis import strategies as st -from tests.testing._shares import examples_of, share +from tests.testing._shares import share_of from zarr_metadata.definition import AcceptedDeclaration, Definition, resolve from zarr_metadata.definition._definition import canonicalize from zarr_metadata.model import ( @@ -340,29 +339,35 @@ def test_a_data_type_no_codec_takes_is_not_drawn(document: Mapping[str, Any]) -> assert document["data_type"] == "int8" +_ARRAY_DOCUMENTS = array_documents() +"""One strategy for every share, so `examples_of` draws its examples once.""" + _DOCUMENT_SHARES: dict[str, tuple[Callable[[Mapping[str, Any]], bool], float, float]] = { - "no axes": (lambda d: d["shape"] == (), 0.04, 0.34), - "more axes than five": (lambda d: len(d["shape"]) > 5, 0.15, 1), + "no axes": (lambda d: d["shape"] == (), 0.07, 0.31), + "more axes than five": (lambda d: len(d["shape"]) > 5, 0.16, 1), "a zero-length axis": (lambda d: 0 in d["shape"], 0.44, 1), - "an axis past 32 bits": (lambda d: any(n >= 2**32 for n in d["shape"]), 0.03, 1), - "the first data type": (lambda d: _name(d["data_type"]) == "bool", 0, 0.18), + "an axis past 32 bits": (lambda d: any(n >= 2**32 for n in d["shape"]), 0.02, 1), } """What a share of array documents holds, as a low and a high bound: what a test running only a few hundred examples meets. -Each bound is set as `_shares` says, from 100 seeds. +Each bound is set as `_shares` says. """ -@functools.cache -def _array_document_examples() -> list[Mapping[str, Any]]: - """Drawn once for every target, as each counts the same examples.""" - return examples_of(array_documents()) - - @pytest.mark.parametrize("target", _DOCUMENT_SHARES) def test_array_documents_share(target: str) -> None: counts, low, high = _DOCUMENT_SHARES[target] - assert low <= share(_array_document_examples(), counts) <= high + assert low <= share_of(_ARRAY_DOCUMENTS, counts) <= high + + +def test_array_documents_do_not_favor_the_first_data_type() -> None: + """`bool`, first of the data types by name, is drawn no more often than the rest would allow. + + Over 1000 examples: over 300, its share moves too much across seeds to + tell a fair draw from one that draws it a fifth of the time. + """ + share = share_of(_ARRAY_DOCUMENTS, lambda d: _name(d["data_type"]) == "bool", examples=1000) + assert share <= 0.13 def _empty_listings(document: Mapping[str, Any]) -> list[Any]: diff --git a/packages/zarr-metadata/tests/testing/test_shares.py b/packages/zarr-metadata/tests/testing/test_shares.py new file mode 100644 index 0000000000..d49c20143c --- /dev/null +++ b/packages/zarr-metadata/tests/testing/test_shares.py @@ -0,0 +1,26 @@ +"""The helpers that measure and bound how often a strategy draws something.""" + +from __future__ import annotations + +import itertools + +import pytest +from hypothesis import strategies as st + +from tests.testing._shares import bounds, examples_of + +_COUNTER = itertools.count() + + +def test_bounds() -> None: + """Past each extreme by a quarter of its distance from the median, rounded outward.""" + assert bounds([0.1, 0.2, 0.3]) == (0.07, 0.33) + assert bounds([0.3, 0.1, 0.2]) == (0.07, 0.33) + assert bounds([0.5, 0.5, 0.5]) == (0.5, 0.5) + assert bounds([0.0, 0.4, 0.4, 1.0]) == (-0.1, 1.15) + + +def test_a_strategy_that_draws_otherwise_on_a_second_run_is_an_assertion_error() -> None: + drifting = st.builds(lambda: next(_COUNTER)) + with pytest.raises(AssertionError, match="second derandomized run"): + examples_of(drifting, examples=5) diff --git a/packages/zarr-metadata/tests/testing/test_strategies.py b/packages/zarr-metadata/tests/testing/test_strategies.py index 6a7a66035e..97dd2075ee 100644 --- a/packages/zarr-metadata/tests/testing/test_strategies.py +++ b/packages/zarr-metadata/tests/testing/test_strategies.py @@ -1068,7 +1068,9 @@ def _float_class(value: object) -> str | None: ) -@pytest.mark.parametrize(("kind", "at_least"), [("normal", 0.2), ("NaN", 0.26), ("infinity", 0.01)]) +@pytest.mark.parametrize( + ("kind", "at_least"), [("normal", 0.22), ("NaN", 0.25), ("infinity", 0.01)] +) def test_hex_fill_values_are_not_only_small_numbers(kind: str, at_least: float) -> None: """Bits drawn as an integer alone are nearly all subnormal.""" assert share_of(_HEX_FILL_VALUES, lambda value: _float_class(value) == kind) >= at_least @@ -1108,7 +1110,7 @@ def _shard_inner_lengths(draw: st.DrawFn) -> tuple[int, ...]: def test_shards_of_a_rectilinear_grid_are_often_divided() -> None: """Edges drawn apart rarely share a factor, which leaves a shard only inner chunks of length 1.""" - assert share_of(_shard_inner_lengths(), lambda inner: any(n > 1 for n in inner)) >= 0.39 + assert share_of(_shard_inner_lengths(), lambda inner: any(n > 1 for n in inner)) >= 0.36 def _edge_factor(configuration: Mapping[str, Any]) -> int: @@ -1156,7 +1158,7 @@ def test_a_long_dimension_keeps_its_edge_factor() -> None: edges = chunk_grid_configurations(RECTILINEAR_CHUNK_GRID, (2**16 + 1,)).filter( lambda configuration: not isinstance(configuration["chunk_shapes"][0], int) ) - assert share_of(edges, lambda configuration: _edge_factor(configuration) > 1) >= 0.32 + assert share_of(edges, lambda configuration: _edge_factor(configuration) > 1) >= 0.36 def test_divisors_of_a_very_long_shard_are_found_quickly() -> None: @@ -1178,5 +1180,12 @@ def _field_types(declaration: Any) -> list[str]: # noqa: ANN401 def test_struct_fields_do_not_favor_the_first_data_type() -> None: - """A type drawn after a field's name would be `bool` a fifth of the time.""" - assert share_of(declarations(STRUCT_DATA_TYPE), lambda d: _field_types(d)[0] == "bool") <= 0.16 + """A type drawn after a field's name would be `bool` a fifth of the time. + + Over 1000 examples: over 300, the share of `bool` moves too much across + seeds to tell a fair draw from one that favors it. + """ + first_is_bool = share_of( + declarations(STRUCT_DATA_TYPE), lambda d: _field_types(d)[0] == "bool", examples=1000 + ) + assert first_is_bool <= 0.13