From 56729b616ab8937d4c306e651a341ef78883cb4f Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 14 Nov 2026 17:24:36 +0100 Subject: [PATCH 1/3] fix(zarr-metadata): draw the same codecs on every derandomized run `_codec` drew from `st.sampled_from` over `(definition, configurations)` pairs whose configurations are strategies built during the draw. Hypothesis labels such a `sampled_from` by hashing its elements, and a strategy hashes by address, so these spans were labeled by memory layout. Hypothesis's mutator copies choices between spans that share a label, so which spans it grouped, and so which examples a derandomized run drew, changed with whatever moved the allocator: another process, machine load. `_codec` now samples an index into the options, labeled by a `range`. The share tests measured this drift. `examples_of` now draws twice and fails if the runs differ, which catches an address-labeled strategy in the same process; the array-document targets share one draw. Bounds are reset from the shares of 100 seeds: the "no axes" ceiling (0.25) sat under the seed maximum (0.303), and the "axis past 32 bits" floor (0.1) over the seed minimum (0.06). Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- .../src/zarr_metadata/testing/_draw.py | 7 +- .../zarr-metadata/tests/testing/_shares.py | 68 +++++++++++++++---- .../tests/testing/test_documents.py | 26 ++++--- .../tests/testing/test_strategies.py | 8 +-- 4 files changed, 82 insertions(+), 27 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py b/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py index f2c0ec3c0b..e63d4afe1c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py @@ -278,7 +278,12 @@ def pipelines_for( def _codec( draw: st.DrawFn, options: Sequence[tuple[K, Configurations]] ) -> tuple[K, ZarrV3DeclarationJSON]: - definition, configurations = draw(st.sampled_from(options)) + # An index, not the options: Hypothesis labels a `sampled_from` by + # hashing its elements, and each option holds a strategy built for this + # draw, whose hash is its address. Labels that follow addresses send + # Hypothesis's mutator down different paths from one run to the next, so + # a derandomized run would not draw the same examples twice. + definition, configurations = options[draw(st.sampled_from(range(len(options))))] return definition, draw(any_spelling(definition, configurations)) diff --git a/packages/zarr-metadata/tests/testing/_shares.py b/packages/zarr-metadata/tests/testing/_shares.py index 27517e2653..3d70bf11aa 100644 --- a/packages/zarr-metadata/tests/testing/_shares.py +++ b/packages/zarr-metadata/tests/testing/_shares.py @@ -2,9 +2,15 @@ A reach test asks whether a target can be drawn at all, in as many examples as it takes, which says nothing of whether a test, at the 100 examples it -runs by default, ever meets it. Runs are derandomized, which keeps most shares -fixed, but a share can still move a few points between processes, so a bound -needs room: this is no place to bound what is drawn a few percent of the time. +runs by default, ever meets it. Runs are derandomized, so a share is a fixed +number for a given strategy. A change to the strategy draws other examples, +as another seed would, so a bound sits where the share stays across seeds: +this is no place to bound what is drawn a few percent of the time. + +A bound is set from the shares of 100 seeds, each run as `examples_of` +runs, but with `derandomize=False` under `hypothesis.seed`. It lies past +the most extreme share on its side by a quarter of that share's distance +from the median, rounded outward to a hundredth. """ from __future__ import annotations @@ -14,24 +20,58 @@ from hypothesis import HealthCheck, given, settings if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Sequence from hypothesis.strategies import SearchStrategy T = TypeVar("T") +_DERANDOMIZED = settings( + database=None, + derandomize=True, + deadline=None, + suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow], +) + + +def examples_of(strategy: SearchStrategy[T], *, examples: int = 300) -> list[T]: + """The examples a derandomized run of `strategy` draws, checked to be the same in a second run. + + Hypothesis's mutator copies spans between the parts of an example that + share a label, and a label can follow an object's address: a + `sampled_from` of elements that hold a strategy built for the draw is + labeled by that strategy's `id`. Such a strategy draws other examples + from run to run, and from machine to machine, however derandomized. A + second run in the same process lays objects out at other addresses, so + it catches this. + """ + first = _draw(strategy, examples) + second = _draw(strategy, examples) + if [repr(value) for value in first] != [repr(value) for value in second]: + raise AssertionError( + f"{strategy!r} drew other examples on a second derandomized run. " + "A strategy built during a draw is likely in a `sampled_from`, which " + "labels it by its address; sample an index into the options instead." + ) + return first + + +def share(values: Sequence[T], counts: Callable[[T], bool]) -> float: + """The share of `values` that `counts` holds of.""" + return sum(counts(value) for value in values) / len(values) + def share_of( strategy: SearchStrategy[T], counts: Callable[[T], bool], *, examples: int = 300 ) -> float: """The share of the examples of `strategy` that `counts` holds of.""" - tally: list[bool] = [] - run = settings( - max_examples=examples, - database=None, - derandomize=True, - deadline=None, - suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow], - )(given(strategy)(lambda value: tally.append(counts(value)))) - run() - return sum(tally) / len(tally) + return share(examples_of(strategy, examples=examples), counts) + + +def _draw(strategy: SearchStrategy[T], examples: int) -> list[T]: + drawn: list[T] = [] + # A lambda, not `drawn.append`: Hypothesis cannot wrap a builtin method, + # and derandomizes from the test function's source, so this line is the seed. + run = given(strategy)(lambda value: drawn.append(value)) + settings(_DERANDOMIZED, max_examples=examples)(run)() + return drawn diff --git a/packages/zarr-metadata/tests/testing/test_documents.py b/packages/zarr-metadata/tests/testing/test_documents.py index 6c20a3c29f..53cd047724 100644 --- a/packages/zarr-metadata/tests/testing/test_documents.py +++ b/packages/zarr-metadata/tests/testing/test_documents.py @@ -8,6 +8,7 @@ from __future__ import annotations +import functools import json from typing import TYPE_CHECKING, Any, cast @@ -15,7 +16,7 @@ from hypothesis import find, given, settings from hypothesis import strategies as st -from tests.testing._shares import share_of +from tests.testing._shares import examples_of, share from zarr_metadata.definition import AcceptedDeclaration, Definition, resolve from zarr_metadata.definition._definition import canonicalize from zarr_metadata.model import ( @@ -340,19 +341,28 @@ def test_a_data_type_no_codec_takes_is_not_drawn(document: Mapping[str, Any]) -> _DOCUMENT_SHARES: dict[str, tuple[Callable[[Mapping[str, Any]], bool], float, float]] = { - "no axes": (lambda d: d["shape"] == (), 0.03, 0.25), - "more axes than five": (lambda d: len(d["shape"]) > 5, 0.1, 1), - "a zero-length axis": (lambda d: 0 in d["shape"], 0.1, 1), - "an axis past 32 bits": (lambda d: any(n >= 2**32 for n in d["shape"]), 0.1, 1), - "the first data type": (lambda d: _name(d["data_type"]) == "bool", 0, 0.12), + "no axes": (lambda d: d["shape"] == (), 0.04, 0.34), + "more axes than five": (lambda d: len(d["shape"]) > 5, 0.15, 1), + "a zero-length axis": (lambda d: 0 in d["shape"], 0.44, 1), + "an axis past 32 bits": (lambda d: any(n >= 2**32 for n in d["shape"]), 0.03, 1), + "the first data type": (lambda d: _name(d["data_type"]) == "bool", 0, 0.18), } -"""What a share of array documents holds, as a low and a high bound: what a test running only a few hundred examples meets.""" +"""What a share of array documents holds, as a low and a high bound: what a test running only a few hundred examples meets. + +Each bound is set as `_shares` says, from 100 seeds. +""" + + +@functools.cache +def _array_document_examples() -> list[Mapping[str, Any]]: + """Drawn once for every target, as each counts the same examples.""" + return examples_of(array_documents()) @pytest.mark.parametrize("target", _DOCUMENT_SHARES) def test_array_documents_share(target: str) -> None: counts, low, high = _DOCUMENT_SHARES[target] - assert low <= share_of(array_documents(), counts) <= high + assert low <= share(_array_document_examples(), counts) <= high def _empty_listings(document: Mapping[str, Any]) -> list[Any]: diff --git a/packages/zarr-metadata/tests/testing/test_strategies.py b/packages/zarr-metadata/tests/testing/test_strategies.py index 8ef6023265..4d4a6d78d0 100644 --- a/packages/zarr-metadata/tests/testing/test_strategies.py +++ b/packages/zarr-metadata/tests/testing/test_strategies.py @@ -1057,7 +1057,7 @@ def _float_class(value: object) -> str | None: ) -@pytest.mark.parametrize(("kind", "at_least"), [("normal", 0.2), ("NaN", 0.1), ("infinity", 0.01)]) +@pytest.mark.parametrize(("kind", "at_least"), [("normal", 0.2), ("NaN", 0.26), ("infinity", 0.01)]) def test_hex_fill_values_are_not_only_small_numbers(kind: str, at_least: float) -> None: """Bits drawn as an integer alone are nearly all subnormal.""" assert share_of(_HEX_FILL_VALUES, lambda value: _float_class(value) == kind) >= at_least @@ -1097,7 +1097,7 @@ def _shard_inner_lengths(draw: st.DrawFn) -> tuple[int, ...]: def test_shards_of_a_rectilinear_grid_are_often_divided() -> None: """Edges drawn apart rarely share a factor, which leaves a shard only inner chunks of length 1.""" - assert share_of(_shard_inner_lengths(), lambda inner: any(n > 1 for n in inner)) >= 0.4 + assert share_of(_shard_inner_lengths(), lambda inner: any(n > 1 for n in inner)) >= 0.39 def _edge_factor(configuration: Mapping[str, Any]) -> int: @@ -1145,7 +1145,7 @@ def test_a_long_dimension_keeps_its_edge_factor() -> None: edges = chunk_grid_configurations(RECTILINEAR_CHUNK_GRID, (2**16 + 1,)).filter( lambda configuration: not isinstance(configuration["chunk_shapes"][0], int) ) - assert share_of(edges, lambda configuration: _edge_factor(configuration) > 1) >= 0.3 + assert share_of(edges, lambda configuration: _edge_factor(configuration) > 1) >= 0.32 def test_divisors_of_a_very_long_shard_are_found_quickly() -> None: @@ -1168,4 +1168,4 @@ def _field_types(declaration: Any) -> list[str]: # noqa: ANN401 def test_struct_fields_do_not_favor_the_first_data_type() -> None: """A type drawn after a field's name would be `bool` a fifth of the time.""" - assert share_of(declarations(STRUCT_DATA_TYPE), lambda d: _field_types(d)[0] == "bool") <= 0.12 + assert share_of(declarations(STRUCT_DATA_TYPE), lambda d: _field_types(d)[0] == "bool") <= 0.16 From 6ecbaebd520800c8b2e018d142e3f512f6f96fa5 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 14 Nov 2026 18:18:52 +0100 Subject: [PATCH 2/3] docs(zarr-metadata): add changelog fragment for #4635 Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- packages/zarr-metadata/changes/4635.bugfix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 packages/zarr-metadata/changes/4635.bugfix.md diff --git a/packages/zarr-metadata/changes/4635.bugfix.md b/packages/zarr-metadata/changes/4635.bugfix.md new file mode 100644 index 0000000000..d92c9b5193 --- /dev/null +++ b/packages/zarr-metadata/changes/4635.bugfix.md @@ -0,0 +1 @@ +Strategies in `zarr_metadata.testing.strategies` that draw codec pipelines, such as `array_documents` and `pipelines`, now draw the same examples on every derandomized Hypothesis run. A codec was sampled from options that hold strategies built during the draw, which Hypothesis labels by memory address, so the examples its mutator produced varied from one process to the next. From d587d31748cde5fe703b457280bd232874f9117c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 14 Nov 2026 21:42:52 +0100 Subject: [PATCH 3/3] fix(zarr-metadata): sample codec names and make share bounds reproducible `_codec` samples definition names rather than an index. An index into options of the same length shares one label with every other, so the mutator could copy a choice between unrelated codec slots; names key the label on the option set, as the chunk grid and data type draws already do. `examples_of` is cached per strategy, so the share tests that count one strategy draw it once, and its docstring calls the second-run check what it is, a tripwire. `across_seeds` and `bounds` put the rule the bounds follow into code, so a bound can be reset rather than re-derived; every bound is reset by it. The two first-data-type checks run over 1000 examples: over 300, `bool`'s share across seeds reached 0.13 to 0.15, too near the 0.2 they guard against. The changelog fragment no longer says the options' strategies are built during the draw: some are cached. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- packages/zarr-metadata/changes/4635.bugfix.md | 2 +- .../src/zarr_metadata/testing/_draw.py | 16 +-- .../zarr-metadata/tests/testing/_shares.py | 99 +++++++++++++------ .../tests/testing/test_documents.py | 35 ++++--- .../tests/testing/test_shares.py | 26 +++++ .../tests/testing/test_strategies.py | 19 +++- 6 files changed, 138 insertions(+), 59 deletions(-) create mode 100644 packages/zarr-metadata/tests/testing/test_shares.py diff --git a/packages/zarr-metadata/changes/4635.bugfix.md b/packages/zarr-metadata/changes/4635.bugfix.md index d92c9b5193..791809568b 100644 --- a/packages/zarr-metadata/changes/4635.bugfix.md +++ b/packages/zarr-metadata/changes/4635.bugfix.md @@ -1 +1 @@ -Strategies in `zarr_metadata.testing.strategies` that draw codec pipelines, such as `array_documents` and `pipelines`, now draw the same examples on every derandomized Hypothesis run. A codec was sampled from options that hold strategies built during the draw, which Hypothesis labels by memory address, so the examples its mutator produced varied from one process to the next. +Strategies in `zarr_metadata.testing.strategies` that draw codec pipelines, such as `array_documents` and `pipelines`, now draw the same examples on every derandomized Hypothesis run. A codec was sampled from options that hold strategies, which Hypothesis labels by their memory address, so the examples its mutator produced varied from one process to the next. diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py b/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py index e63d4afe1c..2c42d4f1b3 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py @@ -278,12 +278,16 @@ def pipelines_for( def _codec( draw: st.DrawFn, options: Sequence[tuple[K, Configurations]] ) -> tuple[K, ZarrV3DeclarationJSON]: - # An index, not the options: Hypothesis labels a `sampled_from` by - # hashing its elements, and each option holds a strategy built for this - # draw, whose hash is its address. Labels that follow addresses send - # Hypothesis's mutator down different paths from one run to the next, so - # a derandomized run would not draw the same examples twice. - definition, configurations = options[draw(st.sampled_from(range(len(options))))] + # A name, not an option: Hypothesis labels a `sampled_from` by hashing its + # elements, and an option holds a strategy, which hashes by its `id`. A + # label that follows where a strategy sits in memory sends Hypothesis's + # mutator down other paths from one run to the next, so a derandomized + # run would not draw the same examples twice. Names also keep one set of + # options apart from another, as an index into either would not. + named = { + definition.name: (definition, configurations) for definition, configurations in options + } + definition, configurations = named[draw(st.sampled_from(list(named)))] return definition, draw(any_spelling(definition, configurations)) diff --git a/packages/zarr-metadata/tests/testing/_shares.py b/packages/zarr-metadata/tests/testing/_shares.py index 3d70bf11aa..b69ee73a80 100644 --- a/packages/zarr-metadata/tests/testing/_shares.py +++ b/packages/zarr-metadata/tests/testing/_shares.py @@ -4,61 +4,55 @@ as it takes, which says nothing of whether a test, at the 100 examples it runs by default, ever meets it. Runs are derandomized, so a share is a fixed number for a given strategy. A change to the strategy draws other examples, -as another seed would, so a bound sits where the share stays across seeds: -this is no place to bound what is drawn a few percent of the time. +as another seed would, and so does a change to the line in `_draw` that +Hypothesis derandomizes from, which reseeds every share at once. So a bound +sits where the share stays across seeds: this is no place to bound what is +drawn a few percent of the time. -A bound is set from the shares of 100 seeds, each run as `examples_of` -runs, but with `derandomize=False` under `hypothesis.seed`. It lies past -the most extreme share on its side by a quarter of that share's distance -from the median, rounded outward to a hundredth. +A bound is reset with `bounds(across_seeds(strategy, statistic))`, which +runs 100 seeds and applies the rule `bounds` states. """ from __future__ import annotations -from typing import TYPE_CHECKING, TypeVar +import functools +import math +from typing import TYPE_CHECKING, Any, TypeVar, cast -from hypothesis import HealthCheck, given, settings +from hypothesis import HealthCheck, given, seed, settings if TYPE_CHECKING: - from collections.abc import Callable, Sequence + from collections.abc import Callable, Iterable, Sequence from hypothesis.strategies import SearchStrategy T = TypeVar("T") -_DERANDOMIZED = settings( +_SETTINGS = settings( database=None, - derandomize=True, deadline=None, suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow], ) -def examples_of(strategy: SearchStrategy[T], *, examples: int = 300) -> list[T]: - """The examples a derandomized run of `strategy` draws, checked to be the same in a second run. +def examples_of(strategy: SearchStrategy[T], *, examples: int = 300) -> tuple[T, ...]: + """The examples a derandomized run of `strategy` draws, drawn once per strategy and checked against a second run. Hypothesis's mutator copies spans between the parts of an example that share a label, and a label can follow an object's address: a - `sampled_from` of elements that hold a strategy built for the draw is - labeled by that strategy's `id`. Such a strategy draws other examples - from run to run, and from machine to machine, however derandomized. A - second run in the same process lays objects out at other addresses, so - it catches this. + `sampled_from` of elements that hold a strategy is labeled by that + strategy's `id`. Such a strategy draws other examples from run to run, + and from machine to machine, however derandomized. A second run in the + same process often lays objects out at other addresses, so it often + catches this, but it is a tripwire, not a proof. """ - first = _draw(strategy, examples) - second = _draw(strategy, examples) - if [repr(value) for value in first] != [repr(value) for value in second]: - raise AssertionError( - f"{strategy!r} drew other examples on a second derandomized run. " - "A strategy built during a draw is likely in a `sampled_from`, which " - "labels it by its address; sample an index into the options instead." - ) - return first + return cast("tuple[T, ...]", _checked(strategy, examples)) -def share(values: Sequence[T], counts: Callable[[T], bool]) -> float: +def share(values: Iterable[T], counts: Callable[[T], bool]) -> float: """The share of `values` that `counts` holds of.""" - return sum(counts(value) for value in values) / len(values) + tally = [counts(value) for value in values] + return sum(tally) / len(tally) def share_of( @@ -68,10 +62,51 @@ def share_of( return share(examples_of(strategy, examples=examples), counts) -def _draw(strategy: SearchStrategy[T], examples: int) -> list[T]: +def across_seeds( + strategy: SearchStrategy[T], + statistic: Callable[[Sequence[T]], float], + *, + seeds: Iterable[int] = range(100), + examples: int = 300, +) -> list[float]: + """`statistic` of the examples of `strategy` under each of `seeds`, as a change to the strategy would move it.""" + return [statistic(_draw(strategy, examples, chosen)) for chosen in seeds] + + +def bounds(statistics: Sequence[float]) -> tuple[float, float]: + """A low and a high bound for a statistic measured across seeds. + + Each lies past the most extreme measure on its side by a quarter of that + measure's distance from the median, rounded outward to a hundredth. + """ + ordered = sorted(statistics) + median = ordered[len(ordered) // 2] + low = ordered[0] - (median - ordered[0]) / 4 + high = ordered[-1] + (ordered[-1] - median) / 4 + return math.floor(round(low * 100, 6)) / 100, math.ceil(round(high * 100, 6)) / 100 + + +@functools.cache +def _checked(strategy: SearchStrategy[Any], examples: int) -> tuple[Any, ...]: + first = _draw(strategy, examples) + second = _draw(strategy, examples) + if [repr(value) for value in first] != [repr(value) for value in second]: + raise AssertionError( + f"{strategy!r} drew other examples on a second derandomized run. " + "A strategy is likely in a `sampled_from`, which labels it by its " + "address; sample a name or an index instead." + ) + return first + + +def _draw(strategy: SearchStrategy[T], examples: int, chosen: int | None = None) -> tuple[T, ...]: + """Derandomized examples of `strategy`, or seeded with `chosen`.""" drawn: list[T] = [] # A lambda, not `drawn.append`: Hypothesis cannot wrap a builtin method, # and derandomizes from the test function's source, so this line is the seed. run = given(strategy)(lambda value: drawn.append(value)) - settings(_DERANDOMIZED, max_examples=examples)(run)() - return drawn + if chosen is None: + settings(_SETTINGS, max_examples=examples, derandomize=True)(run)() + else: + settings(_SETTINGS, max_examples=examples, derandomize=False)(seed(chosen)(run))() + return tuple(drawn) diff --git a/packages/zarr-metadata/tests/testing/test_documents.py b/packages/zarr-metadata/tests/testing/test_documents.py index 53cd047724..6a0d153542 100644 --- a/packages/zarr-metadata/tests/testing/test_documents.py +++ b/packages/zarr-metadata/tests/testing/test_documents.py @@ -8,7 +8,6 @@ from __future__ import annotations -import functools import json from typing import TYPE_CHECKING, Any, cast @@ -16,7 +15,7 @@ from hypothesis import find, given, settings from hypothesis import strategies as st -from tests.testing._shares import examples_of, share +from tests.testing._shares import share_of from zarr_metadata.definition import AcceptedDeclaration, Definition, resolve from zarr_metadata.definition._definition import canonicalize from zarr_metadata.model import ( @@ -340,29 +339,35 @@ def test_a_data_type_no_codec_takes_is_not_drawn(document: Mapping[str, Any]) -> assert document["data_type"] == "int8" +_ARRAY_DOCUMENTS = array_documents() +"""One strategy for every share, so `examples_of` draws its examples once.""" + _DOCUMENT_SHARES: dict[str, tuple[Callable[[Mapping[str, Any]], bool], float, float]] = { - "no axes": (lambda d: d["shape"] == (), 0.04, 0.34), - "more axes than five": (lambda d: len(d["shape"]) > 5, 0.15, 1), - "a zero-length axis": (lambda d: 0 in d["shape"], 0.44, 1), - "an axis past 32 bits": (lambda d: any(n >= 2**32 for n in d["shape"]), 0.03, 1), - "the first data type": (lambda d: _name(d["data_type"]) == "bool", 0, 0.18), + "no axes": (lambda d: d["shape"] == (), 0.09, 0.3), + "more axes than five": (lambda d: len(d["shape"]) > 5, 0.2, 1), + "a zero-length axis": (lambda d: 0 in d["shape"], 0.45, 1), + "an axis past 32 bits": (lambda d: any(n >= 2**32 for n in d["shape"]), 0.02, 1), } """What a share of array documents holds, as a low and a high bound: what a test running only a few hundred examples meets. -Each bound is set as `_shares` says, from 100 seeds. +Each bound is set as `_shares` says. """ -@functools.cache -def _array_document_examples() -> list[Mapping[str, Any]]: - """Drawn once for every target, as each counts the same examples.""" - return examples_of(array_documents()) - - @pytest.mark.parametrize("target", _DOCUMENT_SHARES) def test_array_documents_share(target: str) -> None: counts, low, high = _DOCUMENT_SHARES[target] - assert low <= share(_array_document_examples(), counts) <= high + assert low <= share_of(_ARRAY_DOCUMENTS, counts) <= high + + +def test_array_documents_do_not_favor_the_first_data_type() -> None: + """`bool`, first of the data types by name, is drawn no more often than the rest would allow. + + Over 1000 examples: over 300, its share moves too much across seeds to + tell a fair draw from one that draws it a fifth of the time. + """ + share = share_of(_ARRAY_DOCUMENTS, lambda d: _name(d["data_type"]) == "bool", examples=1000) + assert share <= 0.1 def _empty_listings(document: Mapping[str, Any]) -> list[Any]: diff --git a/packages/zarr-metadata/tests/testing/test_shares.py b/packages/zarr-metadata/tests/testing/test_shares.py new file mode 100644 index 0000000000..d49c20143c --- /dev/null +++ b/packages/zarr-metadata/tests/testing/test_shares.py @@ -0,0 +1,26 @@ +"""The helpers that measure and bound how often a strategy draws something.""" + +from __future__ import annotations + +import itertools + +import pytest +from hypothesis import strategies as st + +from tests.testing._shares import bounds, examples_of + +_COUNTER = itertools.count() + + +def test_bounds() -> None: + """Past each extreme by a quarter of its distance from the median, rounded outward.""" + assert bounds([0.1, 0.2, 0.3]) == (0.07, 0.33) + assert bounds([0.3, 0.1, 0.2]) == (0.07, 0.33) + assert bounds([0.5, 0.5, 0.5]) == (0.5, 0.5) + assert bounds([0.0, 0.4, 0.4, 1.0]) == (-0.1, 1.15) + + +def test_a_strategy_that_draws_otherwise_on_a_second_run_is_an_assertion_error() -> None: + drifting = st.builds(lambda: next(_COUNTER)) + with pytest.raises(AssertionError, match="second derandomized run"): + examples_of(drifting, examples=5) diff --git a/packages/zarr-metadata/tests/testing/test_strategies.py b/packages/zarr-metadata/tests/testing/test_strategies.py index 4d4a6d78d0..04a5a51f4d 100644 --- a/packages/zarr-metadata/tests/testing/test_strategies.py +++ b/packages/zarr-metadata/tests/testing/test_strategies.py @@ -1057,7 +1057,9 @@ def _float_class(value: object) -> str | None: ) -@pytest.mark.parametrize(("kind", "at_least"), [("normal", 0.2), ("NaN", 0.26), ("infinity", 0.01)]) +@pytest.mark.parametrize( + ("kind", "at_least"), [("normal", 0.22), ("NaN", 0.25), ("infinity", 0.01)] +) def test_hex_fill_values_are_not_only_small_numbers(kind: str, at_least: float) -> None: """Bits drawn as an integer alone are nearly all subnormal.""" assert share_of(_HEX_FILL_VALUES, lambda value: _float_class(value) == kind) >= at_least @@ -1097,7 +1099,7 @@ def _shard_inner_lengths(draw: st.DrawFn) -> tuple[int, ...]: def test_shards_of_a_rectilinear_grid_are_often_divided() -> None: """Edges drawn apart rarely share a factor, which leaves a shard only inner chunks of length 1.""" - assert share_of(_shard_inner_lengths(), lambda inner: any(n > 1 for n in inner)) >= 0.39 + assert share_of(_shard_inner_lengths(), lambda inner: any(n > 1 for n in inner)) >= 0.36 def _edge_factor(configuration: Mapping[str, Any]) -> int: @@ -1145,7 +1147,7 @@ def test_a_long_dimension_keeps_its_edge_factor() -> None: edges = chunk_grid_configurations(RECTILINEAR_CHUNK_GRID, (2**16 + 1,)).filter( lambda configuration: not isinstance(configuration["chunk_shapes"][0], int) ) - assert share_of(edges, lambda configuration: _edge_factor(configuration) > 1) >= 0.32 + assert share_of(edges, lambda configuration: _edge_factor(configuration) > 1) >= 0.36 def test_divisors_of_a_very_long_shard_are_found_quickly() -> None: @@ -1167,5 +1169,12 @@ def _field_types(declaration: Any) -> list[str]: # noqa: ANN401 def test_struct_fields_do_not_favor_the_first_data_type() -> None: - """A type drawn after a field's name would be `bool` a fifth of the time.""" - assert share_of(declarations(STRUCT_DATA_TYPE), lambda d: _field_types(d)[0] == "bool") <= 0.16 + """A type drawn after a field's name would be `bool` a fifth of the time. + + Over 1000 examples: over 300, the share of `bool` moves too much across + seeds to tell a fair draw from one that favors it. + """ + first_is_bool = share_of( + declarations(STRUCT_DATA_TYPE), lambda d: _field_types(d)[0] == "bool", examples=1000 + ) + assert first_is_bool <= 0.13